diff --git a/TODO.txt b/TODO.txt
index 81f82d5..24f0cce 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -1,9 +1,8 @@
* document openbsd pcap wireshark fix
* printf '\x6c' | dd seek=20 bs=1 count=1 conv=notrunc of=test.dump
-* show quota status on preloads page
* add copyright footer
-* skip preloads page logging
-* sort preloads by status
* add stopped status?
* replace stop with preload link
* should redirect back to preloads page
+* config file?
+ * whitelist allowed src paths
diff --git a/cmd/cachefs/main.go b/cmd/cachefs/main.go
index 2b4386b..e7e3b66 100644
--- a/cmd/cachefs/main.go
+++ b/cmd/cachefs/main.go
@@ -29,7 +29,7 @@ var (
listenWebdav string
listenCache string
quota int64
- maxq int
+ max int
log logr.Logger
)
@@ -44,14 +44,14 @@ func main() {
flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
- flag.IntVar(&maxq, "maxq", -1, "max parallel preloads")
+ flag.IntVar(&max, "max", -1, "max parallel preloads")
flag.Int64Var("a, "quota", 1, "max disk usage quota in GiB")
flag.Parse()
log = klogr.New().WithName("main")
log.Info("starting cachefs", "version", version)
- filesystem, err := fs.NewFS(quota*gib, maxq, src, dst, metadata, klogr.New().WithName("fs"))
+ filesystem, err := fs.NewFS(quota*gib, max, src, dst, metadata, klogr.New().WithName("fs"))
if err != nil {
klog.Fatalf("init failed: %s", err)
}
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
index 72fb709..9e22389 100644
--- a/pkg/fs/fs.go
+++ b/pkg/fs/fs.go
@@ -4,139 +4,65 @@ package fs
import (
"context"
- "encoding/json"
"errors"
"io"
stdfs "io/fs"
- "math"
"net/http"
"os"
"path"
"path/filepath"
"strings"
- "sync"
- "time"
"github.com/go-logr/logr"
- "golang.org/x/exp/maps"
- "golang.org/x/exp/slices"
)
type FS struct {
- mu sync.RWMutex
- log logr.Logger
- NoCache http.Handler
- src string
- dst string
- statCancel func()
- flushCancel func()
- preloadCancel func()
- mdf *os.File
- jenc *json.Encoder
- dc *DirCache
- sc *StatCache
- q *Quota
- md MetadataMap
-
- pm map[string]*Preload
- prec chan *Preload
- pref chan string
- preq int
- maxq int
-}
-
-type Preload struct {
- Name string
+ log logr.Logger
cancel func()
- Status int
- Running bool
+ NoCache http.Handler
+ src string
+ dst string
+ dc *DirCache
+ sc *StatCache
+
+ q *Quota
+ mh *MetadataHandler
+ ph *PreloadHandler
}
-type Preloads []*Preload
-
-func (Preloads) Less(i, j *Preload) bool { return i.Name < j.Name }
-
-func NewFS(max int64, maxq int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
+func NewFS(quota int64, max int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
if !filepath.IsAbs(src) {
return nil, errors.New("src path is not absolute")
}
if !filepath.IsAbs(dst) {
return nil, errors.New("dst path is not absolute")
}
- if !filepath.IsAbs(metadata) {
- return nil, errors.New("metadata path is not absolute")
- }
if src == dst {
return nil, errors.New("src and dst path can not be equal")
}
- mdf, err := os.OpenFile(metadata, os.O_RDWR|os.O_CREATE, 0o640)
- if err != nil {
- return
- }
- statCtx, statCancel := context.WithCancel(context.Background())
- flushCtx, flushCancel := context.WithCancel(context.Background())
- preloadCtx, preloadCancel := context.WithCancel(context.Background())
- if maxq < -1 {
- maxq = -1
- }
+ ctx, cancel := context.WithCancel(context.Background())
fs = &FS{
- log: log,
- NoCache: http.FileServer(http.Dir(src)),
- src: src,
- dst: dst,
- mdf: mdf,
- statCancel: statCancel,
- flushCancel: flushCancel,
- preloadCancel: preloadCancel,
- dc: NewDirCache(),
- sc: NewStatCache(statCtx),
- md: make(MetadataMap),
- jenc: json.NewEncoder(mdf),
-
- pm: make(map[string]*Preload),
- prec: make(chan *Preload, 1),
- pref: make(chan string, 1),
- maxq: maxq,
+ log: log,
+ cancel: cancel,
+ NoCache: http.FileServer(http.Dir(src)),
+ src: src,
+ dst: dst,
+ dc: NewDirCache(),
}
- fs.q, err = NewQuota(max, fs, fs.log.WithName("quota"))
+
+ fs.mh, err = NewMetadataHandler(ctx, fs, metadata, dst, log.WithName("metadata"))
if err != nil {
return
}
- err = json.NewDecoder(mdf).Decode(&fs.md)
- if err == io.EOF {
- err = nil
+ fs.q, err = NewQuota(fs.mh.Size(), quota, fs, log.WithName("quota"))
+ if err != nil {
+ return
}
- fs.initMetadata()
- fs.q.Init()
- go fs.flusher(flushCtx)
- go fs.preload(preloadCtx)
- go fs.preloadFinish(preloadCtx)
- return fs, err
-}
-
-func (fs *FS) initMetadata() {
- log := fs.log
- maps.DeleteFunc(fs.md, func(k string, v *Metadata) bool {
- _, err := fs.statDst(k)
- if errors.Is(err, os.ErrNotExist) {
- log.V(2).Info("removing not existing file from metadata", "file", k)
- return true
- }
- if len(v.Chunks) == 0 {
- log.V(2).Info("removing empty file", "file", k)
- err := fs.RemoveDst(k)
- if err != nil {
- log.Error(err, "error removing empty file", "file", k)
- return false
- }
- return true
- }
- v.fs = fs
- v.name = k
- return false
- })
+ fs.ph = NewPreloadHandler(ctx, fs, max, log.WithName("preload"))
+ fs.sc = NewStatCache(ctx)
+ return
}
func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
@@ -193,9 +119,9 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
return
}
- md := fs.metadata(name, fi.Size())
+ md := fs.mh.Metadata(name, fi.Size())
f = &File{
- log: log,
+ log: log.WithName("file"),
f: file,
md: md,
offline: true,
@@ -203,104 +129,20 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
return
}
-func (fs *FS) CancelPreload(name string) {
- fs.mu.Lock()
- defer fs.mu.Unlock()
- if p, ok := fs.pm[name]; ok && p.Running {
- p.cancel()
- }
+func (fs *FS) QuotaUsage() (cur, max int64) {
+ return fs.q.Usage()
}
-func (fs *FS) Preloads() (ps Preloads) {
- fs.mu.RLock()
- defer fs.mu.RUnlock()
- for _, p := range fs.pm {
- ps = append(ps, &Preload{
- Name: p.Name,
- Running: p.Running,
- Status: fs.CacheStatus(p.Name),
- })
- }
- slices.SortFunc(ps, ps.Less)
- return
+func (fs *FS) CancelPreload(name string) {
+ fs.ph.CancelPreload(name)
+}
+
+func (fs *FS) Preloads() Preloads {
+ return fs.ph.Preloads()
}
func (fs *FS) Preload(name string) {
- fs.mu.Lock()
- defer fs.mu.Unlock()
-
- p, ok := fs.pm[name]
- if !ok {
- p = &Preload{Name: name}
- fs.pm[name] = p
- if fs.maxq == -1 || fs.preq < fs.maxq {
- fs.prec <- p
- }
- return
- }
-
- if p.Running {
- fs.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
- }
- return
-}
-
-func (fs *FS) nextPreload() {
- if fs.preq == fs.maxq {
- return
- }
- for _, p := range fs.pm {
- if p.Running {
- continue
- }
- fs.prec <- p
- return
- }
-}
-
-func (fs *FS) preload(ctx context.Context) {
- var p *Preload
- for {
- select {
- case <-ctx.Done():
- return
- case p = <-fs.prec:
- }
- name := p.Name
- file, err := fs.Open(name)
- if err != nil {
- fs.log.Error(err, "error staring next preload", "file", name)
- return
- }
- f, ok := file.(*File)
- if !ok {
- continue
- }
- ctx, cancel := context.WithCancel(context.Background())
- fs.mu.Lock()
- fs.preq++
- p.Running = true
- p.cancel = cancel
- fs.mu.Unlock()
- go f.Preload(ctx, func() {
- fs.pref <- name
- })
- }
-}
-
-func (fs *FS) preloadFinish(ctx context.Context) {
- for {
- select {
- case <-ctx.Done():
- return
- case name := <-fs.pref:
- fs.mu.Lock()
- delete(fs.pm, name)
- fs.preq--
- fs.nextPreload()
- fs.mu.Unlock()
- }
- }
+ fs.ph.Preload(name)
}
func skipLog(name string) bool {
@@ -355,9 +197,9 @@ func (fs *FS) Open(name string) (f http.File, err error) {
if offline {
log.V(2).Info("file offline mode", "path", dp)
}
- md := fs.metadata(name, fi.Size())
+ md := fs.mh.Metadata(name, fi.Size())
f = &File{
- log: log,
+ log: log.WithName("file"),
f: file,
md: md,
offline: offline,
@@ -401,124 +243,12 @@ func (fs *FS) openCacheFile(name string, size int64) (df *os.File, err error) {
return
}
-func (fs *FS) metadata(name string, size int64) (md *Metadata) {
- fs.mu.Lock()
- defer fs.mu.Unlock()
- if md, ok := fs.md[name]; ok {
- if size > 0 && md.Size != size {
- log := fs.log.WithValues("file", name)
- log.V(2).Info("source file changed: deleting cache file", "src", size, "dst", md.Size)
- err := md.Delete()
- if err != nil {
- log.Error(err, "error deleting cache file")
- }
- }
- return md
- }
- md = &Metadata{Size: size, fs: fs, name: name}
- fs.md[name] = md
- return
-}
-
func (fs *FS) CacheStatus(name string) int {
- fs.mu.RLock()
- defer fs.mu.RUnlock()
- if md, ok := fs.md[name]; ok {
- size := md.Chunks.Size()
- if size == 0 {
- return -1
- }
- return int(math.Round(float64(size) / float64(md.Size) * 100))
- }
- return -1
-}
-
-func (fs *FS) flusher(ctx context.Context) {
- for {
- select {
- case <-ctx.Done():
- return
- case <-time.After(time.Minute * 5):
- }
- fs.cleanupEmptyDirs()
- fs.flushMetadata()
- }
-}
-
-func (fs *FS) cleanupEmptyDirs() {
- filepath.WalkDir(fs.dst, func(path string, d stdfs.DirEntry, err error) error {
- if err == nil && d.IsDir() {
- if path == fs.dst {
- return nil
- }
- empty, err := dirEmpty(path)
- if err != nil || !empty {
- return nil
- }
- log := fs.log.WithValues("dir", path)
- log.V(2).Info("removing empty dir")
- err = os.Remove(path)
- if err != nil {
- log.Error(err, "failed to remove empty dir")
- }
- }
- return nil
- })
-}
-
-func (fs *FS) flushMetadata() {
- log := fs.log
- fs.mu.Lock()
- defer fs.mu.Unlock()
- log.V(2).Info("flushing metadata to disk")
- md := maps.Clone(fs.md)
- maps.DeleteFunc(md, func(_ string, v *Metadata) bool {
- return len(v.Chunks) == 0
- })
- _, err := fs.mdf.Seek(0, io.SeekStart)
- if err != nil {
- log.Error(err, "failure seeking metadata file")
- return
- }
- err = fs.mdf.Truncate(0)
- if err != nil {
- log.Error(err, "failure truncating metadata file")
- return
- }
- err = fs.jenc.Encode(md)
- if err != nil {
- log.Error(err, "failure flushing metadata file")
- }
+ return fs.mh.CacheStatus(name)
}
func (fs *FS) Close() {
- fs.mu.Lock()
- fs.preloadCancel()
- fs.flushCancel()
- fs.statCancel()
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
- go func() {
- for {
- select {
- case <-ctx.Done():
- return
- case <-fs.prec:
- case <-fs.pref:
- }
- }
- }()
- for _, p := range fs.pm {
- if p.Running {
- p.cancel()
- }
- }
- for _, md := range fs.md {
- md.f.Sync()
- md.f.Close()
- }
- fs.mu.Unlock()
- fs.flushMetadata()
+ fs.cancel()
}
func dirEmpty(name string) (bool, error) {
diff --git a/pkg/fs/metadata.go b/pkg/fs/metadata.go
index a59e0a2..1827148 100644
--- a/pkg/fs/metadata.go
+++ b/pkg/fs/metadata.go
@@ -3,17 +3,220 @@
package fs
import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ stdfs "io/fs"
+ "math"
"os"
+ "path/filepath"
"sync"
"sync/atomic"
"time"
+
+ "github.com/go-logr/logr"
+ "golang.org/x/exp/maps"
)
func now() int64 {
return time.Now().UTC().Unix()
}
-type MetadataMap map[string]*Metadata
+type MetadataHandler struct {
+ mu sync.RWMutex
+ log logr.Logger
+ md map[string]*Metadata
+ fs *FS
+ dst string
+ f *os.File
+ enc *json.Encoder
+}
+
+func NewMetadataHandler(ctx context.Context, fs *FS, file, dst string, log logr.Logger) (mh *MetadataHandler, err error) {
+ if !filepath.IsAbs(file) {
+ return nil, errors.New("metadata path is not absolute")
+ }
+ mh = &MetadataHandler{
+ log: log,
+ md: make(map[string]*Metadata),
+ fs: fs,
+ dst: dst,
+ }
+ mh.f, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640)
+ if err != nil {
+ return
+ }
+ err = json.NewDecoder(mh.f).Decode(&mh.md)
+ if err != nil && err == io.EOF {
+ return
+ }
+ mh.enc = json.NewEncoder(mh.f)
+ err = nil
+ mh.init()
+ go mh.flusher(ctx)
+ return
+}
+
+func (mh *MetadataHandler) DeleteOldest() (s int64) {
+ mh.mu.Lock()
+ defer mh.mu.Unlock()
+ atime := int64(math.MaxInt64)
+ var m *Metadata
+ for _, v := range mh.md {
+ if v.Atime != 0 && v.f != nil && v.Atime < atime {
+ atime = v.Atime
+ m = v
+ }
+ }
+ if m != nil {
+ log := mh.log.WithValues("file", m.name)
+ log.Info("deleting oldest file")
+ err := m.Delete()
+ if err != nil {
+ log.Error(err, "error deleting oldest file")
+ return
+ }
+ s = m.Size
+ }
+ return
+}
+
+func (mh *MetadataHandler) Size() (s int64) {
+ mh.mu.RLock()
+ defer mh.mu.RUnlock()
+ for _, md := range mh.md {
+ s += md.Chunks.Size()
+ }
+ return
+}
+
+func (mh *MetadataHandler) CacheStatus(name string) int {
+ mh.mu.RLock()
+ md, ok := mh.md[name]
+ mh.mu.RUnlock()
+ if !ok {
+ return -1
+ }
+ size := md.Chunks.Size()
+ if size == 0 {
+ return -1
+ }
+ return int(math.Round(float64(size) / float64(md.Size) * 100))
+}
+
+func (mh *MetadataHandler) Metadata(name string, size int64) (md *Metadata) {
+ mh.mu.Lock()
+ defer mh.mu.Unlock()
+ if md, ok := mh.md[name]; ok {
+ if size > 0 && md.Size != size {
+ log := mh.log.WithValues("file", name)
+ log.V(2).Info("source file changed: deleting cache file", "src", size, "dst", md.Size)
+ err := md.Delete()
+ if err != nil {
+ log.Error(err, "error deleting cache file")
+ }
+ }
+ return md
+ }
+ md = &Metadata{Size: size, fs: mh.fs, name: name}
+ mh.md[name] = md
+ return
+}
+
+func (mh *MetadataHandler) init() {
+ log := mh.log
+ maps.DeleteFunc(mh.md, func(k string, v *Metadata) bool {
+ _, err := mh.fs.statDst(k)
+ if errors.Is(err, os.ErrNotExist) {
+ log.V(2).Info("removing not existing file from metadata", "file", k)
+ return true
+ }
+ if len(v.Chunks) == 0 {
+ log.V(2).Info("removing empty file", "file", k)
+ err := mh.fs.RemoveDst(k)
+ if err != nil {
+ log.Error(err, "error removing empty file", "file", k)
+ return false
+ }
+ return true
+ }
+ v.fs = mh.fs
+ v.name = k
+ return false
+ })
+}
+
+func (mh *MetadataHandler) flusher(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ mh.close()
+ return
+ case <-time.After(time.Minute * 5):
+ }
+ mh.cleanupEmptyDirs()
+ mh.flush()
+ }
+}
+
+func (mh *MetadataHandler) cleanupEmptyDirs() {
+ filepath.WalkDir(mh.dst, func(path string, d stdfs.DirEntry, err error) error {
+ if err == nil && d.IsDir() {
+ if path == mh.dst {
+ return nil
+ }
+ empty, err := dirEmpty(path)
+ if err != nil || !empty {
+ return nil
+ }
+ log := mh.log.WithValues("dir", path)
+ log.V(2).Info("removing empty dir")
+ err = os.Remove(path)
+ if err != nil {
+ log.Error(err, "failed to remove empty dir")
+ }
+ }
+ return nil
+ })
+}
+
+func (mh *MetadataHandler) flush() {
+ log := mh.log
+ mh.mu.Lock()
+ defer mh.mu.Unlock()
+ log.V(2).Info("flushing metadata to disk")
+ md := maps.Clone(mh.md)
+ maps.DeleteFunc(mh.md, func(_ string, v *Metadata) bool {
+ return len(v.Chunks) == 0
+ })
+ _, err := mh.f.Seek(0, io.SeekStart)
+ if err != nil {
+ log.Error(err, "failure seeking metadata file")
+ return
+ }
+ err = mh.f.Truncate(0)
+ if err != nil {
+ log.Error(err, "failure truncating metadata file")
+ return
+ }
+ err = mh.enc.Encode(md)
+ if err != nil {
+ log.Error(err, "failure flushing metadata file")
+ }
+ return
+}
+
+func (mh *MetadataHandler) close() {
+ mh.mu.Lock()
+ for _, md := range mh.md {
+ md.f.Sync()
+ md.f.Close()
+ }
+ mh.mu.Unlock()
+ mh.flush()
+ mh.f.Close()
+}
type Metadata struct {
mu sync.RWMutex `json:"-"`
diff --git a/pkg/fs/preload.go b/pkg/fs/preload.go
new file mode 100644
index 0000000..664f854
--- /dev/null
+++ b/pkg/fs/preload.go
@@ -0,0 +1,168 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package fs
+
+import (
+ "context"
+ "errors"
+ "sync"
+
+ "github.com/go-logr/logr"
+ "golang.org/x/exp/slices"
+)
+
+type Preload struct {
+ Name string
+ cancel func()
+ Status int
+ Running bool
+}
+
+type Preloads []*Preload
+
+func (Preloads) Less(i, j *Preload) bool { return i.Name < j.Name }
+
+type PreloadHandler struct {
+ mu sync.RWMutex
+ log logr.Logger
+ fs *FS
+ pm map[string]*Preload
+ sched chan *Preload
+ fin chan string
+ pre int
+ max int
+}
+
+func NewPreloadHandler(ctx context.Context, fs *FS, max int, log logr.Logger) (ph *PreloadHandler) {
+ if max < -1 {
+ max = -1
+ }
+ ph = &PreloadHandler{
+ log: log,
+ fs: fs,
+ pm: make(map[string]*Preload),
+ sched: make(chan *Preload, 1),
+ fin: make(chan string, 1),
+ max: max,
+ }
+ go ph.preload(ctx)
+ go ph.preloadFinish(ctx)
+ return
+}
+
+func (ph *PreloadHandler) CancelPreload(name string) {
+ ph.mu.Lock()
+ defer ph.mu.Unlock()
+ if p, ok := ph.pm[name]; ok && p.Running {
+ p.cancel()
+ }
+}
+
+func (ph *PreloadHandler) Preloads() (ps Preloads) {
+ ph.mu.RLock()
+ defer ph.mu.RUnlock()
+ for _, p := range ph.pm {
+ ps = append(ps, &Preload{
+ Name: p.Name,
+ Running: p.Running,
+ Status: ph.fs.CacheStatus(p.Name),
+ })
+ }
+ slices.SortFunc(ps, ps.Less)
+ return
+}
+
+func (ph *PreloadHandler) Preload(name string) {
+ ph.mu.Lock()
+ defer ph.mu.Unlock()
+
+ p, ok := ph.pm[name]
+ if !ok {
+ p = &Preload{Name: name}
+ ph.pm[name] = p
+ if ph.max == -1 || ph.pre < ph.max {
+ ph.sched <- p
+ }
+ return
+ }
+
+ if p.Running {
+ ph.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
+ }
+ return
+}
+
+func (ph *PreloadHandler) nextPreload() {
+ if ph.pre == ph.max {
+ return
+ }
+ for _, p := range ph.pm {
+ if p.Running {
+ continue
+ }
+ ph.sched <- p
+ return
+ }
+}
+
+func (ph *PreloadHandler) preload(ctx context.Context) {
+ var p *Preload
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case p = <-ph.sched:
+ }
+ name := p.Name
+ file, err := ph.fs.Open(name)
+ if err != nil {
+ ph.log.Error(err, "error staring next preload", "file", name)
+ return
+ }
+ f, ok := file.(*File)
+ if !ok {
+ continue
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ ph.mu.Lock()
+ ph.pre++
+ p.Running = true
+ p.cancel = cancel
+ ph.mu.Unlock()
+ go f.Preload(ctx, func() {
+ ph.fin <- name
+ })
+ }
+}
+
+func (ph *PreloadHandler) preloadFinish(ctx context.Context) {
+ for {
+ select {
+ case <-ctx.Done():
+ ph.close()
+ return
+ case name := <-ph.fin:
+ ph.mu.Lock()
+ delete(ph.pm, name)
+ ph.pre--
+ ph.nextPreload()
+ ph.mu.Unlock()
+ }
+ }
+}
+
+func (ph *PreloadHandler) close() {
+ for _, p := range ph.pm {
+ if p.Running {
+ p.cancel()
+ }
+ }
+ for {
+ select {
+ case <-ph.sched:
+ case <-ph.fin:
+ default:
+ return
+ }
+ }
+}
diff --git a/pkg/fs/quota.go b/pkg/fs/quota.go
index ae16b80..8b68e01 100644
--- a/pkg/fs/quota.go
+++ b/pkg/fs/quota.go
@@ -4,7 +4,7 @@ package fs
import (
"errors"
- "math"
+ "sync"
"sync/atomic"
"github.com/go-logr/logr"
@@ -13,52 +13,37 @@ import (
var InvalidMaxQuota = errors.New("invalid max quota")
type Quota struct {
+ mu sync.Mutex
log logr.Logger
fs *FS
- max int64
cur int64
+ max int64
}
-func NewQuota(max int64, fs *FS, log logr.Logger) (q *Quota, err error) {
+func NewQuota(cur, max int64, fs *FS, log logr.Logger) (q *Quota, err error) {
if max <= 0 {
err = InvalidMaxQuota
return
}
- q = &Quota{log: log, fs: fs, max: max}
+ q = &Quota{log: log, fs: fs, cur: cur, max: max}
+ q.log.Info("quota usage", "current", q.cur, "max", q.max)
return
}
-func (q *Quota) Init() {
- for _, v := range q.fs.md {
- q.cur += v.Chunks.Size()
- }
- q.log.Info("quota usage", "current", q.cur, "max", q.max)
+func (q *Quota) Usage() (cur, max int64) {
+ cur = atomic.LoadInt64(&q.cur)
+ max = q.max
+ return
}
func (q *Quota) cleanup() {
- if !q.fs.mu.TryLock() {
+ if !q.mu.TryLock() {
return
}
- q.log.Info("quota usage", "current", q.cur, "max", q.max)
- defer q.fs.mu.Unlock()
- atime := int64(math.MaxInt64)
- var m *Metadata
- for _, v := range q.fs.md {
- if v.Atime != 0 && v.f != nil && v.Atime < atime {
- atime = v.Atime
- m = v
- }
- }
- if m != nil {
- log := q.log.WithValues("file", m.name)
- log.Info("deleting oldest file")
- err := m.Delete()
- if err != nil {
- q.log.Error(err, "error deleting oldest file")
- return
- }
- atomic.AddInt64(&q.cur, -m.Size)
- }
+ q.log.Info("quota usage", "current", atomic.LoadInt64(&q.cur), "max", q.max)
+ defer q.mu.Unlock()
+ size := q.fs.mh.DeleteOldest()
+ atomic.AddInt64(&q.cur, -size)
}
func (q *Quota) Add(n int) {
diff --git a/pkg/srv/interceptor.go b/pkg/srv/interceptor.go
index 2c7ef6f..bf058a0 100644
--- a/pkg/srv/interceptor.go
+++ b/pkg/srv/interceptor.go
@@ -139,7 +139,9 @@ func getPreloads(path string, fs *fs.FS) (dc dirContents) {
Running: p.Running,
})
}
- slices.SortFunc(dc.Files, dc.Files.Less)
+ slices.SortFunc(dc.Files, func(i, _ file) bool {
+ return i.Running
+ })
return
}
diff --git a/pkg/srv/srv.go b/pkg/srv/srv.go
index e8cde9a..6362327 100644
--- a/pkg/srv/srv.go
+++ b/pkg/srv/srv.go
@@ -6,6 +6,7 @@ import (
"errors"
"io"
stdfs "io/fs"
+ "math"
"net/http"
"path"
"strings"
@@ -19,8 +20,16 @@ const (
csp = "Content-Security-Policy"
indexCSP = "default-src 'none';style-src 'unsafe-inline';frame-ancestors 'none'"
videoCSP = indexCSP + ";script-src 'sha256-ZLSc/s5/US9uVMMZOJ/yWiS1tj9nEoi+5Qoohy3QetU=';media-src 'self'"
+
+ gib = 1024 * 1024 * 1024
)
+type data struct {
+ QuotaCur float64
+ QuotaMax float64
+ Paths dirContents
+}
+
type FileServer struct {
log logr.Logger
fs *fs.FS
@@ -32,7 +41,8 @@ func NewFileServer(fs *fs.FS, log logr.Logger) http.Handler {
}
func skipLog(path string) bool {
- return strings.HasSuffix(path, "favicon.ico")
+ return path == "/?o=preloads" ||
+ strings.HasSuffix(path, "favicon.ico")
}
func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -67,7 +77,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h := w.Header()
if option == "v" {
h.Set(csp, videoCSP)
- err = video.Execute(w, r.URL.Path)
+ err = video.Execute(w, data{Paths: dirContents{Base: r.URL.Path}})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
@@ -82,6 +92,10 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
fs.fs.CancelPreload(p)
}
rdir := "/"
+ if r.FormValue("r") == "preloads" {
+ http.Redirect(w, r, rdir+"?o=preloads", http.StatusFound)
+ return
+ }
if i := strings.LastIndex(p, "/"); i > 0 {
rdir = p[:i]
}
@@ -97,7 +111,12 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if option == "preloads" {
h.Set(csp, indexCSP)
- err = preloads.Execute(w, getPreloads(p, fs.fs))
+ cur, max := fs.fs.QuotaUsage()
+ err = preloads.Execute(w, data{
+ QuotaCur: math.Round(float64(cur)/gib*100) / 100,
+ QuotaMax: float64(max) / gib,
+ Paths: getPreloads(p, fs.fs),
+ })
if err != nil {
fs.log.Error(err, "error rendering preloads")
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -124,7 +143,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.Set(csp, indexCSP)
w.WriteHeader(i.Status())
- err = index.Execute(w, paths)
+ err = index.Execute(w, data{Paths: paths})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
diff --git a/pkg/srv/templates.go b/pkg/srv/templates.go
index 5707a0c..45fd526 100644
--- a/pkg/srv/templates.go
+++ b/pkg/srv/templates.go
@@ -2,231 +2,39 @@
package srv
-import "html/template"
+import (
+ "embed"
+ "html/template"
+ "io/fs"
+)
+
+//go:embed templates/*
+var templates embed.FS
+
+func subMust(fsys fs.FS, dir string) fs.FS {
+ fsys, err := fs.Sub(fsys, dir)
+ if err != nil {
+ panic("sub: " + err.Error())
+ }
+ return fsys
+}
+
+var tsub = subMust(templates, "templates")
var (
- indexHead = `
-
-
-
-
-
-
- cachefs | {{.Base}}
-`
-
- index = template.Must(template.New("index").Parse(indexHead + `
-
- [v]: show video
-[n]: skip file caching
-[p]: preload file
-[Preloads]
-
-
- | ../ |
- [dir] |
-
- |
-{{range $s := .Dirs -}}
-
- | {{$s.Name}} |
- [dir] |
-
- |
-{{end -}}
-{{range $s := .Files -}}
-
- | {{$s.Name}} |
-
- {{if ge $s.Status 0}}{{$s.Status}}%{{end}}[v][n][p]
- |
-
- |
-{{if ge $s.Status 0}}
-
-{{end -}}
- |
-{{end -}}
-
-
-
-`))
- preloads = template.Must(template.New("preloads").Parse(indexHead + `
-
- [v]: show video
-[s]: stop preloading
-
-
- | / |
- [dir] |
-
-{{range $s := .Files -}}
-
- | {{$s.Name}} |
-
- {{if ge $s.Status 0}}{{$s.Status}}%{{end}}[running{{else}}yellow">[queued{{end}}][v][s]
- |
-
- |
-{{if ge $s.Status 0}}
-
-{{end -}}
- |
-{{end -}}
-
-
-
-