major refactoring

TODO.txt

@@ -1,9 +1,8 @@
 * document openbsd pcap wireshark fix
   * printf '\x6c' | dd seek=20 bs=1 count=1 conv=notrunc of=test.dump
-* show quota status on preloads page
 * add copyright footer
-* skip preloads page logging
-* sort preloads by status
 * add stopped status?
   * replace stop with preload link
   * should redirect back to preloads page
+* config file?
+  * whitelist allowed src paths

cmd/cachefs/main.go

@@ -29,7 +29,7 @@ var (
 	listenWebdav string
 	listenCache  string
 	quota        int64
-	maxq         int
+	max          int
 
 	log logr.Logger
 )
@@ -44,14 +44,14 @@ func main() {
 	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
 	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
 	flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
-	flag.IntVar(&maxq, "maxq", -1, "max parallel preloads")
+	flag.IntVar(&max, "max", -1, "max parallel preloads")
 	flag.Int64Var(&quota, "quota", 1, "max disk usage quota in GiB")
 	flag.Parse()
 
 	log = klogr.New().WithName("main")
 	log.Info("starting cachefs", "version", version)
 
-	filesystem, err := fs.NewFS(quota*gib, maxq, src, dst, metadata, klogr.New().WithName("fs"))
+	filesystem, err := fs.NewFS(quota*gib, max, src, dst, metadata, klogr.New().WithName("fs"))
 	if err != nil {
 		klog.Fatalf("init failed: %s", err)
 	}

pkg/fs/fs.go

@@ -4,139 +4,65 @@ package fs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"io"
 	stdfs "io/fs"
-	"math"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
-	"time"
 
 	"github.com/go-logr/logr"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 )
 
 type FS struct {
-	mu            sync.RWMutex
 	log     logr.Logger
+	cancel  func()
 	NoCache http.Handler
 	src     string
 	dst     string
-	statCancel    func()
-	flushCancel   func()
-	preloadCancel func()
-	mdf           *os.File
-	jenc          *json.Encoder
 	dc      *DirCache
 	sc      *StatCache
+
 	q  *Quota
-	md            MetadataMap
+	mh *MetadataHandler
-
+	ph *PreloadHandler
-	pm   map[string]*Preload
-	prec chan *Preload
-	pref chan string
-	preq int
-	maxq int
 }
 
-type Preload struct {
+func NewFS(quota int64, max int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
-	Name    string
-	cancel  func()
-	Status  int
-	Running bool
-}
-
-type Preloads []*Preload
-
-func (Preloads) Less(i, j *Preload) bool { return i.Name < j.Name }
-
-func NewFS(max int64, maxq int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
 	if !filepath.IsAbs(src) {
 		return nil, errors.New("src path is not absolute")
 	}
 	if !filepath.IsAbs(dst) {
 		return nil, errors.New("dst path is not absolute")
 	}
-	if !filepath.IsAbs(metadata) {
-		return nil, errors.New("metadata path is not absolute")
-	}
 	if src == dst {
 		return nil, errors.New("src and dst path can not be equal")
 	}
-	mdf, err := os.OpenFile(metadata, os.O_RDWR|os.O_CREATE, 0o640)
-	if err != nil {
-		return
-	}
-	statCtx, statCancel := context.WithCancel(context.Background())
-	flushCtx, flushCancel := context.WithCancel(context.Background())
-	preloadCtx, preloadCancel := context.WithCancel(context.Background())
 
-	if maxq < -1 {
+	ctx, cancel := context.WithCancel(context.Background())
-		maxq = -1
-	}
 
 	fs = &FS{
 		log:     log,
+		cancel:  cancel,
 		NoCache: http.FileServer(http.Dir(src)),
 		src:     src,
 		dst:     dst,
-		mdf:           mdf,
-		statCancel:    statCancel,
-		flushCancel:   flushCancel,
-		preloadCancel: preloadCancel,
 		dc:      NewDirCache(),
-		sc:            NewStatCache(statCtx),
-		md:            make(MetadataMap),
-		jenc:          json.NewEncoder(mdf),
-
-		pm:   make(map[string]*Preload),
-		prec: make(chan *Preload, 1),
-		pref: make(chan string, 1),
-		maxq: maxq,
 	}
-	fs.q, err = NewQuota(max, fs, fs.log.WithName("quota"))
+
+	fs.mh, err = NewMetadataHandler(ctx, fs, metadata, dst, log.WithName("metadata"))
 	if err != nil {
 		return
 	}
-	err = json.NewDecoder(mdf).Decode(&fs.md)
+	fs.q, err = NewQuota(fs.mh.Size(), quota, fs, log.WithName("quota"))
-	if err == io.EOF {
-		err = nil
-	}
-	fs.initMetadata()
-	fs.q.Init()
-	go fs.flusher(flushCtx)
-	go fs.preload(preloadCtx)
-	go fs.preloadFinish(preloadCtx)
-	return fs, err
-}
-
-func (fs *FS) initMetadata() {
-	log := fs.log
-	maps.DeleteFunc(fs.md, func(k string, v *Metadata) bool {
-		_, err := fs.statDst(k)
-		if errors.Is(err, os.ErrNotExist) {
-			log.V(2).Info("removing not existing file from metadata", "file", k)
-			return true
-		}
-		if len(v.Chunks) == 0 {
-			log.V(2).Info("removing empty file", "file", k)
-			err := fs.RemoveDst(k)
 	if err != nil {
-				log.Error(err, "error removing empty file", "file", k)
+		return
-				return false
 	}
-			return true
+	fs.ph = NewPreloadHandler(ctx, fs, max, log.WithName("preload"))
-		}
+	fs.sc = NewStatCache(ctx)
-		v.fs = fs
+	return
-		v.name = k
-		return false
-	})
 }
 
 func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
@@ -193,9 +119,9 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
 		f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
 		return
 	}
-	md := fs.metadata(name, fi.Size())
+	md := fs.mh.Metadata(name, fi.Size())
 	f = &File{
-		log:     log,
+		log:     log.WithName("file"),
 		f:       file,
 		md:      md,
 		offline: true,
@@ -203,104 +129,20 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
 	return
 }
 
-func (fs *FS) CancelPreload(name string) {
+func (fs *FS) QuotaUsage() (cur, max int64) {
-	fs.mu.Lock()
+	return fs.q.Usage()
-	defer fs.mu.Unlock()
-	if p, ok := fs.pm[name]; ok && p.Running {
-		p.cancel()
-	}
 }
 
-func (fs *FS) Preloads() (ps Preloads) {
+func (fs *FS) CancelPreload(name string) {
-	fs.mu.RLock()
+	fs.ph.CancelPreload(name)
-	defer fs.mu.RUnlock()
-	for _, p := range fs.pm {
-		ps = append(ps, &Preload{
-			Name:    p.Name,
-			Running: p.Running,
-			Status:  fs.CacheStatus(p.Name),
-		})
 }
-	slices.SortFunc(ps, ps.Less)
+
-	return
+func (fs *FS) Preloads() Preloads {
+	return fs.ph.Preloads()
 }
 
 func (fs *FS) Preload(name string) {
-	fs.mu.Lock()
+	fs.ph.Preload(name)
-	defer fs.mu.Unlock()
-
-	p, ok := fs.pm[name]
-	if !ok {
-		p = &Preload{Name: name}
-		fs.pm[name] = p
-		if fs.maxq == -1 || fs.preq < fs.maxq {
-			fs.prec <- p
-		}
-		return
-	}
-
-	if p.Running {
-		fs.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
-	}
-	return
-}
-
-func (fs *FS) nextPreload() {
-	if fs.preq == fs.maxq {
-		return
-	}
-	for _, p := range fs.pm {
-		if p.Running {
-			continue
-		}
-		fs.prec <- p
-		return
-	}
-}
-
-func (fs *FS) preload(ctx context.Context) {
-	var p *Preload
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case p = <-fs.prec:
-		}
-		name := p.Name
-		file, err := fs.Open(name)
-		if err != nil {
-			fs.log.Error(err, "error staring next preload", "file", name)
-			return
-		}
-		f, ok := file.(*File)
-		if !ok {
-			continue
-		}
-		ctx, cancel := context.WithCancel(context.Background())
-		fs.mu.Lock()
-		fs.preq++
-		p.Running = true
-		p.cancel = cancel
-		fs.mu.Unlock()
-		go f.Preload(ctx, func() {
-			fs.pref <- name
-		})
-	}
-}
-
-func (fs *FS) preloadFinish(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case name := <-fs.pref:
-			fs.mu.Lock()
-			delete(fs.pm, name)
-			fs.preq--
-			fs.nextPreload()
-			fs.mu.Unlock()
-		}
-	}
 }
 
 func skipLog(name string) bool {
@@ -355,9 +197,9 @@ func (fs *FS) Open(name string) (f http.File, err error) {
 	if offline {
 		log.V(2).Info("file offline mode", "path", dp)
 	}
-	md := fs.metadata(name, fi.Size())
+	md := fs.mh.Metadata(name, fi.Size())
 	f = &File{
-		log:     log,
+		log:     log.WithName("file"),
 		f:       file,
 		md:      md,
 		offline: offline,
@@ -401,124 +243,12 @@ func (fs *FS) openCacheFile(name string, size int64) (df *os.File, err error) {
 	return
 }
 
-func (fs *FS) metadata(name string, size int64) (md *Metadata) {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if md, ok := fs.md[name]; ok {
-		if size > 0 && md.Size != size {
-			log := fs.log.WithValues("file", name)
-			log.V(2).Info("source file changed: deleting cache file", "src", size, "dst", md.Size)
-			err := md.Delete()
-			if err != nil {
-				log.Error(err, "error deleting cache file")
-			}
-		}
-		return md
-	}
-	md = &Metadata{Size: size, fs: fs, name: name}
-	fs.md[name] = md
-	return
-}
-
 func (fs *FS) CacheStatus(name string) int {
-	fs.mu.RLock()
+	return fs.mh.CacheStatus(name)
-	defer fs.mu.RUnlock()
-	if md, ok := fs.md[name]; ok {
-		size := md.Chunks.Size()
-		if size == 0 {
-			return -1
-		}
-		return int(math.Round(float64(size) / float64(md.Size) * 100))
-	}
-	return -1
-}
-
-func (fs *FS) flusher(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(time.Minute * 5):
-		}
-		fs.cleanupEmptyDirs()
-		fs.flushMetadata()
-	}
-}
-
-func (fs *FS) cleanupEmptyDirs() {
-	filepath.WalkDir(fs.dst, func(path string, d stdfs.DirEntry, err error) error {
-		if err == nil && d.IsDir() {
-			if path == fs.dst {
-				return nil
-			}
-			empty, err := dirEmpty(path)
-			if err != nil || !empty {
-				return nil
-			}
-			log := fs.log.WithValues("dir", path)
-			log.V(2).Info("removing empty dir")
-			err = os.Remove(path)
-			if err != nil {
-				log.Error(err, "failed to remove empty dir")
-			}
-		}
-		return nil
-	})
-}
-
-func (fs *FS) flushMetadata() {
-	log := fs.log
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	log.V(2).Info("flushing metadata to disk")
-	md := maps.Clone(fs.md)
-	maps.DeleteFunc(md, func(_ string, v *Metadata) bool {
-		return len(v.Chunks) == 0
-	})
-	_, err := fs.mdf.Seek(0, io.SeekStart)
-	if err != nil {
-		log.Error(err, "failure seeking metadata file")
-		return
-	}
-	err = fs.mdf.Truncate(0)
-	if err != nil {
-		log.Error(err, "failure truncating metadata file")
-		return
-	}
-	err = fs.jenc.Encode(md)
-	if err != nil {
-		log.Error(err, "failure flushing metadata file")
-	}
 }
 
 func (fs *FS) Close() {
-	fs.mu.Lock()
+	fs.cancel()
-	fs.preloadCancel()
-	fs.flushCancel()
-	fs.statCancel()
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-fs.prec:
-			case <-fs.pref:
-			}
-		}
-	}()
-	for _, p := range fs.pm {
-		if p.Running {
-			p.cancel()
-		}
-	}
-	for _, md := range fs.md {
-		md.f.Sync()
-		md.f.Close()
-	}
-	fs.mu.Unlock()
-	fs.flushMetadata()
 }
 
 func dirEmpty(name string) (bool, error) {

pkg/fs/metadata.go

@@ -3,17 +3,220 @@
 package fs
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	stdfs "io/fs"
+	"math"
 	"os"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/go-logr/logr"
+	"golang.org/x/exp/maps"
 )
 
 func now() int64 {
 	return time.Now().UTC().Unix()
 }
 
-type MetadataMap map[string]*Metadata
+type MetadataHandler struct {
+	mu  sync.RWMutex
+	log logr.Logger
+	md  map[string]*Metadata
+	fs  *FS
+	dst string
+	f   *os.File
+	enc *json.Encoder
+}
+
+func NewMetadataHandler(ctx context.Context, fs *FS, file, dst string, log logr.Logger) (mh *MetadataHandler, err error) {
+	if !filepath.IsAbs(file) {
+		return nil, errors.New("metadata path is not absolute")
+	}
+	mh = &MetadataHandler{
+		log: log,
+		md:  make(map[string]*Metadata),
+		fs:  fs,
+		dst: dst,
+	}
+	mh.f, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640)
+	if err != nil {
+		return
+	}
+	err = json.NewDecoder(mh.f).Decode(&mh.md)
+	if err != nil && err == io.EOF {
+		return
+	}
+	mh.enc = json.NewEncoder(mh.f)
+	err = nil
+	mh.init()
+	go mh.flusher(ctx)
+	return
+}
+
+func (mh *MetadataHandler) DeleteOldest() (s int64) {
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	atime := int64(math.MaxInt64)
+	var m *Metadata
+	for _, v := range mh.md {
+		if v.Atime != 0 && v.f != nil && v.Atime < atime {
+			atime = v.Atime
+			m = v
+		}
+	}
+	if m != nil {
+		log := mh.log.WithValues("file", m.name)
+		log.Info("deleting oldest file")
+		err := m.Delete()
+		if err != nil {
+			log.Error(err, "error deleting oldest file")
+			return
+		}
+		s = m.Size
+	}
+	return
+}
+
+func (mh *MetadataHandler) Size() (s int64) {
+	mh.mu.RLock()
+	defer mh.mu.RUnlock()
+	for _, md := range mh.md {
+		s += md.Chunks.Size()
+	}
+	return
+}
+
+func (mh *MetadataHandler) CacheStatus(name string) int {
+	mh.mu.RLock()
+	md, ok := mh.md[name]
+	mh.mu.RUnlock()
+	if !ok {
+		return -1
+	}
+	size := md.Chunks.Size()
+	if size == 0 {
+		return -1
+	}
+	return int(math.Round(float64(size) / float64(md.Size) * 100))
+}
+
+func (mh *MetadataHandler) Metadata(name string, size int64) (md *Metadata) {
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	if md, ok := mh.md[name]; ok {
+		if size > 0 && md.Size != size {
+			log := mh.log.WithValues("file", name)
+			log.V(2).Info("source file changed: deleting cache file", "src", size, "dst", md.Size)
+			err := md.Delete()
+			if err != nil {
+				log.Error(err, "error deleting cache file")
+			}
+		}
+		return md
+	}
+	md = &Metadata{Size: size, fs: mh.fs, name: name}
+	mh.md[name] = md
+	return
+}
+
+func (mh *MetadataHandler) init() {
+	log := mh.log
+	maps.DeleteFunc(mh.md, func(k string, v *Metadata) bool {
+		_, err := mh.fs.statDst(k)
+		if errors.Is(err, os.ErrNotExist) {
+			log.V(2).Info("removing not existing file from metadata", "file", k)
+			return true
+		}
+		if len(v.Chunks) == 0 {
+			log.V(2).Info("removing empty file", "file", k)
+			err := mh.fs.RemoveDst(k)
+			if err != nil {
+				log.Error(err, "error removing empty file", "file", k)
+				return false
+			}
+			return true
+		}
+		v.fs = mh.fs
+		v.name = k
+		return false
+	})
+}
+
+func (mh *MetadataHandler) flusher(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			mh.close()
+			return
+		case <-time.After(time.Minute * 5):
+		}
+		mh.cleanupEmptyDirs()
+		mh.flush()
+	}
+}
+
+func (mh *MetadataHandler) cleanupEmptyDirs() {
+	filepath.WalkDir(mh.dst, func(path string, d stdfs.DirEntry, err error) error {
+		if err == nil && d.IsDir() {
+			if path == mh.dst {
+				return nil
+			}
+			empty, err := dirEmpty(path)
+			if err != nil || !empty {
+				return nil
+			}
+			log := mh.log.WithValues("dir", path)
+			log.V(2).Info("removing empty dir")
+			err = os.Remove(path)
+			if err != nil {
+				log.Error(err, "failed to remove empty dir")
+			}
+		}
+		return nil
+	})
+}
+
+func (mh *MetadataHandler) flush() {
+	log := mh.log
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	log.V(2).Info("flushing metadata to disk")
+	md := maps.Clone(mh.md)
+	maps.DeleteFunc(mh.md, func(_ string, v *Metadata) bool {
+		return len(v.Chunks) == 0
+	})
+	_, err := mh.f.Seek(0, io.SeekStart)
+	if err != nil {
+		log.Error(err, "failure seeking metadata file")
+		return
+	}
+	err = mh.f.Truncate(0)
+	if err != nil {
+		log.Error(err, "failure truncating metadata file")
+		return
+	}
+	err = mh.enc.Encode(md)
+	if err != nil {
+		log.Error(err, "failure flushing metadata file")
+	}
+	return
+}
+
+func (mh *MetadataHandler) close() {
+	mh.mu.Lock()
+	for _, md := range mh.md {
+		md.f.Sync()
+		md.f.Close()
+	}
+	mh.mu.Unlock()
+	mh.flush()
+	mh.f.Close()
+}
 
 type Metadata struct {
 	mu     sync.RWMutex `json:"-"`

pkg/fs/preload.go

@@ -0,0 +1,168 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package fs
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"github.com/go-logr/logr"
+	"golang.org/x/exp/slices"
+)
+
+type Preload struct {
+	Name    string
+	cancel  func()
+	Status  int
+	Running bool
+}
+
+type Preloads []*Preload
+
+func (Preloads) Less(i, j *Preload) bool { return i.Name < j.Name }
+
+type PreloadHandler struct {
+	mu    sync.RWMutex
+	log   logr.Logger
+	fs    *FS
+	pm    map[string]*Preload
+	sched chan *Preload
+	fin   chan string
+	pre   int
+	max   int
+}
+
+func NewPreloadHandler(ctx context.Context, fs *FS, max int, log logr.Logger) (ph *PreloadHandler) {
+	if max < -1 {
+		max = -1
+	}
+	ph = &PreloadHandler{
+		log:   log,
+		fs:    fs,
+		pm:    make(map[string]*Preload),
+		sched: make(chan *Preload, 1),
+		fin:   make(chan string, 1),
+		max:   max,
+	}
+	go ph.preload(ctx)
+	go ph.preloadFinish(ctx)
+	return
+}
+
+func (ph *PreloadHandler) CancelPreload(name string) {
+	ph.mu.Lock()
+	defer ph.mu.Unlock()
+	if p, ok := ph.pm[name]; ok && p.Running {
+		p.cancel()
+	}
+}
+
+func (ph *PreloadHandler) Preloads() (ps Preloads) {
+	ph.mu.RLock()
+	defer ph.mu.RUnlock()
+	for _, p := range ph.pm {
+		ps = append(ps, &Preload{
+			Name:    p.Name,
+			Running: p.Running,
+			Status:  ph.fs.CacheStatus(p.Name),
+		})
+	}
+	slices.SortFunc(ps, ps.Less)
+	return
+}
+
+func (ph *PreloadHandler) Preload(name string) {
+	ph.mu.Lock()
+	defer ph.mu.Unlock()
+
+	p, ok := ph.pm[name]
+	if !ok {
+		p = &Preload{Name: name}
+		ph.pm[name] = p
+		if ph.max == -1 || ph.pre < ph.max {
+			ph.sched <- p
+		}
+		return
+	}
+
+	if p.Running {
+		ph.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
+	}
+	return
+}
+
+func (ph *PreloadHandler) nextPreload() {
+	if ph.pre == ph.max {
+		return
+	}
+	for _, p := range ph.pm {
+		if p.Running {
+			continue
+		}
+		ph.sched <- p
+		return
+	}
+}
+
+func (ph *PreloadHandler) preload(ctx context.Context) {
+	var p *Preload
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case p = <-ph.sched:
+		}
+		name := p.Name
+		file, err := ph.fs.Open(name)
+		if err != nil {
+			ph.log.Error(err, "error staring next preload", "file", name)
+			return
+		}
+		f, ok := file.(*File)
+		if !ok {
+			continue
+		}
+		ctx, cancel := context.WithCancel(context.Background())
+		ph.mu.Lock()
+		ph.pre++
+		p.Running = true
+		p.cancel = cancel
+		ph.mu.Unlock()
+		go f.Preload(ctx, func() {
+			ph.fin <- name
+		})
+	}
+}
+
+func (ph *PreloadHandler) preloadFinish(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			ph.close()
+			return
+		case name := <-ph.fin:
+			ph.mu.Lock()
+			delete(ph.pm, name)
+			ph.pre--
+			ph.nextPreload()
+			ph.mu.Unlock()
+		}
+	}
+}
+
+func (ph *PreloadHandler) close() {
+	for _, p := range ph.pm {
+		if p.Running {
+			p.cancel()
+		}
+	}
+	for {
+		select {
+		case <-ph.sched:
+		case <-ph.fin:
+		default:
+			return
+		}
+	}
+}

pkg/fs/quota.go

@@ -4,7 +4,7 @@ package fs
 
 import (
 	"errors"
-	"math"
+	"sync"
 	"sync/atomic"
 
 	"github.com/go-logr/logr"
@@ -13,52 +13,37 @@ import (
 var InvalidMaxQuota = errors.New("invalid max quota")
 
 type Quota struct {
+	mu  sync.Mutex
 	log logr.Logger
 	fs  *FS
-	max int64
 	cur int64
+	max int64
 }
 
-func NewQuota(max int64, fs *FS, log logr.Logger) (q *Quota, err error) {
+func NewQuota(cur, max int64, fs *FS, log logr.Logger) (q *Quota, err error) {
 	if max <= 0 {
 		err = InvalidMaxQuota
 		return
 	}
-	q = &Quota{log: log, fs: fs, max: max}
+	q = &Quota{log: log, fs: fs, cur: cur, max: max}
+	q.log.Info("quota usage", "current", q.cur, "max", q.max)
 	return
 }
 
-func (q *Quota) Init() {
+func (q *Quota) Usage() (cur, max int64) {
-	for _, v := range q.fs.md {
+	cur = atomic.LoadInt64(&q.cur)
-		q.cur += v.Chunks.Size()
+	max = q.max
-	}
+	return
-	q.log.Info("quota usage", "current", q.cur, "max", q.max)
 }
 
 func (q *Quota) cleanup() {
-	if !q.fs.mu.TryLock() {
+	if !q.mu.TryLock() {
 		return
 	}
-	q.log.Info("quota usage", "current", q.cur, "max", q.max)
+	q.log.Info("quota usage", "current", atomic.LoadInt64(&q.cur), "max", q.max)
-	defer q.fs.mu.Unlock()
+	defer q.mu.Unlock()
-	atime := int64(math.MaxInt64)
+	size := q.fs.mh.DeleteOldest()
-	var m *Metadata
+	atomic.AddInt64(&q.cur, -size)
-	for _, v := range q.fs.md {
-		if v.Atime != 0 && v.f != nil && v.Atime < atime {
-			atime = v.Atime
-			m = v
-		}
-	}
-	if m != nil {
-		log := q.log.WithValues("file", m.name)
-		log.Info("deleting oldest file")
-		err := m.Delete()
-		if err != nil {
-			q.log.Error(err, "error deleting oldest file")
-			return
-		}
-		atomic.AddInt64(&q.cur, -m.Size)
-	}
 }
 
 func (q *Quota) Add(n int) {

pkg/srv/interceptor.go

@@ -139,7 +139,9 @@ func getPreloads(path string, fs *fs.FS) (dc dirContents) {
 			Running: p.Running,
 		})
 	}
-	slices.SortFunc(dc.Files, dc.Files.Less)
+	slices.SortFunc(dc.Files, func(i, _ file) bool {
+		return i.Running
+	})
 	return
 }
 

pkg/srv/srv.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	stdfs "io/fs"
+	"math"
 	"net/http"
 	"path"
 	"strings"
@@ -19,8 +20,16 @@ const (
 	csp      = "Content-Security-Policy"
 	indexCSP = "default-src 'none';style-src 'unsafe-inline';frame-ancestors 'none'"
 	videoCSP = indexCSP + ";script-src 'sha256-ZLSc/s5/US9uVMMZOJ/yWiS1tj9nEoi+5Qoohy3QetU=';media-src 'self'"
+
+	gib = 1024 * 1024 * 1024
 )
 
+type data struct {
+	QuotaCur float64
+	QuotaMax float64
+	Paths    dirContents
+}
+
 type FileServer struct {
 	log logr.Logger
 	fs  *fs.FS
@@ -32,7 +41,8 @@ func NewFileServer(fs *fs.FS, log logr.Logger) http.Handler {
 }
 
 func skipLog(path string) bool {
-	return strings.HasSuffix(path, "favicon.ico")
+	return path == "/?o=preloads" ||
+		strings.HasSuffix(path, "favicon.ico")
 }
 
 func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -67,7 +77,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h := w.Header()
 	if option == "v" {
 		h.Set(csp, videoCSP)
-		err = video.Execute(w, r.URL.Path)
+		err = video.Execute(w, data{Paths: dirContents{Base: r.URL.Path}})
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
@@ -82,6 +92,10 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				fs.fs.CancelPreload(p)
 			}
 			rdir := "/"
+			if r.FormValue("r") == "preloads" {
+				http.Redirect(w, r, rdir+"?o=preloads", http.StatusFound)
+				return
+			}
 			if i := strings.LastIndex(p, "/"); i > 0 {
 				rdir = p[:i]
 			}
@@ -97,7 +111,12 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	if option == "preloads" {
 		h.Set(csp, indexCSP)
-		err = preloads.Execute(w, getPreloads(p, fs.fs))
+		cur, max := fs.fs.QuotaUsage()
+		err = preloads.Execute(w, data{
+			QuotaCur: math.Round(float64(cur)/gib*100) / 100,
+			QuotaMax: float64(max) / gib,
+			Paths:    getPreloads(p, fs.fs),
+		})
 		if err != nil {
 			fs.log.Error(err, "error rendering preloads")
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -124,7 +143,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.Set(csp, indexCSP)
 
 	w.WriteHeader(i.Status())
-	err = index.Execute(w, paths)
+	err = index.Execute(w, data{Paths: paths})
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return

pkg/srv/templates.go

@@ -2,231 +2,39 @@
 
 package srv
 
-import "html/template"
+import (
+	"embed"
+	"html/template"
+	"io/fs"
+)
+
+//go:embed templates/*
+var templates embed.FS
+
+func subMust(fsys fs.FS, dir string) fs.FS {
+	fsys, err := fs.Sub(fsys, dir)
+	if err != nil {
+		panic("sub: " + err.Error())
+	}
+	return fsys
+}
+
+var tsub = subMust(templates, "templates")
 
 var (
-	indexHead = `<!doctype html>
+	index = template.Must(template.ParseFS(tsub,
-<html lang="en">
+		"head.html",
-<head>
+		"index.html",
-  <meta name="viewport" content="width=device-width, initial-scale=1">
+	))
-  <meta name="theme-color" content="#222222">
+	preloads = template.Must(template.ParseFS(tsub,
-  <meta name="description" content="cachefs">
+		"head.html",
-  <style>
+		"preloads.html",
-body {
+	))
-  font: 20px Helvetica, sans-serif;
+	cache = template.Must(template.ParseFS(tsub,
-  background-color: #111;
+		"head.html",
-  color: #e6e6e6;
+		"cache.html",
-  margin: 0px;
+	))
-  padding: 0 150px 0 150px;
+	video = template.Must(template.ParseFS(tsub,
-}
+		"video.html",
-@media only screen and (max-width: 800px) {
+	))
-  body {
-    padding: 0;
-  }
-}
-article {
-  background-color: #222;
-}
-pre {
-  margin: 0;
-  padding: 10px 0 14px 0;
-}
-a:link {
-  color: #e6e6e6;
-  text-decoration: none;
-}
-a:visited {
-  color: #e6e6e6;
-}
-.listpath a:visited {
-  color: #b2b2b2;
-}
-.up a:visited {
-  color: #e6e6e6;
-}
-a:hover, .listpath a:hover {
-  color: #fff;
-  text-decoration: underline;
-}
-a:active {
-  color: #a6a6a6;
-}
-table {
-  border-collapse: collapse;
-  margin-bottom: 50px;
-}
-td {
-  padding: 0;
-}
-.listitem {
-  background-color: #303030;
-}
-.listitem:hover {
-  background-color: #404040;
-}
-.listitem:active {
-  background-color: #363636;
-}
-.listpath {
-  white-space: normal;
-  overflow: hidden;
-  width: 100%;
-}
-.listpath a {
-  display: block;
-  min-height: 45px;
-  padding: 0.55em 0 0 5px;
-  overflow: hidden;
-  width: 100%;
-  box-sizing: border-box;
-  word-break: break-all;
-}
-.listoptions {
-  padding-right: 5px;
-  white-space: nowrap;
-  text-align: right;
-}
-.listoptions span, .listoptions a {
-  margin-left: 10px;
-}
-.spacer {
-  height: 3px;
-}
-.red {
-  background-color: #e74c3c;
-}
-.yellow {
-  background-color: #f39c12;
-}
-.green {
-  background-color: #4caf50;
-}
-.status-yellow {
-  color: #f39c12;
-}
-.status-green {
-  color: #4caf50;
-}
-span {
-  color: #b2b2b2;
-  font-size: 15px;
-}
-  </style>
-  <title>cachefs | {{.Base}}</title>
-</head>`
-
-	index = template.Must(template.New("index").Parse(indexHead + `<body>
-  <article>
-  <pre>[v]: show video
-[n]: skip file caching
-[p]: preload file
-<a href="/?o=preloads">[Preloads]</a></pre>
-  <table>
-    <tr class="listitem">
-      <td class="listpath up"><a href="../">../</a></td>
-      <td class="listoptions">[dir]</td>
-    </tr>
-    <tr class="spacer"><td colspan="2"></td></tr>
-{{range $s := .Dirs -}}
-    <tr class="listitem">
-      <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
-      <td class="listoptions">[dir]</td>
-    </tr>
-    <tr class="spacer"><td colspan="2"></td></tr>
-{{end -}}
-{{range $s := .Files -}}
-    <tr class="listitem">
-      <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
-      <td class="listoptions">
-        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a>
-      </td>
-    </tr>
-    <tr class="spacer"><td colspan="2">
-{{if ge $s.Status 0}}
-      <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
-{{end -}}
-    </td></tr>
-{{end -}}
-  </table>
-  </article>
-</body>
-</html>`))
-	preloads = template.Must(template.New("preloads").Parse(indexHead + `<body>
-  <article>
-  <pre>[v]: show video
-[s]: stop preloading</pre>
-  <table>
-    <tr class="listitem">
-      <td class="listpath up"><a href="/">/</a></td>
-      <td class="listoptions">[dir]</td>
-    </tr>
-{{range $s := .Files -}}
-    <tr class="listitem">
-      <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
-      <td class="listoptions">
-        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span class="status-{{if $s.Running}}green">[running{{else}}yellow">[queued{{end}}]</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s">[s]</a>
-      </td>
-    </tr>
-    <tr class="spacer"><td colspan="2">
-{{if ge $s.Status 0}}
-      <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
-{{end -}}
-    </td></tr>
-{{end -}}
-  </table>
-  </article>
-</body>
-</html>`))
-	cache = template.Must(template.New("cache").Parse(indexHead + `<body>
-  <article>
-  <pre>[v]: show video</pre>
-  <table>
-    <tr class="listitem">
-      <td class="listpath up"><a href="../">../</a></td>
-      <td class="listoptions">[dir]</td>
-    </tr>
-    <tr class="spacer"><td colspan="2"></td></tr>
-{{range $s := .Dirs -}}
-    <tr class="listitem">
-      <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
-      <td class="listoptions">[dir]</td>
-    </tr>
-    <tr class="spacer"><td colspan="2"></td></tr>
-{{end -}}
-{{range $s := .Files -}}
-    <tr class="listitem">
-      <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
-      <td class="listoptions">
-        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a>
-      </td>
-    </tr>
-    <tr class="spacer"><td colspan="2">
-{{if ge $s.Status 0}}
-      <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
-{{end -}}
-    </td></tr>
-{{end -}}
-  </table>
-  </article>
-</body>
-</html>`))
-	video = template.Must(template.New("video").Parse(`<!doctype html>
-<html lang="en">
-<head>
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <meta name="theme-color" content="#111111">
-  <meta name="description" content="cachefs">
-  <title>cachefs | {{.}}</title>
-  <style>
-body {
-  background-color: #111;
-  margin: 0px;
-}
-  </style>
-</head>
-<body>
-  <video id="video" style="width: 100%; height: 100%;" src="{{.}}" controls=""></video>
-  <script>document.getElementById("video").volume=0.5;</script>
-</body>
-</html>`))
 )

pkg/srv/templates/cache.html

@@ -0,0 +1,30 @@
+{{define "body" -}}
+<pre>[v]: show video</pre>
+<table>
+  <tr class="listitem">
+    <td class="listpath up"><a href="../">../</a></td>
+    <td class="listoptions">[dir]</td>
+  </tr>
+  <tr class="spacer"><td colspan="2"></td></tr>
+{{range $s := .Dirs -}}
+  <tr class="listitem">
+    <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listoptions">[dir]</td>
+  </tr>
+  <tr class="spacer"><td colspan="2"></td></tr>
+{{end -}}
+{{range $s := .Files -}}
+  <tr class="listitem">
+    <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listoptions">
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a>
+    </td>
+  </tr>
+  <tr class="spacer"><td colspan="2">
+{{if ge $s.Status 0 -}}
+    <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
+{{end -}}
+  </td></tr>
+{{end -}}
+</table>
+{{end -}}

pkg/srv/templates/head.html

@@ -0,0 +1,115 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="theme-color" content="#222222">
+  <meta name="description" content="cachefs">
+  <style>
+body {
+  font: 20px Helvetica, sans-serif;
+  background-color: #111;
+  color: #e6e6e6;
+  margin: 0px;
+  padding: 0 150px 0 150px;
+}
+@media only screen and (max-width: 800px) {
+  body {
+    padding: 0;
+  }
+}
+article {
+  background-color: #222;
+}
+pre {
+  margin: 0;
+  padding: 10px 0 14px 0;
+}
+a:link {
+  color: #e6e6e6;
+  text-decoration: none;
+}
+a:visited {
+  color: #e6e6e6;
+}
+.listpath a:visited {
+  color: #b2b2b2;
+}
+.up a:visited {
+  color: #e6e6e6;
+}
+a:hover, .listpath a:hover {
+  color: #fff;
+  text-decoration: underline;
+}
+a:active {
+  color: #a6a6a6;
+}
+table {
+  border-collapse: collapse;
+  margin-bottom: 50px;
+}
+td {
+  padding: 0;
+}
+.listitem {
+  background-color: #303030;
+}
+.listitem:hover {
+  background-color: #404040;
+}
+.listitem:active {
+  background-color: #363636;
+}
+.listpath {
+  white-space: normal;
+  overflow: hidden;
+  width: 100%;
+}
+.listpath a {
+  display: block;
+  min-height: 45px;
+  padding: 0.55em 0 0 5px;
+  overflow: hidden;
+  width: 100%;
+  box-sizing: border-box;
+  word-break: break-all;
+}
+.listoptions {
+  padding-right: 5px;
+  white-space: nowrap;
+  text-align: right;
+}
+.listoptions span, .listoptions a {
+  margin-left: 10px;
+}
+.spacer {
+  height: 3px;
+}
+.red {
+  background-color: #e74c3c;
+}
+.yellow {
+  background-color: #f39c12;
+}
+.green {
+  background-color: #4caf50;
+}
+.status-yellow {
+  color: #f39c12;
+}
+.status-green {
+  color: #4caf50;
+}
+span {
+  color: #b2b2b2;
+  font-size: 15px;
+}
+  </style>
+  <title>cachefs | {{.Paths.Base}}</title>
+</head>
+<body>
+  <article>
+{{template "body" .}}
+  </article>
+</body>
+</html>

pkg/srv/templates/index.html

@@ -0,0 +1,33 @@
+{{define "body" -}}
+<pre>[v]: show video
+[n]: skip file caching
+[p]: preload file
+<a href="/?o=preloads">[Preloads]</a></pre>
+<table>
+  <tr class="listitem">
+    <td class="listpath up"><a href="../">../</a></td>
+    <td class="listoptions">[dir]</td>
+  </tr>
+  <tr class="spacer"><td colspan="2"></td></tr>
+{{range $s := .Paths.Dirs -}}
+  <tr class="listitem">
+    <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listoptions">[dir]</td>
+  </tr>
+  <tr class="spacer"><td colspan="2"></td></tr>
+{{end -}}
+{{range $s := .Paths.Files -}}
+  <tr class="listitem">
+    <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listoptions">
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a>
+    </td>
+  </tr>
+  <tr class="spacer"><td colspan="2">
+{{if ge $s.Status 0 -}}
+    <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
+{{end -}}
+  </td></tr>
+{{end -}}
+</table>
+{{end -}}

pkg/srv/templates/preloads.html

@@ -0,0 +1,25 @@
+{{define "body" -}}
+<pre>[v]: show video
+[s]: stop preloading
+Quota: {{.QuotaCur}} / {{.QuotaMax}} GiB
+</pre>
+<table>
+  <tr class="listitem">
+    <td class="listpath up"><a href="/">/</a></td>
+    <td class="listoptions">[dir]</td>
+  </tr>
+{{range $s := .Paths.Files -}}
+  <tr class="listitem">
+    <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listoptions">
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span class="status-{{if $s.Running}}green">[running{{else}}yellow">[queued{{end}}]</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s&r=preloads">[s]</a>
+    </td>
+  </tr>
+  <tr class="spacer"><td colspan="2">
+{{if ge $s.Status 0 -}}
+    <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
+{{end -}}
+  </td></tr>
+{{end -}}
+</table>
+{{end -}}

pkg/srv/templates/video.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <meta name="theme-color" content="#111111">
+  <meta name="description" content="cachefs">
+  <title>cachefs | {{.Paths.Base}}</title>
+  <style>
+body {
+  background-color: #111;
+  margin: 0px;
+}
+  </style>
+</head>
+<body>
+  <video id="video" style="width: 100%; height: 100%;" src="{{.Paths.Base}}" controls=""></video>
+  <script>document.getElementById("video").volume=0.5;</script>
+</body>
+</html>