added preloads page

cmd/cachefs/main.go

@@ -29,6 +29,7 @@ var (
 	listenWebdav string
 	listenCache  string
 	quota        int64
+	maxq         int
 
 	log logr.Logger
 )
@@ -43,13 +44,14 @@ func main() {
 	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
 	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
 	flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
+	flag.IntVar(&maxq, "maxq", -1, "max parallel preloads")
 	flag.Int64Var(&quota, "quota", 1, "max disk usage quota in GiB")
 	flag.Parse()
 
 	log = klogr.New().WithName("main")
 	log.Info("starting cachefs", "version", version)
 
-	filesystem, err := fs.NewFS(quota*gib, src, dst, metadata, klogr.New().WithName("fs"))
+	filesystem, err := fs.NewFS(quota*gib, maxq, src, dst, metadata, klogr.New().WithName("fs"))
 	if err != nil {
 		klog.Fatalf("init failed: %s", err)
 	}

pkg/fs/fs.go

@@ -19,26 +19,44 @@ import (
 
 	"github.com/go-logr/logr"
 	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 type FS struct {
-	mu          sync.RWMutex
+	mu            sync.RWMutex
-	log         logr.Logger
+	log           logr.Logger
-	NoCache     http.Handler
+	NoCache       http.Handler
-	src         string
+	src           string
-	dst         string
+	dst           string
-	statCancel  func()
+	statCancel    func()
-	flushCancel func()
+	flushCancel   func()
-	mdf         *os.File
+	preloadCancel func()
-	jenc        *json.Encoder
+	mdf           *os.File
-	dc          *DirCache
+	jenc          *json.Encoder
-	sc          *StatCache
+	dc            *DirCache
-	q           *Quota
+	sc            *StatCache
-	md          MetadataMap
+	q             *Quota
-	pm          map[string]func()
+	md            MetadataMap
+
+	pm   map[string]*Preload
+	prec chan *Preload
+	pref chan string
+	preq int
+	maxq int
 }
 
-func NewFS(max int64, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
+type Preload struct {
+	Name    string
+	cancel  func()
+	Status  int
+	Running bool
+}
+
+type Preloads []*Preload
+
+func (Preloads) Less(i, j *Preload) bool { return i.Name < j.Name }
+
+func NewFS(max int64, maxq int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
 	if !filepath.IsAbs(src) {
 		return nil, errors.New("src path is not absolute")
 	}
@@ -57,19 +75,30 @@ func NewFS(max int64, src, dst, metadata string, log logr.Logger) (fs *FS, err e
 	}
 	statCtx, statCancel := context.WithCancel(context.Background())
 	flushCtx, flushCancel := context.WithCancel(context.Background())
+	preloadCtx, preloadCancel := context.WithCancel(context.Background())
+
+	if maxq < -1 {
+		maxq = -1
+	}
+
 	fs = &FS{
-		log:         log,
+		log:           log,
-		NoCache:     http.FileServer(http.Dir(src)),
+		NoCache:       http.FileServer(http.Dir(src)),
-		src:         src,
+		src:           src,
-		dst:         dst,
+		dst:           dst,
-		mdf:         mdf,
+		mdf:           mdf,
-		statCancel:  statCancel,
+		statCancel:    statCancel,
-		flushCancel: flushCancel,
+		flushCancel:   flushCancel,
-		dc:          NewDirCache(),
+		preloadCancel: preloadCancel,
-		sc:          NewStatCache(statCtx),
+		dc:            NewDirCache(),
-		md:          make(MetadataMap),
+		sc:            NewStatCache(statCtx),
-		pm:          make(map[string]func()),
+		md:            make(MetadataMap),
-		jenc:        json.NewEncoder(mdf),
+		jenc:          json.NewEncoder(mdf),
+
+		pm:   make(map[string]*Preload),
+		prec: make(chan *Preload, 1),
+		pref: make(chan string, 1),
+		maxq: maxq,
 	}
 	fs.q, err = NewQuota(max, fs, fs.log.WithName("quota"))
 	if err != nil {
@@ -82,6 +111,8 @@ func NewFS(max int64, src, dst, metadata string, log logr.Logger) (fs *FS, err e
 	fs.initMetadata()
 	fs.q.Init()
 	go fs.flusher(flushCtx)
+	go fs.preload(preloadCtx)
+	go fs.preloadFinish(preloadCtx)
 	return fs, err
 }
 
@@ -173,32 +204,103 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
 }
 
 func (fs *FS) CancelPreload(name string) {
-	if cancel, ok := fs.pm[name]; ok {
+	fs.mu.Lock()
-		cancel()
+	defer fs.mu.Unlock()
+	if p, ok := fs.pm[name]; ok && p.Running {
+		p.cancel()
 	}
 }
 
-func (fs *FS) Preload(name string) {
+func (fs *FS) Preloads() (ps Preloads) {
-	file, err := fs.Open(name)
+	fs.mu.RLock()
-	if err != nil {
+	defer fs.mu.RUnlock()
-		return
+	for _, p := range fs.pm {
+		ps = append(ps, &Preload{
+			Name:    p.Name,
+			Running: p.Running,
+			Status:  fs.CacheStatus(p.Name),
+		})
 	}
-	f, ok := file.(*File)
+	slices.SortFunc(ps, ps.Less)
+	return
+}
+
+func (fs *FS) Preload(name string) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	p, ok := fs.pm[name]
 	if !ok {
+		p = &Preload{Name: name}
+		fs.pm[name] = p
+		if fs.maxq == -1 || fs.preq < fs.maxq {
+			fs.prec <- p
+		}
 		return
 	}
 
-	if f.md.Preload() {
+	if p.Running {
 		fs.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
+	}
+	return
+}
+
+func (fs *FS) nextPreload() {
+	if fs.preq == fs.maxq {
 		return
 	}
-	ctx, cancel := context.WithCancel(context.Background())
+	for _, p := range fs.pm {
-	fs.pm[name] = cancel
+		if p.Running {
-	unlock := func() {
+			continue
-		delete(fs.pm, name)
+		}
-		f.md.UnlockPreload()
+		fs.prec <- p
+		return
+	}
+}
+
+func (fs *FS) preload(ctx context.Context) {
+	var p *Preload
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case p = <-fs.prec:
+		}
+		name := p.Name
+		file, err := fs.Open(name)
+		if err != nil {
+			fs.log.Error(err, "error staring next preload", "file", name)
+			return
+		}
+		f, ok := file.(*File)
+		if !ok {
+			continue
+		}
+		ctx, cancel := context.WithCancel(context.Background())
+		fs.mu.Lock()
+		fs.preq++
+		p.Running = true
+		p.cancel = cancel
+		fs.mu.Unlock()
+		go f.Preload(ctx, func() {
+			fs.pref <- name
+		})
+	}
+}
+
+func (fs *FS) preloadFinish(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case name := <-fs.pref:
+			fs.mu.Lock()
+			delete(fs.pm, name)
+			fs.preq--
+			fs.nextPreload()
+			fs.mu.Unlock()
+		}
 	}
-	go f.Preload(ctx, unlock)
 }
 
 func skipLog(name string) bool {
@@ -390,15 +492,32 @@ func (fs *FS) flushMetadata() {
 }
 
 func (fs *FS) Close() {
+	fs.mu.Lock()
+	fs.preloadCancel()
 	fs.flushCancel()
 	fs.statCancel()
-	for _, cancel := range fs.pm {
+	ctx, cancel := context.WithCancel(context.Background())
-		cancel()
+	defer cancel()
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-fs.prec:
+			case <-fs.pref:
+			}
+		}
+	}()
+	for _, p := range fs.pm {
+		if p.Running {
+			p.cancel()
+		}
 	}
 	for _, md := range fs.md {
 		md.f.Sync()
 		md.f.Close()
 	}
+	fs.mu.Unlock()
 	fs.flushMetadata()
 }
 

pkg/fs/metadata.go

@@ -16,14 +16,13 @@ func now() int64 {
 type MetadataMap map[string]*Metadata
 
 type Metadata struct {
-	mu      sync.RWMutex `json:"-"`
+	mu     sync.RWMutex `json:"-"`
-	fs      *FS          `json:"-"`
+	fs     *FS          `json:"-"`
-	f       *os.File     `json:"-"`
+	f      *os.File     `json:"-"`
-	name    string       `json:"-"`
+	name   string       `json:"-"`
-	Size    int64        `json:"s"`
+	Size   int64        `json:"s"`
-	Atime   int64        `json:"a"`
+	Atime  int64        `json:"a"`
-	Chunks  Chunks       `json:"c"`
+	Chunks Chunks       `json:"c"`
-	preload bool         `json:"-"`
 }
 
 func (md *Metadata) Delete() error {
@@ -82,22 +81,6 @@ func (md *Metadata) openCacheFile() error {
 	return err
 }
 
-func (md *Metadata) Preload() bool {
-	md.mu.Lock()
-	defer md.mu.Unlock()
-	if md.preload {
-		return true
-	}
-	md.preload = true
-	return false
-}
-
-func (md *Metadata) UnlockPreload() {
-	md.mu.Lock()
-	defer md.mu.Unlock()
-	md.preload = false
-}
-
 func (md *Metadata) Stat() (os.FileInfo, error) {
 	if md.f == nil {
 		err := md.openCacheFile()

pkg/srv/interceptor.go

@@ -56,10 +56,11 @@ type dirs []dir
 func (dirs) Less(i, j dir) bool { return i.Name < j.Name }
 
 type file struct {
-	Name   template.HTML
+	Name    template.HTML
-	URI    template.HTML
+	URI     template.HTML
-	Anchor string
+	Anchor  string
-	Status int
+	Status  int
+	Running bool
 }
 
 type files []file
@@ -128,6 +129,20 @@ func (r *responseInterceptor) GetPaths(path string, fs *fs.FS, relative bool) (d
 	return
 }
 
+func getPreloads(path string, fs *fs.FS) (dc dirContents) {
+	dc.Base = path
+	for _, p := range fs.Preloads() {
+		dc.Files = append(dc.Files, file{
+			Name:    template.HTML(p.Name),
+			URI:     template.HTML(p.Name),
+			Status:  p.Status,
+			Running: p.Running,
+		})
+	}
+	slices.SortFunc(dc.Files, dc.Files.Less)
+	return
+}
+
 func anchor(s string) string {
 	if i := strings.LastIndex(s, "/"); i > 0 && i+1 < len(s) {
 		s = s[i+1:]

pkg/srv/srv.go

@@ -95,6 +95,15 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		fs.h.ServeHTTP(w, r)
 		return
 	}
+	if option == "preloads" {
+		h.Set(csp, indexCSP)
+		err = preloads.Execute(w, getPreloads(p, fs.fs))
+		if err != nil {
+			fs.log.Error(err, "error rendering preloads")
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+		return
+	}
 
 	i := &responseInterceptor{w: w}
 	r.Header.Del("If-Modified-Since")

pkg/srv/templates.go

@@ -11,7 +11,6 @@ var (
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <meta name="theme-color" content="#222222">
   <meta name="description" content="cachefs">
-  <title>cachefs | {{.Base}}</title>
   <style>
 body {
   font: 20px Helvetica, sans-serif;
@@ -102,11 +101,18 @@ td {
 .green {
   background-color: #4caf50;
 }
+.status-yellow {
+  color: #f39c12;
+}
+.status-green {
+  color: #4caf50;
+}
 span {
   color: #b2b2b2;
   font-size: 15px;
 }
   </style>
+  <title>cachefs | {{.Base}}</title>
 </head>`
 
 	index = template.Must(template.New("index").Parse(indexHead + `<body>
@@ -114,7 +120,7 @@ span {
   <pre>[v]: show video
 [n]: skip file caching
 [p]: preload file
-[s]: stop preloading</pre>
+<a href="/?o=preloads">[Preloads]</a></pre>
   <table>
     <tr class="listitem">
       <td class="listpath up"><a href="../">../</a></td>
@@ -132,7 +138,33 @@ span {
     <tr class="listitem">
       <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
       <td class="listoptions">
-        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a><a href="{{$s.URI}}?o=s">[s]</a>
+        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a>
+      </td>
+    </tr>
+    <tr class="spacer"><td colspan="2">
+{{if ge $s.Status 0}}
+      <div class="spacer {{if le $s.Status 35}}red{{else if le $s.Status 65}}yellow{{else}}green{{end}}" style="width: {{$s.Status}}%;"></div>
+{{end -}}
+    </td></tr>
+{{end -}}
+  </table>
+  </article>
+</body>
+</html>`))
+	preloads = template.Must(template.New("preloads").Parse(indexHead + `<body>
+  <article>
+  <pre>[v]: show video
+[s]: stop preloading</pre>
+  <table>
+    <tr class="listitem">
+      <td class="listpath up"><a href="/">/</a></td>
+      <td class="listoptions">[dir]</td>
+    </tr>
+{{range $s := .Files -}}
+    <tr class="listitem">
+      <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
+      <td class="listoptions">
+        {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span class="status-{{if $s.Running}}green">[running{{else}}yellow">[queued{{end}}]</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s">[s]</a>
       </td>
     </tr>
     <tr class="spacer"><td colspan="2">