fixed preload resumption

pkg/fs/file.go

@@ -53,7 +53,7 @@ func (f *File) Preload(ctx context.Context, fin func(), errf func(), stop func()
 	defer f.Close()
 	if f.offline {
 		log.V(2).Error(errors.New("no preload in offline mode"), "error preloading file")
-		fin()
+		errf()
 		return
 	}
 	if f.md.FullyCached() {

pkg/fs/preload.go

@@ -5,6 +5,7 @@ package fs
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/go-logr/logr"
 	"golang.org/x/exp/slices"
@@ -44,12 +45,23 @@ func (q *queue) Remove(name string) {
 	}
 }
 
+func (q *queue) GetPreload(name string) *Preload {
+	for _, p := range *q {
+		if p.Name == name {
+			return p
+		}
+	}
+	return nil
+}
+
 type Preload struct {
 	ph      *PreloadHandler
+	delay   time.Duration
 	Name    string
 	cancel  func()
 	Prio    int
 	Status  int
+	Errc    int
 	Running bool
 }
 
@@ -113,7 +125,7 @@ func (ph *PreloadHandler) Preload(name string, prio int) {
 func (ph *PreloadHandler) schedule() {
 	for i := 0; i < len(ph.q); i++ {
 		if i < ph.max {
-			ph.q[i].start()
+			go ph.q[i].start()
 		} else {
 			ph.q[i].stop(true)
 		}
@@ -124,10 +136,14 @@ func (p *Preload) start() {
 	if p.Running {
 		return
 	}
+	if p.delay != 0 {
+		time.Sleep(p.delay)
+	}
 	name := p.Name
 	file, err := p.ph.fs.Open(name)
 	if err != nil {
 		p.ph.log.Error(err, "error staring next preload", "file", name)
+		p.ph.err <- name
 		return
 	}
 	f, ok := file.(*File)
@@ -173,8 +189,21 @@ func (ph *PreloadHandler) preloadStatus(ctx context.Context) {
 			ph.mu.Unlock()
 		case name := <-ph.err:
 			ph.mu.Lock()
-			ph.q.Add(ph, name, -1)
+			p := ph.q.GetPreload(name)
-			ph.schedule()
+			if p != nil {
+				p.stop(false)
+				p.delay = time.Second * 2
+				if p.Errc >= 10 {
+					if p.Prio >= 0 {
+						p.Prio = 0
+					}
+					p.Prio -= 1
+					ph.schedule()
+				} else {
+					p.Errc += 1
+					go p.start()
+				}
+			}
 			ph.mu.Unlock()
 		}
 	}

pkg/srv/interceptor.go

@@ -61,6 +61,7 @@ type file struct {
 	Anchor  string
 	Status  int
 	Prio    int
+	Errc    int
 	Running bool
 }
 
@@ -138,6 +139,7 @@ func getPreloads(path string, fs *fs.FS) (dc dirContents) {
 			URI:     template.HTML(p.Name),
 			Status:  p.Status,
 			Prio:    p.Prio,
+			Errc:    p.Errc,
 			Running: p.Running,
 		})
 	}

pkg/srv/templates/head.html

@@ -94,6 +94,9 @@ td {
 .green {
   background-color: #4caf50;
 }
+.status-red {
+  color: #e74c3c;
+}
 .status-yellow {
   color: #f39c12;
 }

pkg/srv/templates/preloads.html

@@ -14,7 +14,17 @@ Quota: {{.QuotaCur}} / {{.QuotaMax}} GiB
   <tr class="listitem">
     <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
     <td class="listoptions">
-      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span class="status-{{if $s.Running}}green">[running{{else}}yellow">[queued{{end}}]</span><span>{{$s.Prio}}</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s&r=preloads">[s]</a><a href="{{$s.Name}}?o=i&r=preloads">[+]</a><a href="{{$s.Name}}?o=d&r=preloads">[-]</a>
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}
+{{if $s.Running -}}
+      <span class="status-green">[running]</span>
+{{else -}}
+{{if ge $s.Errc 10 -}}
+      <span class="status-red">[error]</span>
+{{else -}}
+      <span class="status-yellow">[queued]</span>
+{{end -}}
+{{end -}}
+      <span>{{$s.Prio}}</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s&r=preloads">[s]</a><a href="{{$s.Name}}?o=i&r=preloads">[+]</a><a href="{{$s.Name}}?o=d&r=preloads">[-]</a>
     </td>
   </tr>
   <tr class="spacer"><td colspan="2">