fix metadata generator

cmd/cachectl/main.go

@@ -3,15 +3,19 @@
 package main
 
 import (
+	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/parse"
 	"encoding/hex"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
+	"time"
 )
 
 var (
@@ -176,7 +180,52 @@ func cp(args []string) (err error) {
 		return
 	}
 	defer df.Close()
-	_, err = io.Copy(df, sf)
+	fi, err := sf.Stat()
+	if err != nil {
+		return
+	}
+	r := &progressReader{
+		f:    sf,
+		done: make(chan struct{}),
+		size: fi.Size(),
+	}
+	go r.PrintProgress()
+	_, err = io.Copy(df, r)
+	close(r.done)
+	return
+}
+
+type progressReader struct {
+	f    provider.File
+	done chan struct{}
+	size int64
+	n    int64
+}
+
+func (p *progressReader) Read(b []byte) (n int, err error) {
+	n, err = p.f.Read(b)
+	atomic.AddInt64(&p.n, int64(n))
+	return
+}
+
+func (p *progressReader) ReadAt(b []byte, off int64) (n int, err error) {
+	n, err = p.f.ReadAt(b, off)
+	atomic.AddInt64(&p.n, int64(n))
+	return
+}
+
+func (p *progressReader) PrintProgress() {
+	for {
+		select {
+		case <-p.done:
+			fmt.Println()
+			return
+		case <-time.After(time.Millisecond * 500):
+		}
+		n := atomic.LoadInt64(&p.n)
+		percent := int(math.Round(float64(n) / float64(p.size) * 100))
+		fmt.Printf("progress: %d / %d %d%%\r", n, p.size, percent)
+	}
 	return
 }
 

cmd/metadatagen/main.go

@@ -17,32 +17,39 @@ var (
 
 	dst      string
 	metadata string
+	fstat    bool
 )
 
+func fatal(err error) {
+	fmt.Println(err)
+	os.Exit(1)
+}
+
 func main() {
-	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
+	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache, uses DST_KEY)")
 	flag.StringVar(&metadata, "data", "", "path to metadata file")
+	flag.BoolVar(&fstat, "fstat", false, "use fstat to determine real file size for sparse files")
 	flag.Parse()
 
 	var (
-		dstk []byte
-		err  error
+		dstk      []byte
+		err       error
+		encrypted bool
 	)
 	dstkey := os.Getenv("DST_KEY")
 	if dstkey != "" {
 		dstk, err = hex.DecodeString(dstkey)
-		fmt.Println(err)
-		os.Exit(1)
+		if err != nil {
+			fatal(err)
+		}
+		encrypted = true
 	}
 	dstfs, err := parse.FS(dst, dstk, true)
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		fatal(err)
 	}
-	err = fs.MetadataGenerator(metadata, dstfs)
+	err = fs.MetadataGenerator(metadata, dstfs, fstat, encrypted)
 	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+		fatal(err)
 	}
-	os.Exit(0)
 }

pkg/fs/file.go

@@ -43,9 +43,6 @@ func (p *preload) Read(data []byte) (n int, err error) {
 		return
 	}
 	n, err = p.f.readToCache(data)
-	if n < len(data) {
-		_, err = p.f.Seek(p.f.offset, io.SeekStart)
-	}
 	p.written++
 	return
 }
@@ -137,6 +134,9 @@ func (f *File) readToCache(p []byte) (n int, err error) {
 	}
 	f.md.AddChunk(f.offset, n)
 	f.offset += int64(n)
+	if n > 0 && n < len(p) {
+		_, err = f.Seek(f.offset, io.SeekStart)
+	}
 	return
 }
 

pkg/fs/metadata.go

@@ -37,13 +37,15 @@ type MetadataHandler struct {
 	enc *json.Encoder
 }
 
-func MetadataGenerator(file string, dst provider.FS) error {
+func MetadataGenerator(file string, dst provider.FS, fstat, encrypted bool) error {
 	if !filepath.IsAbs(file) {
 		return errors.New("metadata path is not absolute")
 	}
-	_, err := dst.Fstat(0)
-	if err == provider.ErrFstat {
-		return err
+	if fstat {
+		_, err := dst.Fstat(0)
+		if err == provider.ErrFstat {
+			return err
+		}
 	}
 	f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640)
 	if err != nil {
@@ -56,7 +58,11 @@ func MetadataGenerator(file string, dst provider.FS) error {
 	}
 	enc := json.NewEncoder(f)
 	md := make(map[string]*Metadata)
-	fs.WalkDir(provider.WalkFS(dst), ".", func(path string, d stdfs.DirEntry, err error) error {
+	base := "."
+	if encrypted {
+		base = ""
+	}
+	fs.WalkDir(provider.WalkFS(dst), base, func(path string, d stdfs.DirEntry, err error) error {
 		if err != nil {
 			return nil
 		}
@@ -74,11 +80,18 @@ func MetadataGenerator(file string, dst provider.FS) error {
 			return nil
 		}
 		size := fi.Size()
-		blocks, err := dst.Fstat(f.Fd())
-		if err != nil {
-			return nil
-		}
-		if size <= blocks*512 {
+		if fstat {
+			blocks, err := dst.Fstat(f.Fd())
+			if err != nil {
+				return nil
+			}
+			if size <= blocks*512 {
+				md[strings.TrimPrefix(path, dst.Root())] = &Metadata{
+					Size:   size,
+					Chunks: chunk.Chunks{{0, size}},
+				}
+			}
+		} else {
 			md[strings.TrimPrefix(path, dst.Root())] = &Metadata{
 				Size:   size,
 				Chunks: chunk.Chunks{{0, size}},