diff --git a/cmd/cachectl/main.go b/cmd/cachectl/main.go index 9bf2b32..a92a1f5 100644 --- a/cmd/cachectl/main.go +++ b/cmd/cachectl/main.go @@ -3,15 +3,19 @@ package main import ( + "cachefs/pkg/provider" "cachefs/pkg/provider/parse" "encoding/hex" "errors" "flag" "fmt" "io" + "math" "os" "path/filepath" "strings" + "sync/atomic" + "time" ) var ( @@ -176,7 +180,52 @@ func cp(args []string) (err error) { return } defer df.Close() - _, err = io.Copy(df, sf) + fi, err := sf.Stat() + if err != nil { + return + } + r := &progressReader{ + f: sf, + done: make(chan struct{}), + size: fi.Size(), + } + go r.PrintProgress() + _, err = io.Copy(df, r) + close(r.done) + return +} + +type progressReader struct { + f provider.File + done chan struct{} + size int64 + n int64 +} + +func (p *progressReader) Read(b []byte) (n int, err error) { + n, err = p.f.Read(b) + atomic.AddInt64(&p.n, int64(n)) + return +} + +func (p *progressReader) ReadAt(b []byte, off int64) (n int, err error) { + n, err = p.f.ReadAt(b, off) + atomic.AddInt64(&p.n, int64(n)) + return +} + +func (p *progressReader) PrintProgress() { + for { + select { + case <-p.done: + fmt.Println() + return + case <-time.After(time.Millisecond * 500): + } + n := atomic.LoadInt64(&p.n) + percent := int(math.Round(float64(n) / float64(p.size) * 100)) + fmt.Printf("progress: %d / %d %d%%\r", n, p.size, percent) + } return } diff --git a/cmd/metadatagen/main.go b/cmd/metadatagen/main.go index 308db1a..ca7c875 100644 --- a/cmd/metadatagen/main.go +++ b/cmd/metadatagen/main.go @@ -17,32 +17,39 @@ var ( dst string metadata string + fstat bool ) +func fatal(err error) { + fmt.Println(err) + os.Exit(1) +} + func main() { - flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)") + flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache, uses DST_KEY)") flag.StringVar(&metadata, "data", "", "path to metadata file") + flag.BoolVar(&fstat, "fstat", false, "use fstat to determine real file size for sparse files") flag.Parse() var ( - dstk []byte - err error + dstk []byte + err error + encrypted bool ) dstkey := os.Getenv("DST_KEY") if dstkey != "" { dstk, err = hex.DecodeString(dstkey) - fmt.Println(err) - os.Exit(1) + if err != nil { + fatal(err) + } + encrypted = true } dstfs, err := parse.FS(dst, dstk, true) if err != nil { - fmt.Println(err) - os.Exit(1) + fatal(err) } - err = fs.MetadataGenerator(metadata, dstfs) + err = fs.MetadataGenerator(metadata, dstfs, fstat, encrypted) if err != nil { - fmt.Println(err) - os.Exit(1) + fatal(err) } - os.Exit(0) } diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 8cd7342..ed36431 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -43,9 +43,6 @@ func (p *preload) Read(data []byte) (n int, err error) { return } n, err = p.f.readToCache(data) - if n < len(data) { - _, err = p.f.Seek(p.f.offset, io.SeekStart) - } p.written++ return } @@ -137,6 +134,9 @@ func (f *File) readToCache(p []byte) (n int, err error) { } f.md.AddChunk(f.offset, n) f.offset += int64(n) + if n > 0 && n < len(p) { + _, err = f.Seek(f.offset, io.SeekStart) + } return } diff --git a/pkg/fs/metadata.go b/pkg/fs/metadata.go index d1df085..c621942 100644 --- a/pkg/fs/metadata.go +++ b/pkg/fs/metadata.go @@ -37,13 +37,15 @@ type MetadataHandler struct { enc *json.Encoder } -func MetadataGenerator(file string, dst provider.FS) error { +func MetadataGenerator(file string, dst provider.FS, fstat, encrypted bool) error { if !filepath.IsAbs(file) { return errors.New("metadata path is not absolute") } - _, err := dst.Fstat(0) - if err == provider.ErrFstat { - return err + if fstat { + _, err := dst.Fstat(0) + if err == provider.ErrFstat { + return err + } } f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640) if err != nil { @@ -56,7 +58,11 @@ func MetadataGenerator(file string, dst provider.FS) error { } enc := json.NewEncoder(f) md := make(map[string]*Metadata) - fs.WalkDir(provider.WalkFS(dst), ".", func(path string, d stdfs.DirEntry, err error) error { + base := "." + if encrypted { + base = "" + } + fs.WalkDir(provider.WalkFS(dst), base, func(path string, d stdfs.DirEntry, err error) error { if err != nil { return nil } @@ -74,11 +80,18 @@ func MetadataGenerator(file string, dst provider.FS) error { return nil } size := fi.Size() - blocks, err := dst.Fstat(f.Fd()) - if err != nil { - return nil - } - if size <= blocks*512 { + if fstat { + blocks, err := dst.Fstat(f.Fd()) + if err != nil { + return nil + } + if size <= blocks*512 { + md[strings.TrimPrefix(path, dst.Root())] = &Metadata{ + Size: size, + Chunks: chunk.Chunks{{0, size}}, + } + } + } else { md[strings.TrimPrefix(path, dst.Root())] = &Metadata{ Size: size, Chunks: chunk.Chunks{{0, size}},