// Copyright (C) 2022 Marius Schellenberger package fs import ( "cachefs/pkg/config" "cachefs/pkg/provider" "cachefs/pkg/provider/parse" "context" "encoding/hex" "errors" "fmt" stdfs "io/fs" "net/http" "os" "regexp" "strconv" "strings" "github.com/go-logr/logr" ) type FS struct { log logr.Logger cancel func() pCancel func() NoCache http.Handler src provider.FS dst provider.FS dc *DirCache sc *StatCache q *Quota mh *MetadataHandler ph *PreloadHandler } func NewFSFromConfig(cfg *config.Config, log logr.Logger) (fs *FS, err error) { c := cfg.Cache return NewFS( c.Quota, c.Preloads, c.Src.Path, c.Src.Key, c.Dst.Path, c.Dst.Key, c.Metadata, cfg.REPath, *c.BlockSFTP, log, ) } func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, re *regexp.Regexp, block bool, log logr.Logger) (fs *FS, err error) { if src == dst { return nil, errors.New("src and dst path can not be equal") } var srck, dstk []byte if srckey != "" { srck, err = hex.DecodeString(srckey) if err != nil { return nil, fmt.Errorf("srckey: %w", err) } log.V(2).Info("using source encryption") } if dstkey != "" { dstk, err = hex.DecodeString(dstkey) if err != nil { return nil, fmt.Errorf("dstkey: %w", err) } log.V(2).Info("using destination encryption") } srcfs, err := parse.FS(src, srck, re, block) if err != nil { return nil, fmt.Errorf("srcfs: %w", err) } dstfs, err := parse.FS(dst, dstk, nil, block) if err != nil { return nil, fmt.Errorf("dstfs: %w", err) } ctx, cancel := context.WithCancel(context.Background()) pCtx, pCancel := context.WithCancel(context.Background()) fs = &FS{ log: log, cancel: cancel, pCancel: pCancel, NoCache: http.FileServer(http.Dir(src)), src: srcfs, dst: dstfs, dc: NewDirCache(), } fs.mh, err = NewMetadataHandler(ctx, fs, metadata, dst, log.WithName("metadata")) if err != nil { return nil, fmt.Errorf("metadata: %w", err) } fs.q, err = NewQuota(fs.mh.Size(), quota, fs, log.WithName("quota")) if err != nil { return nil, fmt.Errorf("quota: %w", err) } fs.ph = NewPreloadHandler(pCtx, fs, max, log.WithName("preload")) fs.sc = NewStatCache(ctx) return } func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) { fi, err = fs.sc.Get(name) if err == nil { return } fi, err = fs.src.Stat(name) if err == nil { fs.sc.Set(name, fi) return } fi, err = fs.dst.Stat(name) return } const ( _ = 1 << (iota * 10) KiB MiB GiB TiB ) func FileSize(s int64) string { switch { case s >= TiB: return strconv.FormatFloat(float64(s)/TiB, 'f', 2, 64) + " TiB" case s >= GiB: return strconv.FormatFloat(float64(s)/GiB, 'f', 2, 64) + " GiB" case s >= MiB: return strconv.FormatFloat(float64(s)/MiB, 'f', 2, 64) + " MiB" case s >= KiB: return strconv.FormatFloat(float64(s)/KiB, 'f', 2, 64) + " KiB" } return strconv.Itoa(int(s)) + " B" } func (fs *FS) FileSize(name string) string { fi, err := fs.Stat(name) if err != nil { return "" } return FileSize(fi.Size()) } func (fs *FS) RemoveDst(name string) error { return fs.dst.Remove(name) } func (fs *FS) StatDst(name string) (fi stdfs.FileInfo, err error) { fi, err = fs.sc.Get(name) if err == nil { return } return fs.dst.Stat(name) } func (fs *FS) OpenDst(name string) (f http.File, err error) { log := fs.log.WithValues("file", name) fi, err := fs.StatDst(name) if err != nil { if !skipLog(name) { log.Error(err, "error stat cache file") } return } file, err := fs.dst.Open(name) if err != nil { if !skipLog(name) { log.Error(err, "error opening cache file") } return } if fi.IsDir() { f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc} return } md := fs.mh.Metadata(name, fi.Size()) f = &File{ log: log.WithName("file"), f: file, cache: file, md: md, offline: true, } return } func (fs *FS) QuotaUsage() (cur, max int64) { return fs.q.Usage() } func (fs *FS) RemovePreload(name string) { fs.ph.RemovePreload(name) } func (fs *FS) Preloads() Preloads { return fs.ph.Preloads() } func (fs *FS) Preload(name string, prio int) { fs.ph.Preload(name, prio) } func skipLog(name string) bool { return strings.HasSuffix(name, "index.html") || strings.HasSuffix(name, "favicon.ico") } func (fs *FS) open(name string) (f provider.File, fi os.FileInfo, offline bool, err error) { f, err = fs.src.Open(name) if err == nil { fi, err = f.Stat() if err == nil { return } f.Close() } f, err = fs.dst.Open(name) if err != nil { return } fi, err = f.Stat() if err != nil { f.Close() } offline = true return } func (fs *FS) Open(name string) (f http.File, err error) { log := fs.log.WithValues("file", name) file, fi, offline, err := fs.open(name) if err != nil { if !skipLog(name) { log.Error(err, "error opening source file") } return } if fi.IsDir() { if offline { log.V(2).Info("dir offline mode") } f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc} return } size := fi.Size() md := fs.mh.Metadata(name, size) var cache provider.File if offline { log.V(2).Info("file offline mode") cache = file } else { cache, err = fs.openCacheFile(name, size) if err != nil { if !skipLog(name) { log.Error(err, "error opening cache file") } return } } f = &File{ log: log.WithName("file"), f: file, cache: cache, md: md, offline: offline, } return } func (fs *FS) openCacheFile(name string, size int64) (df provider.File, err error) { log := fs.log if i := strings.LastIndex(name, "/"); i > 0 { dir := name[:i] _, err = fs.dst.Stat(dir) if errors.Is(err, os.ErrNotExist) { err = fs.dst.MkdirAll(dir, 0o755) if err != nil { log.Error(err, "error creating cache dir") return } } } _, err = fs.dst.Stat(name) truncate := errors.Is(err, os.ErrNotExist) df, err = fs.dst.OpenFile(name, os.O_RDWR|os.O_CREATE, 0o644) if err != nil { log.Error(err, "error opening cache file") return } if truncate && size > 0 { err = df.Truncate(size) if err != nil { log.Error(err, "error truncating cache file") return } err = df.Sync() if err != nil { log.Error(err, "error syncing cache file") return } } return } func (fs *FS) CacheStatus(name string) int { return fs.mh.CacheStatus(name) } func (fs *FS) Close() { fs.pCancel() <-fs.ph.Done() fs.cancel() fs.src.Close() fs.dst.Close() } func (fs *FS) Done() <-chan struct{} { return fs.mh.Done() }