major refactoring

pkg/fs/metadata.go

@@ -3,17 +3,220 @@
 package fs
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	stdfs "io/fs"
+	"math"
 	"os"
+	"path/filepath"
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/go-logr/logr"
+	"golang.org/x/exp/maps"
 )
 
 func now() int64 {
 	return time.Now().UTC().Unix()
 }
 
-type MetadataMap map[string]*Metadata
+type MetadataHandler struct {
+	mu  sync.RWMutex
+	log logr.Logger
+	md  map[string]*Metadata
+	fs  *FS
+	dst string
+	f   *os.File
+	enc *json.Encoder
+}
+
+func NewMetadataHandler(ctx context.Context, fs *FS, file, dst string, log logr.Logger) (mh *MetadataHandler, err error) {
+	if !filepath.IsAbs(file) {
+		return nil, errors.New("metadata path is not absolute")
+	}
+	mh = &MetadataHandler{
+		log: log,
+		md:  make(map[string]*Metadata),
+		fs:  fs,
+		dst: dst,
+	}
+	mh.f, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640)
+	if err != nil {
+		return
+	}
+	err = json.NewDecoder(mh.f).Decode(&mh.md)
+	if err != nil && err == io.EOF {
+		return
+	}
+	mh.enc = json.NewEncoder(mh.f)
+	err = nil
+	mh.init()
+	go mh.flusher(ctx)
+	return
+}
+
+func (mh *MetadataHandler) DeleteOldest() (s int64) {
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	atime := int64(math.MaxInt64)
+	var m *Metadata
+	for _, v := range mh.md {
+		if v.Atime != 0 && v.f != nil && v.Atime < atime {
+			atime = v.Atime
+			m = v
+		}
+	}
+	if m != nil {
+		log := mh.log.WithValues("file", m.name)
+		log.Info("deleting oldest file")
+		err := m.Delete()
+		if err != nil {
+			log.Error(err, "error deleting oldest file")
+			return
+		}
+		s = m.Size
+	}
+	return
+}
+
+func (mh *MetadataHandler) Size() (s int64) {
+	mh.mu.RLock()
+	defer mh.mu.RUnlock()
+	for _, md := range mh.md {
+		s += md.Chunks.Size()
+	}
+	return
+}
+
+func (mh *MetadataHandler) CacheStatus(name string) int {
+	mh.mu.RLock()
+	md, ok := mh.md[name]
+	mh.mu.RUnlock()
+	if !ok {
+		return -1
+	}
+	size := md.Chunks.Size()
+	if size == 0 {
+		return -1
+	}
+	return int(math.Round(float64(size) / float64(md.Size) * 100))
+}
+
+func (mh *MetadataHandler) Metadata(name string, size int64) (md *Metadata) {
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	if md, ok := mh.md[name]; ok {
+		if size > 0 && md.Size != size {
+			log := mh.log.WithValues("file", name)
+			log.V(2).Info("source file changed: deleting cache file", "src", size, "dst", md.Size)
+			err := md.Delete()
+			if err != nil {
+				log.Error(err, "error deleting cache file")
+			}
+		}
+		return md
+	}
+	md = &Metadata{Size: size, fs: mh.fs, name: name}
+	mh.md[name] = md
+	return
+}
+
+func (mh *MetadataHandler) init() {
+	log := mh.log
+	maps.DeleteFunc(mh.md, func(k string, v *Metadata) bool {
+		_, err := mh.fs.statDst(k)
+		if errors.Is(err, os.ErrNotExist) {
+			log.V(2).Info("removing not existing file from metadata", "file", k)
+			return true
+		}
+		if len(v.Chunks) == 0 {
+			log.V(2).Info("removing empty file", "file", k)
+			err := mh.fs.RemoveDst(k)
+			if err != nil {
+				log.Error(err, "error removing empty file", "file", k)
+				return false
+			}
+			return true
+		}
+		v.fs = mh.fs
+		v.name = k
+		return false
+	})
+}
+
+func (mh *MetadataHandler) flusher(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			mh.close()
+			return
+		case <-time.After(time.Minute * 5):
+		}
+		mh.cleanupEmptyDirs()
+		mh.flush()
+	}
+}
+
+func (mh *MetadataHandler) cleanupEmptyDirs() {
+	filepath.WalkDir(mh.dst, func(path string, d stdfs.DirEntry, err error) error {
+		if err == nil && d.IsDir() {
+			if path == mh.dst {
+				return nil
+			}
+			empty, err := dirEmpty(path)
+			if err != nil || !empty {
+				return nil
+			}
+			log := mh.log.WithValues("dir", path)
+			log.V(2).Info("removing empty dir")
+			err = os.Remove(path)
+			if err != nil {
+				log.Error(err, "failed to remove empty dir")
+			}
+		}
+		return nil
+	})
+}
+
+func (mh *MetadataHandler) flush() {
+	log := mh.log
+	mh.mu.Lock()
+	defer mh.mu.Unlock()
+	log.V(2).Info("flushing metadata to disk")
+	md := maps.Clone(mh.md)
+	maps.DeleteFunc(mh.md, func(_ string, v *Metadata) bool {
+		return len(v.Chunks) == 0
+	})
+	_, err := mh.f.Seek(0, io.SeekStart)
+	if err != nil {
+		log.Error(err, "failure seeking metadata file")
+		return
+	}
+	err = mh.f.Truncate(0)
+	if err != nil {
+		log.Error(err, "failure truncating metadata file")
+		return
+	}
+	err = mh.enc.Encode(md)
+	if err != nil {
+		log.Error(err, "failure flushing metadata file")
+	}
+	return
+}
+
+func (mh *MetadataHandler) close() {
+	mh.mu.Lock()
+	for _, md := range mh.md {
+		md.f.Sync()
+		md.f.Close()
+	}
+	mh.mu.Unlock()
+	mh.flush()
+	mh.f.Close()
+}
 
 type Metadata struct {
 	mu     sync.RWMutex `json:"-"`