cachefs/pkg/fs/fs.go

// Copyright (C) 2022 Marius Schellenberger

package fs

import (
	"cachefs/pkg/config"
	"cachefs/pkg/provider"
	"cachefs/pkg/provider/parse"
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	stdfs "io/fs"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/go-logr/logr"
)

type FS struct {
	log     logr.Logger
	cancel  func()
	pCancel func()
	NoCache http.Handler
	src     provider.FS
	dst     provider.FS
	dc      *DirCache
	sc      *StatCache

	q  *Quota
	mh *MetadataHandler
	ph *PreloadHandler
}

func NewFSFromConfig(cfg *config.Config, log logr.Logger) (fs *FS, err error) {
	c := cfg.Cache
	return NewFS(
		c.Quota,
		c.Preloads,
		c.Src.Path,
		c.Src.Key,
		c.Dst.Path,
		c.Dst.Key,
		c.Metadata,
		cfg.REPath,
		*c.BlockSFTP,
		log,
	)
}
func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, re *regexp.Regexp, block bool, log logr.Logger) (fs *FS, err error) {
	if src == dst {
		return nil, errors.New("src and dst path can not be equal")
	}
	var srck, dstk []byte
	if srckey != "" {
		srck, err = hex.DecodeString(srckey)
		if err != nil {
			return nil, fmt.Errorf("srckey: %w", err)
		}
		log.V(2).Info("using source encryption")
	}
	if dstkey != "" {
		dstk, err = hex.DecodeString(dstkey)
		if err != nil {
			return nil, fmt.Errorf("dstkey: %w", err)
		}
		log.V(2).Info("using destination encryption")
	}
	srcfs, err := parse.FS(src, srck, re, block)
	if err != nil {
		return nil, fmt.Errorf("srcfs: %w", err)
	}
	dstfs, err := parse.FS(dst, dstk, nil, block)
	if err != nil {
		return nil, fmt.Errorf("dstfs: %w", err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	pCtx, pCancel := context.WithCancel(context.Background())

	fs = &FS{
		log:     log,
		cancel:  cancel,
		pCancel: pCancel,
		NoCache: http.FileServer(http.Dir(src)),
		src:     srcfs,
		dst:     dstfs,
		dc:      NewDirCache(),
	}

	fs.mh, err = NewMetadataHandler(ctx, fs, metadata, dst, log.WithName("metadata"))
	if err != nil {
		return nil, fmt.Errorf("metadata: %w", err)
	}
	fs.q, err = NewQuota(fs.mh.Size(), quota, fs, log.WithName("quota"))
	if err != nil {
		return nil, fmt.Errorf("quota: %w", err)
	}
	fs.ph = NewPreloadHandler(pCtx, fs, max, log.WithName("preload"))
	fs.sc = NewStatCache(ctx)
	return
}

func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
	fi, err = fs.sc.Get(name)
	if err == nil {
		return
	}
	fi, err = fs.src.Stat(name)
	if err == nil {
		fs.sc.Set(name, fi)
		return
	}
	fi, err = fs.dst.Stat(name)
	return
}

const (
	_ = 1 << (iota * 10)
	KiB
	MiB
	GiB
	TiB
)

func FileSize(s int64) string {
	switch {
	case s >= TiB:
		return strconv.FormatFloat(float64(s)/TiB, 'f', 2, 64) + " TiB"
	case s >= GiB:
		return strconv.FormatFloat(float64(s)/GiB, 'f', 2, 64) + " GiB"
	case s >= MiB:
		return strconv.FormatFloat(float64(s)/MiB, 'f', 2, 64) + " MiB"
	case s >= KiB:
		return strconv.FormatFloat(float64(s)/KiB, 'f', 2, 64) + " KiB"
	}
	return strconv.Itoa(int(s)) + " B"
}

func (fs *FS) FileSize(name string) string {
	fi, err := fs.Stat(name)
	if err != nil {
		return ""
	}
	return FileSize(fi.Size())
}

func (fs *FS) RemoveDst(name string) error {
	return fs.dst.Remove(name)
}

func (fs *FS) StatDst(name string) (fi stdfs.FileInfo, err error) {
	fi, err = fs.sc.Get(name)
	if err == nil {
		return
	}
	return fs.dst.Stat(name)
}

func (fs *FS) OpenDst(name string) (f http.File, err error) {
	log := fs.log.WithValues("file", name)
	fi, err := fs.StatDst(name)
	if err != nil {
		if !skipLog(name) {
			log.Error(err, "error stat cache file")
		}
		return
	}
	file, err := fs.dst.Open(name)
	if err != nil {
		if !skipLog(name) {
			log.Error(err, "error opening cache file")
		}
		return
	}
	if fi.IsDir() {
		f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
		return
	}
	md := fs.mh.Metadata(name, fi.Size())
	f = &File{
		log:     log.WithName("file"),
		f:       file,
		cache:   file,
		md:      md,
		offline: true,
	}
	return
}

func (fs *FS) QuotaUsage() (cur, max int64) {
	return fs.q.Usage()
}

func (fs *FS) RemovePreload(name string) {
	fs.ph.RemovePreload(name)
}

func (fs *FS) Preloads() Preloads {
	return fs.ph.Preloads()
}

func (fs *FS) Preload(name string, prio int) {
	fs.ph.Preload(name, prio)
}

func skipLog(name string) bool {
	return strings.HasSuffix(name, "index.html") || strings.HasSuffix(name, "favicon.ico")
}

func (fs *FS) open(name string) (f provider.File, fi os.FileInfo, offline bool, err error) {
	f, err = fs.src.Open(name)
	if err == nil {
		fi, err = f.Stat()
		if err == nil {
			return
		}
		f.Close()
	}
	f, err = fs.dst.Open(name)
	if err != nil {
		return
	}
	fi, err = f.Stat()
	if err != nil {
		f.Close()
	}
	offline = true
	return
}

func (fs *FS) Open(name string) (f http.File, err error) {
	log := fs.log.WithValues("file", name)
	file, fi, offline, err := fs.open(name)
	if err != nil {
		if !skipLog(name) {
			log.Error(err, "error opening source file")
		}
		return
	}
	if fi.IsDir() {
		if offline {
			log.V(2).Info("dir offline mode")
		}
		f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
		return
	}
	size := fi.Size()
	md := fs.mh.Metadata(name, size)

	var cache provider.File
	if offline {
		log.V(2).Info("file offline mode")
		cache = file
	} else {
		cache, err = fs.openCacheFile(name, size)
		if err != nil {
			if !skipLog(name) {
				log.Error(err, "error opening cache file")
			}
			return
		}
	}
	f = &File{
		log:     log.WithName("file"),
		f:       file,
		cache:   cache,
		md:      md,
		offline: offline,
	}
	return
}

func (fs *FS) openCacheFile(name string, size int64) (df provider.File, err error) {
	log := fs.log
	if i := strings.LastIndex(name, "/"); i > 0 {
		dir := name[:i]
		_, err = fs.dst.Stat(dir)
		if errors.Is(err, os.ErrNotExist) {
			err = fs.dst.MkdirAll(dir, 0o755)
			if err != nil {
				log.Error(err, "error creating cache dir")
				return
			}
		}
	}
	_, err = fs.dst.Stat(name)
	truncate := errors.Is(err, os.ErrNotExist)
	df, err = fs.dst.OpenFile(name, os.O_RDWR|os.O_CREATE, 0o644)
	if err != nil {
		log.Error(err, "error opening cache file")
		return
	}
	if truncate && size > 0 {
		err = df.Truncate(size)
		if err != nil {
			log.Error(err, "error truncating cache file")
			return
		}
		err = df.Sync()
		if err != nil {
			log.Error(err, "error syncing cache file")
			return
		}
	}
	return
}

func (fs *FS) CacheStatus(name string) int {
	return fs.mh.CacheStatus(name)
}

func (fs *FS) Close() {
	fs.pCancel()
	<-fs.ph.Done()

	fs.cancel()
	fs.src.Close()
	fs.dst.Close()
}

func (fs *FS) Done() <-chan struct{} {
	return fs.mh.Done()
}