switch to config file and implement src path filter

pkg/config/config.go

@@ -0,0 +1,106 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package config
+
+import (
+	"errors"
+	"os"
+	"regexp"
+
+	"gopkg.in/yaml.v3"
+)
+
+func ParseFile(file string) (cfg *Config, err error) {
+	if file == "" {
+		return nil, errors.New("missing config file")
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	cfg = new(Config)
+	err = yaml.NewDecoder(f).Decode(cfg)
+	if err != nil {
+		return
+	}
+	err = Validate(cfg)
+	return
+}
+
+type Config struct {
+	Server Server         `yaml:"server"`
+	Cache  Cache          `yaml:"cache"`
+	Path   string         `yaml:"path"`
+	REPath *regexp.Regexp `yaml:"-"`
+}
+
+type Server struct {
+	HTTP   Listen `yaml:"http"`
+	WebDav Listen `yaml:"webdav"`
+	Cache  Listen `yaml:"cache"`
+	Plain  Listen `yaml:"plain"`
+}
+
+type Listen struct {
+	Addr string `yaml:"addr"`
+}
+
+type Cache struct {
+	Metadata  string `yaml:"metadata"`
+	Src       Path   `yaml:"src"`
+	Dst       Path   `yaml:"dst"`
+	Preloads  int    `yaml:"preloads"`
+	Buffer    int    `yaml:"buffer"`
+	Quota     int64  `yaml:"quota"`
+	BlockSFTP *bool  `yaml:"blockSFTP"`
+}
+
+type Path struct {
+	Path string `yaml:"path"`
+	Key  string `yaml:"key"`
+}
+
+const (
+	defListenHTTP = "127.0.0.1:8080"
+	defPreloads   = 1
+	defBuffer     = 8192
+	defQuota      = 1
+	defBlockSFTP  = true
+	gib           = 1024 * 1024 * 1024
+)
+
+func Validate(cfg *Config) error {
+	if cfg.Server.HTTP.Addr == "" {
+		cfg.Server.HTTP.Addr = defListenHTTP
+	}
+	c := cfg.Cache
+	if c.Metadata == "" {
+		return errors.New("cache.metadata is empty")
+	}
+	if c.Src.Path == "" {
+		return errors.New("cache.src is empty")
+	}
+	if c.Dst.Path == "" {
+		return errors.New("cache.dst is empty")
+	}
+	if c.Preloads <= 0 {
+		c.Preloads = defPreloads
+	}
+	if c.Buffer < 1024 {
+		c.Buffer = defBuffer
+	}
+	if c.Quota < 1 {
+		c.Quota = defQuota
+	}
+	c.Quota *= gib
+	if c.BlockSFTP == nil {
+		c.BlockSFTP = new(bool)
+		*c.BlockSFTP = defBlockSFTP
+	}
+	var err error
+	if cfg.Path != "" {
+		cfg.REPath, err = regexp.Compile(cfg.Path)
+	}
+	return err
+}

pkg/fs/fs.go

@@ -3,6 +3,7 @@
 package fs
 
 import (
+	"cachefs/pkg/config"
 	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/parse"
 	"context"
@@ -12,6 +13,7 @@ import (
 	stdfs "io/fs"
 	"net/http"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -33,7 +35,22 @@ type FS struct {
 	ph *PreloadHandler
 }
 
-func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, block bool, log logr.Logger) (fs *FS, err error) {
+func NewFSFromConfig(cfg *config.Config, log logr.Logger) (fs *FS, err error) {
+	c := cfg.Cache
+	return NewFS(
+		c.Quota,
+		c.Preloads,
+		c.Src.Path,
+		c.Src.Key,
+		c.Dst.Path,
+		c.Dst.Key,
+		c.Metadata,
+		cfg.REPath,
+		*c.BlockSFTP,
+		log,
+	)
+}
+func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, re *regexp.Regexp, block bool, log logr.Logger) (fs *FS, err error) {
 	if src == dst {
 		return nil, errors.New("src and dst path can not be equal")
 	}
@@ -52,11 +69,11 @@ func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, bloc
 		}
 		log.V(2).Info("using destination encryption")
 	}
-	srcfs, err := parse.FS(src, srck, block)
+	srcfs, err := parse.FS(src, srck, re, block)
 	if err != nil {
 		return nil, fmt.Errorf("srcfs: %w", err)
 	}
-	dstfs, err := parse.FS(dst, dstk, block)
+	dstfs, err := parse.FS(dst, dstk, nil, block)
 	if err != nil {
 		return nil, fmt.Errorf("dstfs: %w", err)
 	}

pkg/provider/filter/file.go

@@ -0,0 +1,53 @@
+package filter
+
+import (
+	"cachefs/pkg/provider"
+	"io/fs"
+	"path/filepath"
+)
+
+var _ provider.File = (*file)(nil)
+
+type file struct {
+	provider.File
+	fs   *FS
+	path string
+}
+
+func newFile(f provider.File, fs *FS, path string) (nf provider.File, err error) {
+	nf = &file{
+		File: f,
+		fs:   fs,
+		path: path,
+	}
+	return nf, nil
+}
+
+func (f *file) Readdir(n int) ([]fs.FileInfo, error) {
+	fis, err := f.File.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.FileInfo, len(fis))
+	i := 0
+	for _, fi := range fis {
+		file := filepath.Join(f.path, fi.Name())
+		if f.fs.match(file) == nil {
+			fio[i] = fi
+			i++
+		}
+	}
+	return fio[:i], nil
+}
+
+func (f *file) ReadDir(n int) ([]fs.DirEntry, error) {
+	fis, err := f.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.DirEntry, len(fis))
+	for i, fi := range fis {
+		fio[i] = fs.FileInfoToDirEntry(fi)
+	}
+	return fio, nil
+}

pkg/provider/filter/fs.go

@@ -0,0 +1,78 @@
+package filter
+
+import (
+	stdfs "io/fs"
+	"os"
+	"regexp"
+
+	"cachefs/pkg/provider"
+)
+
+var _ provider.FS = (*FS)(nil)
+
+type FS struct {
+	provider.FS
+	re *regexp.Regexp
+}
+
+func NewFS(fs provider.FS, re *regexp.Regexp) (*FS, error) {
+	return &FS{
+		FS: fs,
+		re: re,
+	}, nil
+}
+
+func (fs *FS) match(p string) error {
+	if fs.re.MatchString(p) {
+		return nil
+	}
+	return stdfs.ErrNotExist
+}
+
+func (fs *FS) Stat(p string) (stdfs.FileInfo, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	return fs.FS.Stat(p)
+}
+
+func (fs *FS) Remove(p string) error {
+	err := fs.match(p)
+	if err != nil {
+		return err
+	}
+	return fs.FS.Remove(p)
+}
+
+func (fs *FS) Open(p string) (provider.File, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.FS.Open(p)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(f, fs, p)
+}
+
+func (fs *FS) OpenFile(p string, flags int, mode os.FileMode) (provider.File, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.FS.OpenFile(p, flags, mode)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(f, fs, p)
+}
+
+func (fs *FS) MkdirAll(p string, mode os.FileMode) error {
+	err := fs.match(p)
+	if err != nil {
+		return err
+	}
+	return fs.FS.MkdirAll(p, mode)
+}

pkg/provider/parse/parse.go

@@ -5,12 +5,14 @@ package parse
 import (
 	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/crypto"
+	"cachefs/pkg/provider/filter"
 	"cachefs/pkg/provider/os"
 	"cachefs/pkg/provider/sftp"
 	"errors"
 	"fmt"
 	neturl "net/url"
 	"path/filepath"
+	"regexp"
 
 	"k8s.io/klog/v2/klogr"
 )
@@ -36,7 +38,7 @@ func TryParse(url string) error {
 	return nil
 }
 
-func FS(url string, key []byte, block bool) (provider.FS, error) {
+func FS(url string, key []byte, re *regexp.Regexp, block bool) (provider.FS, error) {
 	u, err := neturl.Parse(url)
 	if err != nil {
 		return nil, err
@@ -58,6 +60,12 @@ func FS(url string, key []byte, block bool) (provider.FS, error) {
 	}
 	if key != nil {
 		fs, err = crypto.NewFS(fs, key)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if re != nil {
+		fs, err = filter.NewFS(fs, re)
 	}
 	return fs, err
 }

pkg/srv/plain.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package srv
+
+import (
+	"net/http"
+	"strings"
+
+	"cachefs/pkg/fs"
+
+	"github.com/go-logr/logr"
+)
+
+type PlainServer struct {
+	log logr.Logger
+	fs  *fs.FS
+	h   http.Handler
+}
+
+func NewPlainServer(fs *fs.FS, log logr.Logger) http.Handler {
+	return &PlainServer{log, fs, http.FileServer(fs)}
+}
+
+func skipPlainLog(path string) bool {
+	return strings.HasSuffix(path, "favicon.ico")
+}
+
+func (ps *PlainServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s := &statusInterceptor{w: w}
+	if !skipPlainLog(r.URL.Path) {
+		defer func() {
+			if s.Status() == http.StatusPartialContent {
+				return
+			}
+			ps.log.Info("access",
+				"client", r.RemoteAddr,
+				"method", r.Method,
+				"status", s.Status(),
+				"uri", r.URL.Path,
+			)
+		}()
+	}
+	r.Header.Del("If-Modified-Since")
+	r.Header.Del("Cache-Control")
+	ps.h.ServeHTTP(s, r)
+	w.Header().Del("Last-Modified")
+}