switch to config file and implement src path filter

cmd/cachectl/main.go

@@ -111,7 +111,7 @@ func ls(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	fs, err := parse.FS(args[0], k, true)
+	fs, err := parse.FS(args[0], k, nil, true)
 	if err != nil {
 		return
 	}
@@ -153,12 +153,12 @@ func cp(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	sfs, err := parse.FS(src, sk, true)
+	sfs, err := parse.FS(src, sk, nil, true)
 	if err != nil {
 		return
 	}
 	defer sfs.Close()
-	dfs, err := parse.FS(dst, dk, true)
+	dfs, err := parse.FS(dst, dk, nil, true)
 	if err != nil {
 		return
 	}
@@ -248,7 +248,7 @@ func rm(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	fs, err := parse.FS(src, k, true)
+	fs, err := parse.FS(src, k, nil, true)
 	if err != nil {
 		return
 	}

cmd/cachefs/main.go

@@ -7,12 +7,12 @@ import (
 	"flag"
 	"net/http"
 
-	//_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"cachefs/pkg/config"
 	"cachefs/pkg/fs"
 	"cachefs/pkg/srv"
 
@@ -23,17 +23,7 @@ import (
 
 var (
 	version string
-
-	src          string
-	dst          string
-	metadata     string
-	listenHttp   string
-	listenWebdav string
-	listenCache  string
-	quota        int64
-	max          int
-	bs           int
-	block        bool
+	cfg     string
 
 	log logr.Logger
 )
@@ -42,52 +32,35 @@ const gib = 1024 * 1024 * 1024
 
 func main() {
 	klog.InitFlags(nil)
-	flag.StringVar(&src, "src", "", "url path to source files (example: file:///mnt/nfs)")
-	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
-	flag.StringVar(&metadata, "data", "", "path to metadata file")
-	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
-	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
-	flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
-	flag.IntVar(&max, "max", 1, "max parallel preloads")
-	flag.IntVar(&bs, "bs", -1, "tune preload buffer size in bytes (default: 8192)")
-	flag.Int64Var(&quota, "quota", 1, "max disk usage quota for the dst cache in GiB")
-	flag.BoolVar(&block, "block", true, "block until sftp is connected")
+	flag.StringVar(&cfg, "config", "", "path to config file")
 	flag.Parse()
-	//go func() {
-	//	http.ListenAndServe("127.0.0.1:7777", nil)
-	//}()
 
 	log = klogr.New().WithName("main")
 	log.Info("starting cachefs", "version", version)
 
-	if fs.SetBufferSize(bs) {
-		log.V(2).Info("changed preload buffer size", "size", bs)
+	c, err := config.ParseFile(cfg)
+	if err != nil {
+		klog.Fatalf("init failed: %s", err)
 	}
 
-	filesystem, err := fs.NewFS(
-		quota*gib,
-		max,
-		src,
-		os.Getenv("SRC_KEY"),
-		dst,
-		os.Getenv("DST_KEY"),
-		metadata,
-		block,
-		klogr.New().WithName("fs"),
-	)
+	if fs.SetBufferSize(c.Cache.Buffer) {
+		log.V(2).Info("changed preload buffer size", "size", c.Cache.Buffer)
+	}
+
+	filesystem, err := fs.NewFSFromConfig(c, klogr.New().WithName("fs"))
 	if err != nil {
 		klog.Fatalf("init failed: %s", err)
 	}
 	slog := klogr.New().WithName("srv")
 	s := &http.Server{
-		Addr: listenHttp,
+		Addr: c.Server.HTTP.Addr,
 		Handler: srv.NewFileServer(
 			filesystem,
 			slog,
 		),
 	}
 	go func() {
-		log.Info("starting main server", "addr", listenHttp)
+		log.Info("starting main server", "addr", c.Server.HTTP.Addr)
 		err := s.ListenAndServe()
 		if err != nil && err != http.ErrServerClosed {
 			klog.Fatalf("init failed: %s", err)
@@ -96,11 +69,13 @@ func main() {
 	var (
 		dav   *http.Server
 		cache *http.Server
+		plain *http.Server
 	)
-	if listenWebdav != "" {
-		log.Info("starting webdav server", "addr", listenWebdav)
+	davAddr := c.Server.WebDav.Addr
+	if davAddr != "" {
+		log.Info("starting webdav server", "addr", davAddr)
 		dav = &http.Server{
-			Addr: listenWebdav,
+			Addr: davAddr,
 			Handler: srv.NewWebDavServer(
 				fs.NewWebDavFS(filesystem),
 				slog.WithName("webdav"),
@@ -113,10 +88,11 @@ func main() {
 			}
 		}()
 	}
-	if listenCache != "" {
-		log.Info("starting cache server", "addr", listenCache)
+	cacheAddr := c.Server.Cache.Addr
+	if cacheAddr != "" {
+		log.Info("starting cache server", "addr", cacheAddr)
 		cache = &http.Server{
-			Addr: listenCache,
+			Addr: cacheAddr,
 			Handler: srv.NewCacheServer(
 				filesystem,
 				slog.WithName("cache"),
@@ -129,6 +105,23 @@ func main() {
 			}
 		}()
 	}
+	plainAddr := c.Server.Plain.Addr
+	if plainAddr != "" {
+		log.Info("starting plain server", "addr", plainAddr)
+		plain = &http.Server{
+			Addr: plainAddr,
+			Handler: srv.NewPlainServer(
+				filesystem,
+				slog.WithName("plain"),
+			),
+		}
+		go func() {
+			err := plain.ListenAndServe()
+			if err != nil && err != http.ErrServerClosed {
+				klog.Fatalf("init failed: %s", err)
+			}
+		}()
+	}
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	<-sigs
@@ -146,6 +139,11 @@ func main() {
 		cache.Shutdown(ctx)
 		cancel()
 	}
+	if plain != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		plain.Shutdown(ctx)
+		cancel()
+	}
 	filesystem.Close()
 	<-filesystem.Done()
 	log.Info("cachefs shutdown completed")

cmd/metadatagen/main.go

@@ -44,7 +44,7 @@ func main() {
 		}
 		encrypted = true
 	}
-	dstfs, err := parse.FS(dst, dstk, true)
+	dstfs, err := parse.FS(dst, dstk, nil, true)
 	if err != nil {
 		fatal(err)
 	}