continued work on encryption

cmd/cachectl/main.go

@@ -0,0 +1,196 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package main
+
+import (
+	"cachefs/pkg/provider/parse"
+	"encoding/hex"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	version string
+)
+
+const gib = 1024 * 1024 * 1024
+
+const (
+	srcKey = "SRC_KEY"
+	dstKey = "DST_KEY"
+)
+
+func fatal(err error) {
+	fmt.Println(err)
+	os.Exit(1)
+}
+
+func getKey(key string) (b []byte, err error) {
+	k := os.Getenv(key)
+	if k == "" {
+		return
+	}
+	b, err = hex.DecodeString(k)
+	if err != nil {
+		err = fmt.Errorf("error decoding %s: %w", key, err)
+	}
+	return
+}
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
+		fmt.Fprintf(flag.CommandLine.Output(), `
+optional environment variables:
+
+  `+srcKey+` (source encryption key)
+  `+dstKey+` (destination encryption key)
+
+list files and directories (uses `+srcKey+`):
+  ls <base> [ <path> ]
+
+  ls file:///mnt/nfs /foo
+
+copy file from/to directory:
+  cp <src base> <src> <dst base> <dst>
+
+  cp file:///mnt/nfs foo.txt file:///mnt/cache bar.txt
+
+  cp file:///mnt/cache fizz.txt sftp:///home/user fizz.txt
+
+remove file (uses `+srcKey+`):
+  rm <base> <file>
+
+  rm file:///mnt/nfs /buzz.txt
+
+`)
+		os.Exit(1)
+	}
+	flag.Parse()
+
+	args := flag.Args()
+	if len(args) < 2 {
+		flag.Usage()
+	}
+
+	var err error
+	switch args[0] {
+	case "ls":
+		err = ls(args[1:])
+	case "cp":
+		if len(args) < 5 {
+			flag.Usage()
+		}
+		err = cp(args[1:])
+	case "rm":
+		if len(args) < 3 {
+			flag.Usage()
+		}
+		err = rm(args[1:])
+	}
+	if err != nil {
+		fatal(err)
+	}
+}
+
+func ls(args []string) (err error) {
+	k, err := getKey(srcKey)
+	if err != nil {
+		return
+	}
+	fs, err := parse.FS(args[0], k, true)
+	if err != nil {
+		return
+	}
+	defer fs.Close()
+	path := ""
+	if len(args) == 2 {
+		path = args[1]
+	}
+	f, err := fs.Open(path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	fis, err := f.Readdir(0)
+	if err != nil {
+		return
+	}
+	for _, fi := range fis {
+		p := filepath.Join(fs.Root(), path, fi.Name())
+		if fi.IsDir() {
+			fmt.Println(fi.Mode().String(), fi.Size(), p+"/")
+			continue
+		}
+		fmt.Println(fi.Mode().String(), fi.Size(), p)
+	}
+	return
+}
+
+func cp(args []string) (err error) {
+	src := args[0]
+	sp := args[1]
+	dst := args[2]
+	dp := args[3]
+	sk, err := getKey(srcKey)
+	if err != nil {
+		return
+	}
+	dk, err := getKey(dstKey)
+	if err != nil {
+		return
+	}
+	sfs, err := parse.FS(src, sk, true)
+	if err != nil {
+		return
+	}
+	defer sfs.Close()
+	dfs, err := parse.FS(dst, dk, true)
+	if err != nil {
+		return
+	}
+	defer dfs.Close()
+
+	sf, err := sfs.Open(sp)
+	if err != nil {
+		return
+	}
+	defer sf.Close()
+	if i := strings.LastIndex(dp, "/"); i > 0 {
+		dir := dp[:i]
+		_, err = dfs.Stat(dir)
+		if errors.Is(err, os.ErrNotExist) {
+			err = dfs.MkdirAll(dir, 0o755)
+			if err != nil {
+				return
+			}
+		}
+	}
+	df, err := dfs.OpenFile(dp, os.O_RDWR|os.O_CREATE, 0o644)
+	if err != nil {
+		return
+	}
+	defer df.Close()
+	_, err = io.Copy(df, sf)
+	return
+}
+
+func rm(args []string) (err error) {
+	src := args[0]
+	path := args[1]
+	k, err := getKey(srcKey)
+	if err != nil {
+		return
+	}
+	fs, err := parse.FS(src, k, true)
+	if err != nil {
+		return
+	}
+	defer fs.Close()
+	return fs.Remove(path)
+}

cmd/cachefs/main.go

@@ -24,8 +24,6 @@ var (
 
 	src          string
 	dst          string
-	srckey       string
-	dstkey       string
 	metadata     string
 	listenHttp   string
 	listenWebdav string
@@ -33,6 +31,7 @@ var (
 	quota        int64
 	max          int
 	bs           int
+	block        bool
 
 	log logr.Logger
 )
@@ -42,16 +41,15 @@ const gib = 1024 * 1024 * 1024
 func main() {
 	klog.InitFlags(nil)
 	flag.StringVar(&src, "src", "", "url path to source files (example: file:///mnt/nfs)")
-	flag.StringVar(&srckey, "srckey", "", "hex encoded encryption key for src")
 	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
-	flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
 	flag.StringVar(&metadata, "data", "", "path to metadata file")
 	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
 	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
 	flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
 	flag.IntVar(&max, "max", -1, "max parallel preloads")
-	flag.IntVar(&bs, "bs", -1, "tune reload buffer size in bytes (default: 8192)")
-	flag.Int64Var(&quota, "quota", 1, "max disk usage quota in GiB")
+	flag.IntVar(&bs, "bs", -1, "tune preload buffer size in bytes (default: 8192)")
+	flag.Int64Var(&quota, "quota", 1, "max disk usage quota for the dst cache in GiB")
+	flag.BoolVar(&block, "block", false, "block until sftp is connected")
 	flag.Parse()
 
 	log = klogr.New().WithName("main")
@@ -65,10 +63,11 @@ func main() {
 		quota*gib,
 		max,
 		src,
-		srckey,
+		os.Getenv("SRC_KEY"),
 		dst,
-		dstkey,
+		os.Getenv("DST_KEY"),
 		metadata,
+		block,
 		klogr.New().WithName("fs"),
 	)
 	if err != nil {

cmd/metadatagen/main.go

@@ -16,13 +16,11 @@ var (
 	version string
 
 	dst      string
-	dstkey   string
 	metadata string
 )
 
 func main() {
 	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
-	flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
 	flag.StringVar(&metadata, "data", "", "path to metadata file")
 	flag.Parse()
 
@@ -30,12 +28,13 @@ func main() {
 		dstk []byte
 		err  error
 	)
+	dstkey := os.Getenv("DST_KEY")
 	if dstkey != "" {
 		dstk, err = hex.DecodeString(dstkey)
 		fmt.Println(err)
 		os.Exit(1)
 	}
-	dstfs, err := parse.FS(dst, dstk)
+	dstfs, err := parse.FS(dst, dstk, true)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)