added fs encryption overlay

.gitignore

@@ -1,2 +1,3 @@
 /cachefs
 /metadatagen
+/run.sh

TODO.txt

@@ -1,5 +1,9 @@
 * document openbsd pcap wireshark fix
   * printf '\x6c' | dd seek=20 bs=1 count=1 conv=notrunc of=test.dump
+
+* crypto
+  * flush last chunk in streaming mode
+
 * add breadcrumb
 * add copyright footer
 * add stopped status?

cmd/cachefs/main.go

@@ -24,6 +24,8 @@ var (
 
 	src          string
 	dst          string
+	srckey       string
+	dstkey       string
 	metadata     string
 	listenHttp   string
 	listenWebdav string
@@ -40,7 +42,9 @@ const gib = 1024 * 1024 * 1024
 func main() {
 	klog.InitFlags(nil)
 	flag.StringVar(&src, "src", "", "url path to source files (example: file:///mnt/nfs)")
+	flag.StringVar(&srckey, "srckey", "", "hex encoded encryption key for src")
 	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
+	flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
 	flag.StringVar(&metadata, "data", "", "path to metadata file")
 	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
 	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
@@ -57,7 +61,16 @@ func main() {
 		log.V(2).Info("changed preload buffer size", "size", bs)
 	}
 
-	filesystem, err := fs.NewFS(quota*gib, max, src, dst, metadata, klogr.New().WithName("fs"))
+	filesystem, err := fs.NewFS(
+		quota*gib,
+		max,
+		src,
+		srckey,
+		dst,
+		dstkey,
+		metadata,
+		klogr.New().WithName("fs"),
+	)
 	if err != nil {
 		klog.Fatalf("init failed: %s", err)
 	}

cmd/metadatagen/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"encoding/hex"
 	"flag"
 	"fmt"
 	"os"
@@ -15,15 +16,26 @@ var (
 	version string
 
 	dst      string
+	dstkey   string
 	metadata string
 )
 
 func main() {
 	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
+	flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
 	flag.StringVar(&metadata, "data", "", "path to metadata file")
 	flag.Parse()
 
-	dstfs, err := parse.FS(dst)
+	var (
+		dstk []byte
+		err  error
+	)
+	if dstkey != "" {
+		dstk, err = hex.DecodeString(dstkey)
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	dstfs, err := parse.FS(dst, dstk)
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)

pkg/chunk/chunk.go

@@ -1,6 +1,6 @@
 // Copyright (C) 2022 Marius Schellenberger
 
-package fs
+package chunk
 
 import (
 	"golang.org/x/exp/slices"

pkg/fs/file.go

@@ -49,6 +49,7 @@ func (p *preload) Read(data []byte) (n int, err error) {
 
 func (f *File) Preload(ctx context.Context, unlock func()) {
 	log := f.log
+	defer f.Close()
 	defer unlock()
 	if f.offline {
 		log.V(2).Error(errors.New("no preload in offline mode"), "error preloading file")
@@ -68,6 +69,7 @@ func (f *File) Preload(ctx context.Context, unlock func()) {
 	if err != nil && err != io.EOF {
 		log.Error(err, "error preloading file")
 	}
+	f.md.Close()
 	log.V(2).Info("preload finished", "skipped", p.skipped, "written", p.written)
 }
 

pkg/fs/fs.go

@@ -6,6 +6,7 @@ import (
 	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/parse"
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	stdfs "io/fs"
@@ -31,15 +32,28 @@ type FS struct {
 	ph *PreloadHandler
 }
 
-func NewFS(quota int64, max int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
+func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, log logr.Logger) (fs *FS, err error) {
 	if src == dst {
 		return nil, errors.New("src and dst path can not be equal")
 	}
-	srcfs, err := parse.FS(src)
+	var srck, dstk []byte
+	if srckey != "" {
+		srck, err = hex.DecodeString(srckey)
+		if err != nil {
+			return nil, fmt.Errorf("srckey: %w", err)
+		}
+	}
+	if dstkey != "" {
+		dstk, err = hex.DecodeString(dstkey)
+		if err != nil {
+			return nil, fmt.Errorf("dstkey: %w", err)
+		}
+	}
+	srcfs, err := parse.FS(src, srck)
 	if err != nil {
 		return nil, fmt.Errorf("srcfs: %w", err)
 	}
-	dstfs, err := parse.FS(dst)
+	dstfs, err := parse.FS(dst, dstk)
 	if err != nil {
 		return nil, fmt.Errorf("dstfs: %w", err)
 	}

pkg/fs/metadata.go

@@ -3,6 +3,7 @@
 package fs
 
 import (
+	"cachefs/pkg/chunk"
 	"cachefs/pkg/provider"
 	"context"
 	"encoding/json"
@@ -80,7 +81,7 @@ func MetadataGenerator(file string, dst provider.FS) error {
 		if size <= blocks*512 {
 			md[strings.TrimPrefix(path, dst.Root())] = &Metadata{
 				Size:   size,
-				Chunks: Chunks{{0, size}},
+				Chunks: chunk.Chunks{{0, size}},
 			}
 		}
 		return nil
@@ -328,13 +329,21 @@ type Metadata struct {
 	name   string        `json:"-"`
 	Size   int64         `json:"s"`
 	Atime  int64         `json:"a"`
-	Chunks Chunks        `json:"c"`
+	Chunks chunk.Chunks  `json:"c"`
+}
+
+func (md *Metadata) Close() error {
+	md.mu.Lock()
+	defer md.mu.Unlock()
+	err := md.f.Close()
+	md.f = nil
+	return err
 }
 
 func (md *Metadata) Delete() error {
 	md.mu.Lock()
 	defer md.mu.Unlock()
-	md.Chunks = Chunks{}
+	md.Chunks = chunk.Chunks{}
 	md.f.Close()
 	md.f = nil
 	atomic.StoreInt64(&md.Atime, now())

pkg/fs/preload.go

@@ -130,8 +130,6 @@ func (ph *PreloadHandler) preload(ctx context.Context) {
 		p.cancel = cancel
 		ph.mu.Unlock()
 		go f.Preload(ctx, func() {
-			file := f
-			file.Close()
 			ph.fin <- name
 		})
 	}

pkg/provider/crypto/crypto.go

@@ -0,0 +1,92 @@
+package crypto
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/sha256"
+	"io"
+
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/hkdf"
+)
+
+const (
+	KDFNonceSize = 16
+	NonceSize    = chacha20poly1305.NonceSize
+	Overhead     = chacha20poly1305.Overhead
+	ChunkSize    = 64 * 1024
+	FullSize     = ChunkSize + Overhead
+)
+
+const (
+	infoCBC     = "AES-CBC filename encryption key"
+	infoCBCIV   = "AES-CBC filename encryption initialization vector"
+	infoChaPloy = "ChaCha20-Poly1305 file content encryption"
+)
+
+func cbcKey(key []byte) []byte {
+	return deriveKey(key, nil, infoCBC, 32)
+}
+
+func cbcIV(key []byte) []byte {
+	return deriveKey(key, nil, infoCBCIV, 16)
+}
+
+func streamKey(key, nonce []byte) []byte {
+	return deriveKey(key, nonce, infoChaPloy, chacha20poly1305.KeySize)
+}
+
+func deriveKey(key, nonce []byte, info string, length int) []byte {
+	h := hkdf.New(sha256.New, key, nonce, []byte(info))
+	k := make([]byte, length)
+	if _, err := io.ReadFull(h, k); err != nil {
+		panic("crypto: internal error: failed to read from HKDF: " + err.Error())
+	}
+	return k
+}
+
+type aesCbc struct {
+	key []byte
+	iv  []byte
+}
+
+func newAesCbc(key []byte) (*aesCbc, error) {
+	cbc := &aesCbc{
+		key: cbcKey(key),
+		iv:  cbcIV(key),
+	}
+	_, err := aes.NewCipher(cbc.key)
+	if err != nil {
+		return nil, err
+	}
+	return cbc, nil
+}
+
+func (cbc *aesCbc) encrypt(p []byte) (c []byte) {
+	block, _ := aes.NewCipher(cbc.key)
+	mode := cipher.NewCBCEncrypter(block, cbc.iv)
+	padded := padPKCS5(p, block.BlockSize())
+	c = make([]byte, len(padded))
+	mode.CryptBlocks(c, padded)
+	return
+}
+
+func (cbc *aesCbc) decrypt(c []byte) []byte {
+	block, _ := aes.NewCipher(cbc.key)
+	mode := cipher.NewCBCDecrypter(block, cbc.iv)
+	p := make([]byte, len(c))
+	mode.CryptBlocks(p, c)
+	return trimPKCS5(p)
+}
+
+func padPKCS5(b []byte, bs int) []byte {
+	n := bs - len(b)%bs
+	pad := bytes.Repeat([]byte{byte(n)}, n)
+	return append(b, pad...)
+}
+
+func trimPKCS5(b []byte) []byte {
+	pad := b[len(b)-1]
+	return b[:len(b)-int(pad)]
+}

pkg/provider/crypto/file.go

@@ -0,0 +1,186 @@
+package crypto
+
+import (
+	"cachefs/pkg/provider"
+	"io"
+	"io/fs"
+)
+
+var _ provider.File = (*file)(nil)
+
+type file struct {
+	provider.File
+	fs     *FS
+	r      *reader
+	w      *writer
+	nonce  [NonceSize]byte
+	offset int64
+	name   string
+}
+
+func newFile(key []byte, name string, fs *FS, f provider.File) (ef provider.File, err error) {
+	nonce, err := initNonce(f)
+	if err != nil {
+		return
+	}
+	skey := streamKey(key, nonce)
+	w, err := newWriter(skey, f)
+	if err != nil {
+		return
+	}
+	r, err := newReader(skey, f)
+	if err != nil {
+		return
+	}
+	return &file{
+		File: f,
+		fs:   fs,
+		w:    w,
+		r:    r,
+		name: name,
+	}, nil
+}
+
+func (f *file) Stat() (fs.FileInfo, error) {
+	fi, err := f.File.Stat()
+	return stat{FileInfo: fi, name: f.name}, err
+}
+
+func (f *file) Name() string {
+	return f.name
+}
+
+func (file) Fd() uintptr {
+	return 0
+}
+
+type stat struct {
+	fs.FileInfo
+	name string
+}
+
+func (s stat) Size() int64 {
+	return RealSize(s.FileInfo.Size())
+}
+
+func (s stat) Name() string {
+	return s.name
+}
+
+func (f *file) Close() error {
+	e1 := f.w.Close()
+	e2 := f.File.Close()
+	if e1 != nil {
+		return e1
+	}
+	return e2
+}
+
+func (f *file) Seek(offset int64, whence int) (n int64, err error) {
+	var cn, off, roff int64
+	switch whence {
+	case io.SeekStart:
+		cn, off, roff = align(offset)
+		f.offset = KDFNonceSize + off
+		n, err = f.File.Seek(f.offset, whence)
+	case io.SeekCurrent:
+		if f.offset == 0 {
+			f.offset = KDFNonceSize
+		}
+		cn, off, roff = align(f.offset + f.r.off + offset)
+		f.offset = off
+		_, err = f.File.Seek(f.offset, io.SeekStart)
+		n = offset
+	}
+	f.nonce = [NonceSize]byte{}
+	inc(&f.nonce, cn)
+	f.r.nonce = f.nonce
+	f.r.off = roff
+	f.r.err = nil
+	f.w.nonce = f.nonce
+	f.r.unread = f.r.unread[:0]
+	return
+}
+
+func (f *file) Read(p []byte) (n int, err error) {
+	n, err = f.r.Read(p)
+	return
+}
+
+func (f *file) Write(p []byte) (n int, err error) {
+	n, err = f.w.Write(p)
+	return
+}
+
+func (f *file) ReadAt(p []byte, pos int64) (n int, err error) {
+	_, _, roff := align(pos)
+	//cn, off, roff := align(pos)
+	f.Seek(pos, io.SeekStart)
+	//fmt.Println("readAt:", cn, off+KDFNonceSize, roff, pos)
+	_, err = f.r.readChunk()
+	if err != nil {
+		return
+	}
+	n = copy(p, f.r.unread[roff:])
+	return
+}
+
+func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
+	cn, off, woff := align(pos)
+	//fmt.Println("writeAt:", cn, off+KDFNonceSize, woff, pos)
+	c, ok := f.w.cm[cn]
+	if !ok {
+		c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
+		inc(&c.nonce, cn)
+		f.w.cm[cn] = c
+		if f.w.last == nil {
+			f.w.last = c
+		} else if cn > f.w.last.cn {
+			f.w.last = c
+		}
+	}
+	n, err = c.writeAt(data, woff)
+	if c.full() {
+		_, err = c.flush(notLastChunk)
+		//fmt.Println("flush:", cn)
+		delete(f.w.cm, cn)
+	}
+	return
+}
+
+func (f *file) Readdir(n int) ([]fs.FileInfo, error) {
+	fis, err := f.File.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.FileInfo, len(fis))
+	for i, fi := range fis {
+		p, err := f.fs.decPath(fi.Name())
+		if err != nil {
+			continue
+		}
+		fio[i] = dir{FileInfo: fi, name: p}
+	}
+	return fio, nil
+}
+
+func (f *file) ReadDir(n int) ([]fs.DirEntry, error) {
+	fis, err := f.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.DirEntry, len(fis))
+	for i, fi := range fis {
+		fio[i] = fs.FileInfoToDirEntry(fi)
+	}
+	return fio, nil
+}
+
+type dir struct {
+	fs.FileInfo
+	name string
+}
+
+func (d dir) Name() string {
+	return d.name
+}

pkg/provider/crypto/fs.go

@@ -0,0 +1,129 @@
+package crypto
+
+import (
+	"cachefs/pkg/provider"
+	"encoding/base64"
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+var _ provider.FS = (*FS)(nil)
+
+type FS struct {
+	fs  provider.FS
+	cbc *aesCbc
+	key []byte
+}
+
+func NewFS(fs provider.FS, key []byte) (*FS, error) {
+	if len(key) != 32 {
+		return nil, errors.New("encryption key length must be 32 bytes")
+	}
+	cbc, err := newAesCbc(key)
+	if err != nil {
+		return nil, err
+	}
+	return &FS{
+		fs:  fs,
+		cbc: cbc,
+		key: key,
+	}, nil
+}
+
+func (fs *FS) encPath(path string) (n string, err error) {
+	parts := strings.Split(path, "/")
+	cparts := make([]string, len(parts))
+	for i, p := range parts {
+		if p == "" {
+			continue
+		}
+		cparts[i] = base64.RawURLEncoding.EncodeToString(
+			fs.cbc.encrypt([]byte(p)),
+		)
+	}
+	n = filepath.Join(cparts...)
+	return
+}
+
+func (fs *FS) decPath(path string) (n string, err error) {
+	parts := strings.Split(path, "/")
+	pparts := make([]string, len(parts))
+	for i, p := range parts {
+		pparts[i] = base64.RawURLEncoding.EncodeToString(
+			fs.cbc.decrypt([]byte(p)),
+		)
+	}
+	n = filepath.Join(pparts...)
+	return
+}
+
+func (fs *FS) name(p string) string {
+	return filepath.Join(fs.Root(), p)
+}
+
+func (fs *FS) Stat(p string) (fs.FileInfo, error) {
+	c, err := fs.encPath(p)
+	if err != nil {
+		return nil, err
+	}
+	fi, err := fs.fs.Stat(c)
+	return stat{FileInfo: fi, name: fs.name(p)}, err
+}
+
+func (fs *FS) Remove(p string) error {
+	c, err := fs.encPath(p)
+	if err != nil {
+		return err
+	}
+	return fs.fs.Remove(c)
+}
+
+func (fs *FS) Open(p string) (provider.File, error) {
+	c, err := fs.encPath(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.fs.Open(c)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(fs.key, p, fs, f)
+}
+
+func (fs *FS) OpenFile(p string, flags int, mode os.FileMode) (provider.File, error) {
+	c, err := fs.encPath(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.fs.OpenFile(c, flags, mode)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(fs.key, p, fs, f)
+}
+
+func (fs *FS) MkdirAll(p string, mode os.FileMode) error {
+	c, err := fs.encPath(p)
+	if err != nil {
+		return err
+	}
+	return fs.fs.MkdirAll(c, mode)
+}
+
+func (fs *FS) Root() string {
+	return fs.fs.Root()
+}
+
+func (fs *FS) Fstat(_ uintptr) (int64, error) {
+	return 0, provider.ErrFstat
+}
+
+func (fs *FS) Close() error {
+	fs.key = nil
+	runtime.GC()
+	return nil
+}

pkg/provider/crypto/helper.go

@@ -0,0 +1,78 @@
+package crypto
+
+import (
+	"cachefs/pkg/provider"
+	"crypto/rand"
+	"io"
+	"math"
+)
+
+func isZero(nonce []byte) bool {
+	for i := 0; i < len(nonce); i++ {
+		if nonce[i] != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func initNonce(f provider.File) (nonce []byte, err error) {
+	nonce = make([]byte, KDFNonceSize)
+	n, err := io.ReadFull(f, nonce)
+	if err != nil || n != len(nonce) || isZero(nonce) {
+		_, err = io.ReadFull(rand.Reader, nonce)
+		if err != nil {
+			return
+		}
+		_, err = f.Seek(0, io.SeekStart)
+		if err != nil {
+			return
+		}
+		_, err = f.Write(nonce)
+		return
+	}
+	return
+}
+
+func incNonce(nonce *[NonceSize]byte) {
+	for i := len(nonce) - 2; i >= 0; i-- {
+		nonce[i]++
+		if nonce[i] != 0 {
+			break
+		} else if i == 0 {
+			// The counter is 88 bits, this is unreachable.
+			panic("stream: chunk counter wrapped around")
+		}
+	}
+}
+
+func Size(s int64) int64 {
+	return KDFNonceSize + s + (int64(math.Ceil(float64(s)/ChunkSize)) * Overhead)
+}
+
+func RealSize(s int64) int64 {
+	s -= KDFNonceSize
+	return s - (((s / FullSize) + 1) * Overhead)
+}
+
+func chunkNum(o int64) int64 {
+	return int64(math.Ceil(float64(o) / ChunkSize))
+}
+
+func align(o int64) (cn, off, roff int64) {
+	cn = chunkNum(o)
+	off = cn * FullSize
+	roff = o - (cn * ChunkSize)
+	if roff < 0 {
+		cn--
+		off = cn * FullSize
+		roff = o - (cn * ChunkSize)
+	}
+	return
+}
+
+func inc(nonce *[NonceSize]byte, n int64) {
+	for i := int64(0); i < n; i++ {
+		incNonce(nonce)
+	}
+}

pkg/provider/crypto/reader.go

@@ -0,0 +1,107 @@
+package crypto
+
+import (
+	"crypto/cipher"
+	"errors"
+	"io"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+type cryptoReader interface {
+	io.Reader
+	io.ReaderAt
+}
+
+type reader struct {
+	a   cipher.AEAD
+	src cryptoReader
+
+	unread []byte // decrypted but unread data, backed by buf
+	buf    [FullSize]byte
+	off    int64
+
+	err   error
+	nonce [NonceSize]byte
+}
+
+func newReader(key []byte, src cryptoReader) (*reader, error) {
+	aead, err := chacha20poly1305.New(key)
+	if err != nil {
+		return nil, err
+	}
+	return &reader{
+		a:   aead,
+		src: src,
+	}, nil
+}
+
+func (r *reader) Read(p []byte) (int, error) {
+	if len(r.unread) > 0 {
+		n := copy(p, r.unread)
+		r.unread = r.unread[n:]
+		return n, nil
+	}
+	if r.err != nil {
+		return 0, r.err
+	}
+	if len(p) == 0 {
+		return 0, nil
+	}
+
+	last, err := r.readChunk()
+	if err != nil {
+		r.err = err
+		return 0, err
+	}
+	if r.off > 0 && len(r.unread) >= int(r.off) {
+		r.unread = r.unread[r.off:]
+		r.off = 0
+	}
+
+	n := copy(p, r.unread)
+	r.unread = r.unread[n:]
+
+	if last {
+		r.err = io.EOF
+	}
+
+	return n, nil
+}
+
+func (r *reader) readChunk() (last bool, err error) {
+	if len(r.unread) != 0 {
+		panic("stream: internal error: readChunk called with dirty buffer")
+	}
+
+	in := r.buf[:]
+	n, err := io.ReadFull(r.src, in)
+	switch {
+	case err == io.EOF:
+		// A message can't end without a marked chunk. This message is truncated.
+		return false, io.ErrUnexpectedEOF
+	case err == io.ErrUnexpectedEOF:
+		// The last chunk can be short.
+		in = in[:n]
+		last = true
+		setLastChunkFlag(&r.nonce)
+	case err != nil:
+		return false, err
+	}
+
+	outBuf := make([]byte, 0, ChunkSize)
+	out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
+	if err != nil && !last {
+		// Check if this was a full-length final chunk.
+		last = true
+		setLastChunkFlag(&r.nonce)
+		out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
+	}
+	if err != nil {
+		return false, errors.New("failed to decrypt and authenticate payload chunk")
+	}
+
+	incNonce(&r.nonce)
+	r.unread = r.buf[:copy(r.buf[:], out)]
+	return last, nil
+}

pkg/provider/crypto/writer.go

@@ -0,0 +1,155 @@
+package crypto
+
+import (
+	"cachefs/pkg/chunk"
+	"crypto/cipher"
+	"errors"
+	"io"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+type cryptoWriter interface {
+	io.Writer
+	io.WriterAt
+}
+
+const lastChunkFlag = 0x01
+
+func setLastChunkFlag(nonce *[NonceSize]byte) {
+	nonce[len(nonce)-1] = lastChunkFlag
+}
+
+type writer struct {
+	a         cipher.AEAD
+	dst       cryptoWriter
+	unwritten []byte // backed by buf
+	buf       [FullSize]byte
+	nonce     [NonceSize]byte
+	err       error
+	cm        map[int64]*chunkWriter
+	last      *chunkWriter
+	flush     bool
+}
+
+func newWriter(key []byte, dst cryptoWriter) (*writer, error) {
+	aead, err := chacha20poly1305.New(key)
+	if err != nil {
+		return nil, err
+	}
+	w := &writer{
+		a:   aead,
+		dst: dst,
+		cm:  make(map[int64]*chunkWriter),
+	}
+	w.unwritten = w.buf[:0]
+	return w, nil
+}
+
+type chunkWriter struct {
+	offset int64
+	cn     int64
+	w      *writer
+	buf    [ChunkSize]byte
+	nonce  [NonceSize]byte
+	chunks chunk.Chunks
+}
+
+func (c *chunkWriter) writeAt(p []byte, offset int64) (n int, err error) {
+	n = copy(c.buf[offset:], p)
+	c.chunks.Add(offset, n)
+	return
+}
+
+func (c *chunkWriter) full() bool {
+	if len(c.chunks) == 1 {
+		return c.chunks[0][1] == ChunkSize
+	}
+	return false
+}
+
+func (c *chunkWriter) flush(last bool) (n int, err error) {
+	wb := c.buf[:0]
+	if last {
+		setLastChunkFlag(&c.nonce)
+		s := c.chunks.Size()
+		wb = c.buf[:s]
+		n = int(s)
+	} else {
+		wb = c.buf[:]
+		n = len(c.buf)
+	}
+	buf := c.w.a.Seal(wb[:0], c.nonce[:], wb, nil)
+	_, err = c.w.dst.WriteAt(buf, c.offset)
+	return
+}
+
+func (w *writer) Write(p []byte) (n int, err error) {
+	// TODO: consider refactoring with a bytes.Buffer.
+	if w.err != nil {
+		return 0, w.err
+	}
+	if len(p) == 0 {
+		return 0, nil
+	}
+
+	total := len(p)
+	for len(p) > 0 {
+		freeBuf := w.buf[len(w.unwritten):ChunkSize]
+		n := copy(freeBuf, p)
+		p = p[n:]
+		w.unwritten = w.unwritten[:len(w.unwritten)+n]
+		w.flush = true
+
+		if len(w.unwritten) == ChunkSize && len(p) > 0 {
+			if err := w.flushChunk(notLastChunk); err != nil {
+				w.err = err
+				return 0, err
+			}
+		}
+	}
+	return total, nil
+}
+
+func (w *writer) Close() error {
+	if w.err != nil {
+		return w.err
+	}
+
+	if w.last != nil {
+		_, w.err = w.last.flush(lastChunk)
+		if w.err != nil {
+			return w.err
+		}
+	}
+
+	if w.flush {
+		w.err = w.flushChunk(lastChunk)
+		if w.err != nil {
+			return w.err
+		}
+	}
+
+	w.err = errors.New("stream.Writer is already closed")
+	return nil
+}
+
+const (
+	lastChunk    = true
+	notLastChunk = false
+)
+
+func (w *writer) flushChunk(last bool) error {
+	if !last && len(w.unwritten) != ChunkSize {
+		panic("stream: internal error: flush called with partial chunk")
+	}
+
+	if last {
+		setLastChunkFlag(&w.nonce)
+	}
+	buf := w.a.Seal(w.buf[:0], w.nonce[:], w.unwritten, nil)
+	_, err := w.dst.Write(buf)
+	w.unwritten = w.buf[:0]
+	incNonce(&w.nonce)
+	return err
+}

pkg/provider/interface.go

@@ -4,6 +4,7 @@ package provider
 
 import (
 	"errors"
+	"io"
 	"io/fs"
 	"os"
 )
@@ -34,16 +35,17 @@ func WalkFS(fs FS) fs.FS {
 }
 
 type File interface {
+	io.Reader
+	io.ReaderAt
+	io.Writer
+	io.WriterAt
+	io.Seeker
+	io.Closer
 	Name() string
 	Stat() (fs.FileInfo, error)
 	Truncate(int64) error
 	Sync() error
 	Fd() uintptr
 	ReadDir(int) ([]fs.DirEntry, error)
-	Read([]byte) (int, error)
-	ReadAt([]byte, int64) (int, error)
-	WriteAt([]byte, int64) (int, error)
-	Seek(int64, int) (int64, error)
 	Readdir(int) ([]fs.FileInfo, error)
-	Close() error
 }

pkg/provider/os/os.go

@@ -1,6 +1,6 @@
 // Copyright (C) 2022 Marius Schellenberger
 
-package mount
+package os
 
 import (
 	"cachefs/pkg/provider"

pkg/provider/parse/parse.go

@@ -4,7 +4,8 @@ package parse
 
 import (
 	"cachefs/pkg/provider"
-	"cachefs/pkg/provider/mount"
+	"cachefs/pkg/provider/crypto"
+	"cachefs/pkg/provider/os"
 	"cachefs/pkg/provider/sftp"
 	"errors"
 	"fmt"
@@ -19,7 +20,7 @@ var (
 	ErrUnsupportedScheme = errors.New("unsupported scheme")
 )
 
-func FS(url string) (provider.FS, error) {
+func FS(url string, key []byte) (provider.FS, error) {
 	u, err := neturl.Parse(url)
 	if err != nil {
 		return nil, err
@@ -27,11 +28,20 @@ func FS(url string) (provider.FS, error) {
 	if !filepath.IsAbs(u.Path) {
 		return nil, ErrPathNotAbsolute
 	}
+	var fs provider.FS
 	switch u.Scheme {
 	case "file":
-		return mount.NewFS(u.Path)
+		fs, err = os.NewFS(u.Path)
 	case "sftp":
-		return sftp.NewFS(u, klogr.New().WithName("sftp"))
+		fs, err = sftp.NewFS(u, klogr.New().WithName("sftp"))
+	default:
+		return nil, fmt.Errorf("%w: %s", ErrUnsupportedScheme, u.Scheme)
 	}
-	return nil, fmt.Errorf("%w: %s", ErrUnsupportedScheme, u.Scheme)
+	if err != nil {
+		return nil, err
+	}
+	if key != nil {
+		fs, err = crypto.NewFS(fs, key)
+	}
+	return fs, err
 }

pkg/provider/sftp/sftp.go

@@ -160,7 +160,7 @@ func (fs *FS) OpenFile(p string, flags int, _ os.FileMode) (provider.File, error
 	return &file{f, fs}, err
 }
 
-func (fs *FS) Fstat(uintptr) (int64, error) {
+func (fs *FS) Fstat(_ uintptr) (int64, error) {
 	return 0, provider.ErrFstat
 }
 

run.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-./cachefs \
-  -src /home/marius/Desktop \
-  -dst /home/marius/mirrortmp \
-  -data /home/marius/mirrortmp/md.json \
-  -listen :8080 \
-  -v 2

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go

@@ -0,0 +1,98 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package chacha20poly1305 implements the ChaCha20-Poly1305 AEAD and its
+// extended nonce variant XChaCha20-Poly1305, as specified in RFC 8439 and
+// draft-irtf-cfrg-xchacha-01.
+package chacha20poly1305 // import "golang.org/x/crypto/chacha20poly1305"
+
+import (
+	"crypto/cipher"
+	"errors"
+)
+
+const (
+	// KeySize is the size of the key used by this AEAD, in bytes.
+	KeySize = 32
+
+	// NonceSize is the size of the nonce used with the standard variant of this
+	// AEAD, in bytes.
+	//
+	// Note that this is too short to be safely generated at random if the same
+	// key is reused more than 2³² times.
+	NonceSize = 12
+
+	// NonceSizeX is the size of the nonce used with the XChaCha20-Poly1305
+	// variant of this AEAD, in bytes.
+	NonceSizeX = 24
+
+	// Overhead is the size of the Poly1305 authentication tag, and the
+	// difference between a ciphertext length and its plaintext.
+	Overhead = 16
+)
+
+type chacha20poly1305 struct {
+	key [KeySize]byte
+}
+
+// New returns a ChaCha20-Poly1305 AEAD that uses the given 256-bit key.
+func New(key []byte) (cipher.AEAD, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("chacha20poly1305: bad key length")
+	}
+	ret := new(chacha20poly1305)
+	copy(ret.key[:], key)
+	return ret, nil
+}
+
+func (c *chacha20poly1305) NonceSize() int {
+	return NonceSize
+}
+
+func (c *chacha20poly1305) Overhead() int {
+	return Overhead
+}
+
+func (c *chacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if len(nonce) != NonceSize {
+		panic("chacha20poly1305: bad nonce length passed to Seal")
+	}
+
+	if uint64(len(plaintext)) > (1<<38)-64 {
+		panic("chacha20poly1305: plaintext too large")
+	}
+
+	return c.seal(dst, nonce, plaintext, additionalData)
+}
+
+var errOpen = errors.New("chacha20poly1305: message authentication failed")
+
+func (c *chacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if len(nonce) != NonceSize {
+		panic("chacha20poly1305: bad nonce length passed to Open")
+	}
+	if len(ciphertext) < 16 {
+		return nil, errOpen
+	}
+	if uint64(len(ciphertext)) > (1<<38)-48 {
+		panic("chacha20poly1305: ciphertext too large")
+	}
+
+	return c.open(dst, nonce, ciphertext, additionalData)
+}
+
+// sliceForAppend takes a slice and a requested number of bytes. It returns a
+// slice with the contents of the given slice followed by that many bytes and a
+// second slice that aliases into it and contains only the extra bytes. If the
+// original slice has sufficient capacity then no allocation is performed.
+func sliceForAppend(in []byte, n int) (head, tail []byte) {
+	if total := len(in) + n; cap(in) >= total {
+		head = in[:total]
+	} else {
+		head = make([]byte, total)
+		copy(head, in)
+	}
+	tail = head[len(in):]
+	return
+}

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go

@@ -0,0 +1,87 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build gc && !purego
+// +build gc,!purego
+
+package chacha20poly1305
+
+import (
+	"encoding/binary"
+
+	"golang.org/x/crypto/internal/subtle"
+	"golang.org/x/sys/cpu"
+)
+
+//go:noescape
+func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool
+
+//go:noescape
+func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte)
+
+var (
+	useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2
+)
+
+// setupState writes a ChaCha20 input matrix to state. See
+// https://tools.ietf.org/html/rfc7539#section-2.3.
+func setupState(state *[16]uint32, key *[32]byte, nonce []byte) {
+	state[0] = 0x61707865
+	state[1] = 0x3320646e
+	state[2] = 0x79622d32
+	state[3] = 0x6b206574
+
+	state[4] = binary.LittleEndian.Uint32(key[0:4])
+	state[5] = binary.LittleEndian.Uint32(key[4:8])
+	state[6] = binary.LittleEndian.Uint32(key[8:12])
+	state[7] = binary.LittleEndian.Uint32(key[12:16])
+	state[8] = binary.LittleEndian.Uint32(key[16:20])
+	state[9] = binary.LittleEndian.Uint32(key[20:24])
+	state[10] = binary.LittleEndian.Uint32(key[24:28])
+	state[11] = binary.LittleEndian.Uint32(key[28:32])
+
+	state[12] = 0
+	state[13] = binary.LittleEndian.Uint32(nonce[0:4])
+	state[14] = binary.LittleEndian.Uint32(nonce[4:8])
+	state[15] = binary.LittleEndian.Uint32(nonce[8:12])
+}
+
+func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if !cpu.X86.HasSSSE3 {
+		return c.sealGeneric(dst, nonce, plaintext, additionalData)
+	}
+
+	var state [16]uint32
+	setupState(&state, &c.key, nonce)
+
+	ret, out := sliceForAppend(dst, len(plaintext)+16)
+	if subtle.InexactOverlap(out, plaintext) {
+		panic("chacha20poly1305: invalid buffer overlap")
+	}
+	chacha20Poly1305Seal(out[:], state[:], plaintext, additionalData)
+	return ret
+}
+
+func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if !cpu.X86.HasSSSE3 {
+		return c.openGeneric(dst, nonce, ciphertext, additionalData)
+	}
+
+	var state [16]uint32
+	setupState(&state, &c.key, nonce)
+
+	ciphertext = ciphertext[:len(ciphertext)-16]
+	ret, out := sliceForAppend(dst, len(ciphertext))
+	if subtle.InexactOverlap(out, ciphertext) {
+		panic("chacha20poly1305: invalid buffer overlap")
+	}
+	if !chacha20Poly1305Open(out, state[:], ciphertext, additionalData) {
+		for i := range out {
+			out[i] = 0
+		}
+		return nil, errOpen
+	}
+
+	return ret, nil
+}

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_generic.go

@@ -0,0 +1,81 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package chacha20poly1305
+
+import (
+	"encoding/binary"
+
+	"golang.org/x/crypto/chacha20"
+	"golang.org/x/crypto/internal/poly1305"
+	"golang.org/x/crypto/internal/subtle"
+)
+
+func writeWithPadding(p *poly1305.MAC, b []byte) {
+	p.Write(b)
+	if rem := len(b) % 16; rem != 0 {
+		var buf [16]byte
+		padLen := 16 - rem
+		p.Write(buf[:padLen])
+	}
+}
+
+func writeUint64(p *poly1305.MAC, n int) {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], uint64(n))
+	p.Write(buf[:])
+}
+
+func (c *chacha20poly1305) sealGeneric(dst, nonce, plaintext, additionalData []byte) []byte {
+	ret, out := sliceForAppend(dst, len(plaintext)+poly1305.TagSize)
+	ciphertext, tag := out[:len(plaintext)], out[len(plaintext):]
+	if subtle.InexactOverlap(out, plaintext) {
+		panic("chacha20poly1305: invalid buffer overlap")
+	}
+
+	var polyKey [32]byte
+	s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
+	s.XORKeyStream(polyKey[:], polyKey[:])
+	s.SetCounter(1) // set the counter to 1, skipping 32 bytes
+	s.XORKeyStream(ciphertext, plaintext)
+
+	p := poly1305.New(&polyKey)
+	writeWithPadding(p, additionalData)
+	writeWithPadding(p, ciphertext)
+	writeUint64(p, len(additionalData))
+	writeUint64(p, len(plaintext))
+	p.Sum(tag[:0])
+
+	return ret
+}
+
+func (c *chacha20poly1305) openGeneric(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	tag := ciphertext[len(ciphertext)-16:]
+	ciphertext = ciphertext[:len(ciphertext)-16]
+
+	var polyKey [32]byte
+	s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
+	s.XORKeyStream(polyKey[:], polyKey[:])
+	s.SetCounter(1) // set the counter to 1, skipping 32 bytes
+
+	p := poly1305.New(&polyKey)
+	writeWithPadding(p, additionalData)
+	writeWithPadding(p, ciphertext)
+	writeUint64(p, len(additionalData))
+	writeUint64(p, len(ciphertext))
+
+	ret, out := sliceForAppend(dst, len(ciphertext))
+	if subtle.InexactOverlap(out, ciphertext) {
+		panic("chacha20poly1305: invalid buffer overlap")
+	}
+	if !p.Verify(tag) {
+		for i := range out {
+			out[i] = 0
+		}
+		return nil, errOpen
+	}
+
+	s.XORKeyStream(out, ciphertext)
+	return ret, nil
+}

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go

@@ -0,0 +1,16 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !amd64 || !gc || purego
+// +build !amd64 !gc purego
+
+package chacha20poly1305
+
+func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	return c.sealGeneric(dst, nonce, plaintext, additionalData)
+}
+
+func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	return c.openGeneric(dst, nonce, ciphertext, additionalData)
+}

vendor/golang.org/x/crypto/chacha20poly1305/xchacha20poly1305.go

@@ -0,0 +1,86 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package chacha20poly1305
+
+import (
+	"crypto/cipher"
+	"errors"
+
+	"golang.org/x/crypto/chacha20"
+)
+
+type xchacha20poly1305 struct {
+	key [KeySize]byte
+}
+
+// NewX returns a XChaCha20-Poly1305 AEAD that uses the given 256-bit key.
+//
+// XChaCha20-Poly1305 is a ChaCha20-Poly1305 variant that takes a longer nonce,
+// suitable to be generated randomly without risk of collisions. It should be
+// preferred when nonce uniqueness cannot be trivially ensured, or whenever
+// nonces are randomly generated.
+func NewX(key []byte) (cipher.AEAD, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("chacha20poly1305: bad key length")
+	}
+	ret := new(xchacha20poly1305)
+	copy(ret.key[:], key)
+	return ret, nil
+}
+
+func (*xchacha20poly1305) NonceSize() int {
+	return NonceSizeX
+}
+
+func (*xchacha20poly1305) Overhead() int {
+	return Overhead
+}
+
+func (x *xchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if len(nonce) != NonceSizeX {
+		panic("chacha20poly1305: bad nonce length passed to Seal")
+	}
+
+	// XChaCha20-Poly1305 technically supports a 64-bit counter, so there is no
+	// size limit. However, since we reuse the ChaCha20-Poly1305 implementation,
+	// the second half of the counter is not available. This is unlikely to be
+	// an issue because the cipher.AEAD API requires the entire message to be in
+	// memory, and the counter overflows at 256 GB.
+	if uint64(len(plaintext)) > (1<<38)-64 {
+		panic("chacha20poly1305: plaintext too large")
+	}
+
+	c := new(chacha20poly1305)
+	hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16])
+	copy(c.key[:], hKey)
+
+	// The first 4 bytes of the final nonce are unused counter space.
+	cNonce := make([]byte, NonceSize)
+	copy(cNonce[4:12], nonce[16:24])
+
+	return c.seal(dst, cNonce[:], plaintext, additionalData)
+}
+
+func (x *xchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if len(nonce) != NonceSizeX {
+		panic("chacha20poly1305: bad nonce length passed to Open")
+	}
+	if len(ciphertext) < 16 {
+		return nil, errOpen
+	}
+	if uint64(len(ciphertext)) > (1<<38)-48 {
+		panic("chacha20poly1305: ciphertext too large")
+	}
+
+	c := new(chacha20poly1305)
+	hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16])
+	copy(c.key[:], hKey)
+
+	// The first 4 bytes of the final nonce are unused counter space.
+	cNonce := make([]byte, NonceSize)
+	copy(cNonce[4:12], nonce[16:24])
+
+	return c.open(dst, cNonce[:], ciphertext, additionalData)
+}

vendor/golang.org/x/crypto/hkdf/hkdf.go

@@ -0,0 +1,93 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package hkdf implements the HMAC-based Extract-and-Expand Key Derivation
+// Function (HKDF) as defined in RFC 5869.
+//
+// HKDF is a cryptographic key derivation function (KDF) with the goal of
+// expanding limited input keying material into one or more cryptographically
+// strong secret keys.
+package hkdf // import "golang.org/x/crypto/hkdf"
+
+import (
+	"crypto/hmac"
+	"errors"
+	"hash"
+	"io"
+)
+
+// Extract generates a pseudorandom key for use with Expand from an input secret
+// and an optional independent salt.
+//
+// Only use this function if you need to reuse the extracted key with multiple
+// Expand invocations and different context values. Most common scenarios,
+// including the generation of multiple keys, should use New instead.
+func Extract(hash func() hash.Hash, secret, salt []byte) []byte {
+	if salt == nil {
+		salt = make([]byte, hash().Size())
+	}
+	extractor := hmac.New(hash, salt)
+	extractor.Write(secret)
+	return extractor.Sum(nil)
+}
+
+type hkdf struct {
+	expander hash.Hash
+	size     int
+
+	info    []byte
+	counter byte
+
+	prev []byte
+	buf  []byte
+}
+
+func (f *hkdf) Read(p []byte) (int, error) {
+	// Check whether enough data can be generated
+	need := len(p)
+	remains := len(f.buf) + int(255-f.counter+1)*f.size
+	if remains < need {
+		return 0, errors.New("hkdf: entropy limit reached")
+	}
+	// Read any leftover from the buffer
+	n := copy(p, f.buf)
+	p = p[n:]
+
+	// Fill the rest of the buffer
+	for len(p) > 0 {
+		f.expander.Reset()
+		f.expander.Write(f.prev)
+		f.expander.Write(f.info)
+		f.expander.Write([]byte{f.counter})
+		f.prev = f.expander.Sum(f.prev[:0])
+		f.counter++
+
+		// Copy the new batch into p
+		f.buf = f.prev
+		n = copy(p, f.buf)
+		p = p[n:]
+	}
+	// Save leftovers for next run
+	f.buf = f.buf[n:]
+
+	return need, nil
+}
+
+// Expand returns a Reader, from which keys can be read, using the given
+// pseudorandom key and optional context info, skipping the extraction step.
+//
+// The pseudorandomKey should have been generated by Extract, or be a uniformly
+// random or pseudorandom cryptographically strong key. See RFC 5869, Section
+// 3.3. Most common scenarios will want to use New instead.
+func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader {
+	expander := hmac.New(hash, pseudorandomKey)
+	return &hkdf{expander, expander.Size(), info, 1, nil, nil}
+}
+
+// New returns a Reader, from which keys can be read, using the given hash,
+// secret, salt and context info. Salt and info can be nil.
+func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader {
+	prk := Extract(hash, secret, salt)
+	return Expand(hash, prk, info)
+}

vendor/modules.txt

@@ -12,9 +12,11 @@ github.com/pkg/sftp/internal/encoding/ssh/filexfer
 ## explicit; go 1.17
 golang.org/x/crypto/blowfish
 golang.org/x/crypto/chacha20
+golang.org/x/crypto/chacha20poly1305
 golang.org/x/crypto/curve25519
 golang.org/x/crypto/curve25519/internal/field
 golang.org/x/crypto/ed25519
+golang.org/x/crypto/hkdf
 golang.org/x/crypto/internal/poly1305
 golang.org/x/crypto/internal/subtle
 golang.org/x/crypto/ssh