cache read performance improvements

TODO.txt

@@ -8,9 +8,5 @@
   * implement read ahead buffer?
 
 * add breadcrumb
-* add copyright footer
-* add stopped status?
-  * replace stop with preload link
-  * should redirect back to preloads page
 * config file?
   * whitelist allowed src paths

cmd/cachefs/main.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"flag"
 	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
@@ -51,6 +52,9 @@ func main() {
 	flag.Int64Var(&quota, "quota", 1, "max disk usage quota for the dst cache in GiB")
 	flag.BoolVar(&block, "block", true, "block until sftp is connected")
 	flag.Parse()
+	go func() {
+		http.ListenAndServe("127.0.0.1:7777", nil)
+	}()
 
 	log = klogr.New().WithName("main")
 	log.Info("starting cachefs", "version", version)

pkg/fs/file.go

@@ -15,6 +15,7 @@ import (
 type File struct {
 	log     logr.Logger
 	f       provider.File
+	cache   provider.File
 	md      *Metadata
 	offset  int64
 	offline bool
@@ -83,7 +84,7 @@ func (f *File) Preload(ctx context.Context, unlock func()) {
 func (f *File) Read(p []byte) (n int, err error) {
 	log := f.log
 	if f.hasChunk(len(p)) {
-		n, err = f.md.ReadAt(p, f.offset)
+		n, err = f.cache.ReadAt(p, f.offset)
 		if err == io.EOF {
 			return
 		}
@@ -172,8 +173,8 @@ func (f *File) Stat() (os.FileInfo, error) {
 }
 
 func (f *File) Close() error {
-	if f.offline {
+	if !f.offline && f.cache != nil {
-		return nil
+		f.cache.Close()
 	}
 	return f.f.Close()
 }

pkg/fs/fs.go

@@ -135,6 +135,7 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
 	f = &File{
 		log:     log.WithName("file"),
 		f:       file,
+		cache:   file,
 		md:      md,
 		offline: true,
 	}
@@ -198,13 +199,25 @@ func (fs *FS) Open(name string) (f http.File, err error) {
 		f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
 		return
 	}
+	md := fs.mh.Metadata(name, fi.Size())
+
+	var cache provider.File
 	if offline {
 		log.V(2).Info("file offline mode")
+		cache = file
+	} else {
+		cache, err = fs.dst.Open(name)
+		if err != nil {
+			if !skipLog(name) {
+				log.Error(err, "error opening cache file")
+			}
+			return
+		}
 	}
-	md := fs.mh.Metadata(name, fi.Size())
 	f = &File{
 		log:     log.WithName("file"),
 		f:       file,
+		cache:   cache,
 		md:      md,
 		offline: offline,
 	}
@@ -231,7 +244,7 @@ func (fs *FS) openCacheFile(name string, size int64) (df provider.File, err erro
 		log.Error(err, "error opening cache file")
 		return
 	}
-	if truncate {
+	if truncate && size > 0 {
 		err = df.Truncate(size)
 		if err != nil {
 			log.Error(err, "error truncating cache file")

pkg/fs/metadata.go

@@ -389,16 +389,16 @@ func (md *Metadata) AddChunk(off int64, n int) {
 	md.Chunks.Add(off, n)
 }
 
-func (md *Metadata) ReadAt(p []byte, pos int64) (int, error) {
+//func (md *Metadata) ReadAt(p []byte, pos int64) (int, error) {
-	if md.f == nil {
+//	if md.f == nil {
-		err := md.openCacheFile()
+//		err := md.openCacheFile()
-		if err != nil {
+//		if err != nil {
-			return 0, err
+//			return 0, err
-		}
+//		}
-	}
+//	}
-	atomic.StoreInt64(&md.Atime, now())
+//	atomic.StoreInt64(&md.Atime, now())
-	return md.f.ReadAt(p, pos)
+//	return md.f.ReadAt(p, pos)
-}
+//}
 
 func (md *Metadata) WriteAt(data []byte, pos int64) (int, error) {
 	if md.f == nil {

pkg/fs/preload.go

@@ -22,6 +22,9 @@ func (q *queue) Add(ph *PreloadHandler, name string, prio int) {
 	for _, p := range *q {
 		if p.Name == name {
 			p.Prio += prio
+			if p.Prio < 0 {
+				p.Prio = 0
+			}
 			q.Sort()
 			return
 		}

pkg/provider/crypto/file.go

@@ -139,7 +139,8 @@ func (f *file) Seek(offset int64, whence int) (n int64, err error) {
 		return
 	}
 	f.nonce = [NonceSize]byte{}
-	inc(&f.nonce, cn)
+	//inc(&f.nonce, cn)
+	put(&f.nonce, cn)
 	f.r.nonce = f.nonce
 	f.w.nonce = f.nonce
 	f.r.off = roff
@@ -187,7 +188,8 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
 	c, ok := f.w.cm[cn]
 	if !ok {
 		c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
-		inc(&c.nonce, cn)
+		//inc(&c.nonce, cn)
+		put(&c.nonce, cn)
 		f.w.cm[cn] = c
 		if f.w.last == nil {
 			f.w.last = c
@@ -199,7 +201,6 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
 	if c.full() {
 		// async flush
 		go func() {
-			//_, err = c.flush(notLastChunk)
 			c.flush(notLastChunk)
 		}()
 		delete(f.w.cm, cn)

pkg/provider/crypto/helper.go

@@ -3,6 +3,7 @@ package crypto
 import (
 	"cachefs/pkg/provider"
 	"crypto/rand"
+	"encoding/binary"
 	"io"
 	"math"
 )
@@ -29,7 +30,6 @@ func initNonce(f provider.File) (nonce []byte, err error) {
 			return
 		}
 		_, err = f.Write(nonce)
-		return
 	}
 	return
 }
@@ -76,3 +76,7 @@ func inc(nonce *[NonceSize]byte, n int64) {
 		incNonce(nonce)
 	}
 }
+
+func put(nonce *[NonceSize]byte, n int64) {
+	binary.BigEndian.PutUint64((*nonce)[3:11], uint64(n))
+}

pkg/provider/crypto/helper_test.go

@@ -0,0 +1,15 @@
+package crypto
+
+import (
+	"testing"
+)
+
+func BenchmarkInc(b *testing.B) {
+	var n [NonceSize]byte
+	inc(&n, int64(b.N))
+}
+
+func BenchmarkPut(b *testing.B) {
+	var n [NonceSize]byte
+	put(&n, int64(b.N))
+}

pkg/provider/crypto/reader.go

@@ -10,7 +10,11 @@ import (
 
 type cryptoReader interface {
 	io.Reader
-	io.ReaderAt
+}
+
+type nonce struct {
+	N [NonceSize]byte
+	C int
 }
 
 type reader struct {
@@ -24,6 +28,7 @@ type reader struct {
 	err    error
 	nonce  [NonceSize]byte
 	cn     int64
+	outBuf []byte
 }
 
 func newReader(key []byte, src cryptoReader) (*reader, error) {
@@ -35,6 +40,7 @@ func newReader(key []byte, src cryptoReader) (*reader, error) {
 		a:      aead,
 		src:    src,
 		cn:     -1,
+		outBuf: make([]byte, 0, FullSize),
 	}, nil
 }
 
@@ -91,19 +97,24 @@ func (r *reader) readChunk() (last bool, err error) {
 		return false, err
 	}
 
-	outBuf := make([]byte, 0, ChunkSize)
+	//outBuf := make([]byte, 0, ChunkSize)
-	out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
+	out, err := r.a.Open(r.outBuf, r.nonce[:], in, nil)
 	if err != nil && !last {
 		// Check if this was a full-length final chunk.
 		last = true
 		setLastChunkFlag(&r.nonce)
-		out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
+		out, err = r.a.Open(r.outBuf, r.nonce[:], in, nil)
 	}
 	if err != nil {
 		return false, errors.New("failed to decrypt and authenticate payload chunk")
 	}
 
 	incNonce(&r.nonce)
-	r.unread = r.buf[:copy(r.buf[:], out)]
+	//r.unread = r.buf[:copy(r.buf[:], out)]
+	size := FullSize
+	if len(out) < FullSize {
+		size = len(out)
+	}
+	r.unread = out[:size]
 	return last, nil
 }

pkg/provider/sftp/file.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package sftp
+
+import (
+	"io/fs"
+
+	"github.com/pkg/sftp"
+)
+
+type file struct {
+	*sftp.File
+	fs *FS
+}
+
+func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
+	list, err := f.Readdir(0)
+	if err != nil {
+		return
+	}
+	d = make([]fs.DirEntry, len(list))
+	for i, v := range list {
+		d[i] = fs.FileInfoToDirEntry(v)
+	}
+	return
+}
+
+func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
+	return f.fs.client.ReadDir(f.Name())
+}
+
+func (file) Fd() uintptr {
+	return 0
+}

pkg/provider/sftp/sftp.go

@@ -101,11 +101,16 @@ func (fs *FS) connect(ctx context.Context) {
 				go fs.keepAlive(ctx)
 				fs.keepalive = true
 			}
-			fs.client, err = sftp.NewClient(fs.c,
+			client, err := sftp.NewClient(fs.c,
 				sftp.UseFstat(true),
 				sftp.UseConcurrentReads(true),
 				sftp.UseConcurrentWrites(true),
+				//sftp.MaxPacketChecked(32768*2),
+				//sftp.MaxPacketUnchecked(32768*2),
+				//sftp.MaxConcurrentRequestsPerFile(128),
 			)
+			if err == nil {
+				fs.client = client
 				select {
 				case <-ctx.Done():
 					return
@@ -115,6 +120,9 @@ func (fs *FS) connect(ctx context.Context) {
 					fs.client = connError{}
 					fs.c = nil
 				}
+			} else {
+				log.Error(err, "error creating client")
+			}
 		} else {
 			log.Error(err, "error connecting")
 		}
@@ -197,31 +205,6 @@ func (fs *FS) path(p string) string {
 	return filepath.Join(fs.root, filepath.FromSlash(path.Clean("/"+p)))
 }
 
-type file struct {
-	*sftp.File
-	fs *FS
-}
-
-func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
-	list, err := f.Readdir(0)
-	if err != nil {
-		return
-	}
-	d = make([]fs.DirEntry, len(list))
-	for i, v := range list {
-		d[i] = fs.FileInfoToDirEntry(v)
-	}
-	return
-}
-
-func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
-	return f.fs.client.ReadDir(f.Name())
-}
-
-func (file) Fd() uintptr {
-	return 0
-}
-
 func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 	if u.User == nil {
 		return nil, errors.New("missing username")
@@ -233,8 +216,9 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 	c = &ssh.ClientConfig{
 		Config: ssh.Config{
 			//KeyExchanges: []string{"curve25519-sha256"},
-			Ciphers: []string{"aes128-ctr"},
+			//Ciphers: []string{"aes128-ctr"},
-			MACs:    []string{"hmac-sha2-256"},
+			Ciphers: []string{"chacha20-poly1305@openssh.com"},
+			//MACs:    []string{"hmac-sha2-256"},
 		},
 		User:    u.User.Username(),
 		Timeout: time.Second * 30,

pkg/srv/templates/head.html

@@ -111,5 +111,8 @@ span {
   <article>
 {{template "body" .}}
   </article>
+  <footer>
+    <pre>© CacheFS v1.0 <a href="https://git.giftfish.de/ston1th/cachefs" rel="nofollow noreferrer noopener" target="_blank">[Source]</a></pre>
+  </footer>
 </body>
 </html>

pkg/srv/templates/index.html

@@ -1,6 +1,6 @@
 {{define "body" -}}
 <pre>[v]: show video
-[n]: skip file caching
+[n]: stream file without caching
 [p]: preload file
 <a href="/?o=preloads">[Preloads]</a></pre>
 <table>