diff --git a/TODO.txt b/TODO.txt
index a53ee57..d7ba8ad 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -8,9 +8,5 @@
* implement read ahead buffer?
* add breadcrumb
-* add copyright footer
-* add stopped status?
- * replace stop with preload link
- * should redirect back to preloads page
* config file?
* whitelist allowed src paths
diff --git a/cmd/cachefs/main.go b/cmd/cachefs/main.go
index efccf35..ad97c90 100644
--- a/cmd/cachefs/main.go
+++ b/cmd/cachefs/main.go
@@ -6,6 +6,7 @@ import (
"context"
"flag"
"net/http"
+ _ "net/http/pprof"
"os"
"os/signal"
"syscall"
@@ -51,6 +52,9 @@ func main() {
flag.Int64Var("a, "quota", 1, "max disk usage quota for the dst cache in GiB")
flag.BoolVar(&block, "block", true, "block until sftp is connected")
flag.Parse()
+ go func() {
+ http.ListenAndServe("127.0.0.1:7777", nil)
+ }()
log = klogr.New().WithName("main")
log.Info("starting cachefs", "version", version)
diff --git a/pkg/fs/file.go b/pkg/fs/file.go
index 88a9979..7cc99df 100644
--- a/pkg/fs/file.go
+++ b/pkg/fs/file.go
@@ -15,6 +15,7 @@ import (
type File struct {
log logr.Logger
f provider.File
+ cache provider.File
md *Metadata
offset int64
offline bool
@@ -83,7 +84,7 @@ func (f *File) Preload(ctx context.Context, unlock func()) {
func (f *File) Read(p []byte) (n int, err error) {
log := f.log
if f.hasChunk(len(p)) {
- n, err = f.md.ReadAt(p, f.offset)
+ n, err = f.cache.ReadAt(p, f.offset)
if err == io.EOF {
return
}
@@ -172,8 +173,8 @@ func (f *File) Stat() (os.FileInfo, error) {
}
func (f *File) Close() error {
- if f.offline {
- return nil
+ if !f.offline && f.cache != nil {
+ f.cache.Close()
}
return f.f.Close()
}
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
index 3b0892e..313e1c2 100644
--- a/pkg/fs/fs.go
+++ b/pkg/fs/fs.go
@@ -135,6 +135,7 @@ func (fs *FS) OpenDst(name string) (f http.File, err error) {
f = &File{
log: log.WithName("file"),
f: file,
+ cache: file,
md: md,
offline: true,
}
@@ -198,13 +199,25 @@ func (fs *FS) Open(name string) (f http.File, err error) {
f = &Dir{log: log.WithName("dir"), f: file, dc: fs.dc}
return
}
+ md := fs.mh.Metadata(name, fi.Size())
+
+ var cache provider.File
if offline {
log.V(2).Info("file offline mode")
+ cache = file
+ } else {
+ cache, err = fs.dst.Open(name)
+ if err != nil {
+ if !skipLog(name) {
+ log.Error(err, "error opening cache file")
+ }
+ return
+ }
}
- md := fs.mh.Metadata(name, fi.Size())
f = &File{
log: log.WithName("file"),
f: file,
+ cache: cache,
md: md,
offline: offline,
}
@@ -231,7 +244,7 @@ func (fs *FS) openCacheFile(name string, size int64) (df provider.File, err erro
log.Error(err, "error opening cache file")
return
}
- if truncate {
+ if truncate && size > 0 {
err = df.Truncate(size)
if err != nil {
log.Error(err, "error truncating cache file")
diff --git a/pkg/fs/metadata.go b/pkg/fs/metadata.go
index a4478dd..a36e292 100644
--- a/pkg/fs/metadata.go
+++ b/pkg/fs/metadata.go
@@ -389,16 +389,16 @@ func (md *Metadata) AddChunk(off int64, n int) {
md.Chunks.Add(off, n)
}
-func (md *Metadata) ReadAt(p []byte, pos int64) (int, error) {
- if md.f == nil {
- err := md.openCacheFile()
- if err != nil {
- return 0, err
- }
- }
- atomic.StoreInt64(&md.Atime, now())
- return md.f.ReadAt(p, pos)
-}
+//func (md *Metadata) ReadAt(p []byte, pos int64) (int, error) {
+// if md.f == nil {
+// err := md.openCacheFile()
+// if err != nil {
+// return 0, err
+// }
+// }
+// atomic.StoreInt64(&md.Atime, now())
+// return md.f.ReadAt(p, pos)
+//}
func (md *Metadata) WriteAt(data []byte, pos int64) (int, error) {
if md.f == nil {
diff --git a/pkg/fs/preload.go b/pkg/fs/preload.go
index d4c073e..5a741f2 100644
--- a/pkg/fs/preload.go
+++ b/pkg/fs/preload.go
@@ -22,6 +22,9 @@ func (q *queue) Add(ph *PreloadHandler, name string, prio int) {
for _, p := range *q {
if p.Name == name {
p.Prio += prio
+ if p.Prio < 0 {
+ p.Prio = 0
+ }
q.Sort()
return
}
diff --git a/pkg/provider/crypto/file.go b/pkg/provider/crypto/file.go
index 1927465..e2f067b 100644
--- a/pkg/provider/crypto/file.go
+++ b/pkg/provider/crypto/file.go
@@ -139,7 +139,8 @@ func (f *file) Seek(offset int64, whence int) (n int64, err error) {
return
}
f.nonce = [NonceSize]byte{}
- inc(&f.nonce, cn)
+ //inc(&f.nonce, cn)
+ put(&f.nonce, cn)
f.r.nonce = f.nonce
f.w.nonce = f.nonce
f.r.off = roff
@@ -187,7 +188,8 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
c, ok := f.w.cm[cn]
if !ok {
c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
- inc(&c.nonce, cn)
+ //inc(&c.nonce, cn)
+ put(&c.nonce, cn)
f.w.cm[cn] = c
if f.w.last == nil {
f.w.last = c
@@ -199,7 +201,6 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
if c.full() {
// async flush
go func() {
- //_, err = c.flush(notLastChunk)
c.flush(notLastChunk)
}()
delete(f.w.cm, cn)
diff --git a/pkg/provider/crypto/helper.go b/pkg/provider/crypto/helper.go
index 71530e6..5438f85 100644
--- a/pkg/provider/crypto/helper.go
+++ b/pkg/provider/crypto/helper.go
@@ -3,6 +3,7 @@ package crypto
import (
"cachefs/pkg/provider"
"crypto/rand"
+ "encoding/binary"
"io"
"math"
)
@@ -29,7 +30,6 @@ func initNonce(f provider.File) (nonce []byte, err error) {
return
}
_, err = f.Write(nonce)
- return
}
return
}
@@ -76,3 +76,7 @@ func inc(nonce *[NonceSize]byte, n int64) {
incNonce(nonce)
}
}
+
+func put(nonce *[NonceSize]byte, n int64) {
+ binary.BigEndian.PutUint64((*nonce)[3:11], uint64(n))
+}
diff --git a/pkg/provider/crypto/helper_test.go b/pkg/provider/crypto/helper_test.go
new file mode 100644
index 0000000..0a41ae6
--- /dev/null
+++ b/pkg/provider/crypto/helper_test.go
@@ -0,0 +1,15 @@
+package crypto
+
+import (
+ "testing"
+)
+
+func BenchmarkInc(b *testing.B) {
+ var n [NonceSize]byte
+ inc(&n, int64(b.N))
+}
+
+func BenchmarkPut(b *testing.B) {
+ var n [NonceSize]byte
+ put(&n, int64(b.N))
+}
diff --git a/pkg/provider/crypto/reader.go b/pkg/provider/crypto/reader.go
index 4262aa6..93e65f8 100644
--- a/pkg/provider/crypto/reader.go
+++ b/pkg/provider/crypto/reader.go
@@ -10,7 +10,11 @@ import (
type cryptoReader interface {
io.Reader
- io.ReaderAt
+}
+
+type nonce struct {
+ N [NonceSize]byte
+ C int
}
type reader struct {
@@ -21,9 +25,10 @@ type reader struct {
buf [FullSize]byte
off int64
- err error
- nonce [NonceSize]byte
- cn int64
+ err error
+ nonce [NonceSize]byte
+ cn int64
+ outBuf []byte
}
func newReader(key []byte, src cryptoReader) (*reader, error) {
@@ -32,9 +37,10 @@ func newReader(key []byte, src cryptoReader) (*reader, error) {
return nil, err
}
return &reader{
- a: aead,
- src: src,
- cn: -1,
+ a: aead,
+ src: src,
+ cn: -1,
+ outBuf: make([]byte, 0, FullSize),
}, nil
}
@@ -91,19 +97,24 @@ func (r *reader) readChunk() (last bool, err error) {
return false, err
}
- outBuf := make([]byte, 0, ChunkSize)
- out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
+ //outBuf := make([]byte, 0, ChunkSize)
+ out, err := r.a.Open(r.outBuf, r.nonce[:], in, nil)
if err != nil && !last {
// Check if this was a full-length final chunk.
last = true
setLastChunkFlag(&r.nonce)
- out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
+ out, err = r.a.Open(r.outBuf, r.nonce[:], in, nil)
}
if err != nil {
return false, errors.New("failed to decrypt and authenticate payload chunk")
}
incNonce(&r.nonce)
- r.unread = r.buf[:copy(r.buf[:], out)]
+ //r.unread = r.buf[:copy(r.buf[:], out)]
+ size := FullSize
+ if len(out) < FullSize {
+ size = len(out)
+ }
+ r.unread = out[:size]
return last, nil
}
diff --git a/pkg/provider/sftp/file.go b/pkg/provider/sftp/file.go
new file mode 100644
index 0000000..3ec8f04
--- /dev/null
+++ b/pkg/provider/sftp/file.go
@@ -0,0 +1,34 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package sftp
+
+import (
+ "io/fs"
+
+ "github.com/pkg/sftp"
+)
+
+type file struct {
+ *sftp.File
+ fs *FS
+}
+
+func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
+ list, err := f.Readdir(0)
+ if err != nil {
+ return
+ }
+ d = make([]fs.DirEntry, len(list))
+ for i, v := range list {
+ d[i] = fs.FileInfoToDirEntry(v)
+ }
+ return
+}
+
+func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
+ return f.fs.client.ReadDir(f.Name())
+}
+
+func (file) Fd() uintptr {
+ return 0
+}
diff --git a/pkg/provider/sftp/sftp.go b/pkg/provider/sftp/sftp.go
index 0f0f357..5fb1e9c 100644
--- a/pkg/provider/sftp/sftp.go
+++ b/pkg/provider/sftp/sftp.go
@@ -101,19 +101,27 @@ func (fs *FS) connect(ctx context.Context) {
go fs.keepAlive(ctx)
fs.keepalive = true
}
- fs.client, err = sftp.NewClient(fs.c,
+ client, err := sftp.NewClient(fs.c,
sftp.UseFstat(true),
sftp.UseConcurrentReads(true),
sftp.UseConcurrentWrites(true),
+ //sftp.MaxPacketChecked(32768*2),
+ //sftp.MaxPacketUnchecked(32768*2),
+ //sftp.MaxConcurrentRequestsPerFile(128),
)
- select {
- case <-ctx.Done():
- return
- case e := <-fs.wait(ctx):
- log.V(2).Info("connection lost", "err", e)
- fs.conn.Close()
- fs.client = connError{}
- fs.c = nil
+ if err == nil {
+ fs.client = client
+ select {
+ case <-ctx.Done():
+ return
+ case e := <-fs.wait(ctx):
+ log.V(2).Info("connection lost", "err", e)
+ fs.conn.Close()
+ fs.client = connError{}
+ fs.c = nil
+ }
+ } else {
+ log.Error(err, "error creating client")
}
} else {
log.Error(err, "error connecting")
@@ -197,31 +205,6 @@ func (fs *FS) path(p string) string {
return filepath.Join(fs.root, filepath.FromSlash(path.Clean("/"+p)))
}
-type file struct {
- *sftp.File
- fs *FS
-}
-
-func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
- list, err := f.Readdir(0)
- if err != nil {
- return
- }
- d = make([]fs.DirEntry, len(list))
- for i, v := range list {
- d[i] = fs.FileInfoToDirEntry(v)
- }
- return
-}
-
-func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
- return f.fs.client.ReadDir(f.Name())
-}
-
-func (file) Fd() uintptr {
- return 0
-}
-
func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
if u.User == nil {
return nil, errors.New("missing username")
@@ -233,8 +216,9 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
c = &ssh.ClientConfig{
Config: ssh.Config{
//KeyExchanges: []string{"curve25519-sha256"},
- Ciphers: []string{"aes128-ctr"},
- MACs: []string{"hmac-sha2-256"},
+ //Ciphers: []string{"aes128-ctr"},
+ Ciphers: []string{"chacha20-poly1305@openssh.com"},
+ //MACs: []string{"hmac-sha2-256"},
},
User: u.User.Username(),
Timeout: time.Second * 30,
diff --git a/pkg/srv/templates/head.html b/pkg/srv/templates/head.html
index 290ba76..d1bc28e 100644
--- a/pkg/srv/templates/head.html
+++ b/pkg/srv/templates/head.html
@@ -111,5 +111,8 @@ span {
{{template "body" .}}
+