ston1th/cachefs
0
0
Fork 0
Code Issues Releases Activity

cache read performance improvements

Browse source
This commit is contained in:
ston1th 2022-05-19 01:29:23 +02:00
commit f69d3149f3
14 changed files with 140 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

7
pkg/provider/crypto/file.go
View file

@ -139,7 +139,8 @@ func (f *file) Seek(offset int64, whence int) (n int64, err error) {
return
}
f.nonce = [NonceSize]byte{}
inc(&f.nonce, cn)
//inc(&f.nonce, cn)
put(&f.nonce, cn)
f.r.nonce = f.nonce
f.w.nonce = f.nonce
f.r.off = roff
@ -187,7 +188,8 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
c, ok := f.w.cm[cn]
if !ok {
c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
inc(&c.nonce, cn)
//inc(&c.nonce, cn)
put(&c.nonce, cn)
f.w.cm[cn] = c
if f.w.last == nil {
f.w.last = c
@ -199,7 +201,6 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
if c.full() {
// async flush
go func() {
//_, err = c.flush(notLastChunk)
c.flush(notLastChunk)
}()
delete(f.w.cm, cn)

6
pkg/provider/crypto/helper.go
View file

@ -3,6 +3,7 @@ package crypto
import (
"cachefs/pkg/provider"
"crypto/rand"
"encoding/binary"
"io"
"math"
)
@ -29,7 +30,6 @@ func initNonce(f provider.File) (nonce []byte, err error) {
return
}
_, err = f.Write(nonce)
return
}
return
}
@ -76,3 +76,7 @@ func inc(nonce *[NonceSize]byte, n int64) {
incNonce(nonce)
}
}
func put(nonce *[NonceSize]byte, n int64) {
binary.BigEndian.PutUint64((*nonce)[3:11], uint64(n))
}

15
pkg/provider/crypto/helper_test.go Normal file
View file

@ -0,0 +1,15 @@
package crypto
import (
"testing"
)
func BenchmarkInc(b *testing.B) {
var n [NonceSize]byte
inc(&n, int64(b.N))
}
func BenchmarkPut(b *testing.B) {
var n [NonceSize]byte
put(&n, int64(b.N))
}

33
pkg/provider/crypto/reader.go
View file

@ -10,7 +10,11 @@ import (
type cryptoReader interface {
io.Reader
io.ReaderAt
}
type nonce struct {
N [NonceSize]byte
C int
}
type reader struct {
@ -21,9 +25,10 @@ type reader struct {
buf [FullSize]byte
off int64
err error
nonce [NonceSize]byte
cn int64
err error
nonce [NonceSize]byte
cn int64
outBuf []byte
}
func newReader(key []byte, src cryptoReader) (*reader, error) {
@ -32,9 +37,10 @@ func newReader(key []byte, src cryptoReader) (*reader, error) {
return nil, err
}
return &reader{
a: aead,
src: src,
cn: -1,
a: aead,
src: src,
cn: -1,
outBuf: make([]byte, 0, FullSize),
}, nil
}
@ -91,19 +97,24 @@ func (r *reader) readChunk() (last bool, err error) {
return false, err
}
outBuf := make([]byte, 0, ChunkSize)
out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
//outBuf := make([]byte, 0, ChunkSize)
out, err := r.a.Open(r.outBuf, r.nonce[:], in, nil)
if err != nil && !last {
// Check if this was a full-length final chunk.
last = true
setLastChunkFlag(&r.nonce)
out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
out, err = r.a.Open(r.outBuf, r.nonce[:], in, nil)
}
if err != nil {
return false, errors.New("failed to decrypt and authenticate payload chunk")
}
incNonce(&r.nonce)
r.unread = r.buf[:copy(r.buf[:], out)]
//r.unread = r.buf[:copy(r.buf[:], out)]
size := FullSize
if len(out) < FullSize {
size = len(out)
}
r.unread = out[:size]
return last, nil
}

34
pkg/provider/sftp/file.go Normal file
View file

@ -0,0 +1,34 @@
// Copyright (C) 2022 Marius Schellenberger
package sftp
import (
"io/fs"
"github.com/pkg/sftp"
)
type file struct {
*sftp.File
fs *FS
}
func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
list, err := f.Readdir(0)
if err != nil {
return
}
d = make([]fs.DirEntry, len(list))
for i, v := range list {
d[i] = fs.FileInfoToDirEntry(v)
}
return
}
func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
return f.fs.client.ReadDir(f.Name())
}
func (file) Fd() uintptr {
return 0
}

56
pkg/provider/sftp/sftp.go
View file

@ -101,19 +101,27 @@ func (fs *FS) connect(ctx context.Context) {
go fs.keepAlive(ctx)
fs.keepalive = true
}
fs.client, err = sftp.NewClient(fs.c,
client, err := sftp.NewClient(fs.c,
sftp.UseFstat(true),
sftp.UseConcurrentReads(true),
sftp.UseConcurrentWrites(true),
//sftp.MaxPacketChecked(32768*2),
//sftp.MaxPacketUnchecked(32768*2),
//sftp.MaxConcurrentRequestsPerFile(128),
)
select {
case <-ctx.Done():
return
case e := <-fs.wait(ctx):
log.V(2).Info("connection lost", "err", e)
fs.conn.Close()
fs.client = connError{}
fs.c = nil
if err == nil {
fs.client = client
select {
case <-ctx.Done():
return
case e := <-fs.wait(ctx):
log.V(2).Info("connection lost", "err", e)
fs.conn.Close()
fs.client = connError{}
fs.c = nil
}
} else {
log.Error(err, "error creating client")
}
} else {
log.Error(err, "error connecting")
@ -197,31 +205,6 @@ func (fs *FS) path(p string) string {
return filepath.Join(fs.root, filepath.FromSlash(path.Clean("/"+p)))
}
type file struct {
*sftp.File
fs *FS
}
func (f *file) ReadDir(_ int) (d []fs.DirEntry, err error) {
list, err := f.Readdir(0)
if err != nil {
return
}
d = make([]fs.DirEntry, len(list))
for i, v := range list {
d[i] = fs.FileInfoToDirEntry(v)
}
return
}
func (f *file) Readdir(_ int) ([]fs.FileInfo, error) {
return f.fs.client.ReadDir(f.Name())
}
func (file) Fd() uintptr {
return 0
}
func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
if u.User == nil {
return nil, errors.New("missing username")
@ -233,8 +216,9 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
c = &ssh.ClientConfig{
Config: ssh.Config{
//KeyExchanges: []string{"curve25519-sha256"},
Ciphers: []string{"aes128-ctr"},
MACs: []string{"hmac-sha2-256"},
//Ciphers: []string{"aes128-ctr"},
Ciphers: []string{"chacha20-poly1305@openssh.com"},
//MACs: []string{"hmac-sha2-256"},
},
User: u.User.Username(),
Timeout: time.Second * 30,