ston1th/cachefs
0
0
Fork 0
Code Issues Releases Activity

continued work on encryption

Browse source
This commit is contained in:
ston1th 2022-04-07 00:54:13 +02:00
commit 880e2f06b3
14 changed files with 362 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

97
pkg/provider/crypto/file.go
View file

@ -4,6 +4,7 @@ import (
"cachefs/pkg/provider"
"io"
"io/fs"
"os"
)
var _ provider.File = (*file)(nil)
@ -16,6 +17,7 @@ type file struct {
nonce [NonceSize]byte
offset int64
name string
stat *stat
dir bool
}
@ -41,41 +43,81 @@ func newFile(key []byte, name string, fs *FS, f provider.File, dir bool) (ef pro
if err != nil {
return
}
return &file{
ef = &file{
File: f,
fs: fs,
w: w,
r: r,
name: name,
}, nil
}
return ef, nil
}
func (f *file) Stat() (fs.FileInfo, error) {
if f.stat != nil {
return f.stat, nil
}
fi, err := f.File.Stat()
return stat{FileInfo: fi, name: f.name}, err
if err != nil {
return nil, err
}
if f.dir {
return dir{FileInfo: fi, name: f.name}, nil
}
rs := RealSize(fi.Size())
f.stat = &stat{
FileInfo: fi,
size: rs,
name: f.name,
}
return f.stat, nil
//fmt.Println("rs", rs)
//_, err = f.ReadAt(nil, rs)
//if err != nil && err != io.EOF {
// return nil, err
//}
//f.stat = &stat{
// FileInfo: fi,
// size: rs - (ChunkSize - int64(len(f.r.unread))),
// name: f.name,
//}
//fmt.Println("size", f.stat.Size())
//f.Seek(0, io.SeekStart)
//return f.stat, nil
}
func (f *file) Name() string {
return f.name
}
func (file) Fd() uintptr {
func (f *file) Fd() uintptr {
osf, ok := f.File.(*os.File)
if ok {
return osf.Fd()
}
return 0
}
type stat struct {
fs.FileInfo
size int64
name string
}
func (s stat) Size() int64 {
return RealSize(s.FileInfo.Size())
return s.size
}
func (s stat) Name() string {
return s.name
}
func (f *file) Truncate(n int64) error {
return f.File.Truncate(Size(n))
}
func (f *file) Close() error {
if f.dir {
return f.File.Close()
@ -104,16 +146,23 @@ func (f *file) Seek(offset int64, whence int) (n int64, err error) {
_, err = f.File.Seek(f.offset, io.SeekStart)
n = offset
}
f.reset()
if err != nil {
return
}
f.nonce = [NonceSize]byte{}
inc(&f.nonce, cn)
f.r.nonce = f.nonce
f.r.off = roff
f.r.err = nil
f.w.nonce = f.nonce
f.r.unread = f.r.unread[:0]
f.r.off = roff
return
}
func (f *file) reset() {
f.r.err = nil
f.r.unread = f.r.unread[:0]
}
func (f *file) Read(p []byte) (n int, err error) {
n, err = f.r.Read(p)
return
@ -125,21 +174,28 @@ func (f *file) Write(p []byte) (n int, err error) {
}
func (f *file) ReadAt(p []byte, pos int64) (n int, err error) {
_, _, roff := align(pos)
//cn, off, roff := align(pos)
f.Seek(pos, io.SeekStart)
//fmt.Println("readAt:", cn, off+KDFNonceSize, roff, pos)
_, err = f.r.readChunk()
if err != nil {
return
cn, _, roff := align(pos)
var last bool
if f.r.cn != cn {
_, err = f.Seek(pos, io.SeekStart)
if err != nil {
return
}
last, err = f.r.readChunk()
if err != nil {
return
}
f.r.cn = cn
}
n = copy(p, f.r.unread[roff:])
if last && len(f.r.unread) == n+int(roff) {
err = io.EOF
}
return
}
func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
cn, off, woff := align(pos)
//fmt.Println("writeAt:", cn, off+KDFNonceSize, woff, pos)
c, ok := f.w.cm[cn]
if !ok {
c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
@ -153,8 +209,11 @@ func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
}
n, err = c.writeAt(data, woff)
if c.full() {
_, err = c.flush(notLastChunk)
//fmt.Println("flush:", cn)
// async flush
go func() {
//_, err = c.flush(notLastChunk)
c.flush(notLastChunk)
}()
delete(f.w.cm, cn)
}
return
@ -171,7 +230,7 @@ func (f *file) Readdir(n int) ([]fs.FileInfo, error) {
if err != nil {
return nil, err
}
fio[i] = dir{FileInfo: fi, name: name}
fio[i] = stat{FileInfo: fi, size: RealSize(fi.Size()), name: name}
}
return fio, nil
}

7
pkg/provider/crypto/fs.go
View file

@ -2,6 +2,7 @@ package crypto
import (
"cachefs/pkg/provider"
osp "cachefs/pkg/provider/os"
"encoding/base64"
"errors"
"io/fs"
@ -102,7 +103,11 @@ func (fs *FS) Root() string {
return fs.fs.Root()
}
func (fs *FS) Fstat(_ uintptr) (int64, error) {
func (fs *FS) Fstat(fd uintptr) (int64, error) {
osp, ok := fs.fs.(*osp.FS)
if ok {
return osp.Fstat(fd)
}
return 0, provider.ErrFstat
}

5
pkg/provider/crypto/reader.go
View file

@ -3,6 +3,7 @@ package crypto
import (
"crypto/cipher"
"errors"
"fmt"
"io"
"golang.org/x/crypto/chacha20poly1305"
@ -23,6 +24,7 @@ type reader struct {
err error
nonce [NonceSize]byte
cn int64
}
func newReader(key []byte, src cryptoReader) (*reader, error) {
@ -33,6 +35,7 @@ func newReader(key []byte, src cryptoReader) (*reader, error) {
return &reader{
a: aead,
src: src,
cn: -1,
}, nil
}
@ -84,7 +87,9 @@ func (r *reader) readChunk() (last bool, err error) {
// The last chunk can be short.
in = in[:n]
last = true
fmt.Println("last nonce", r.nonce)
setLastChunkFlag(&r.nonce)
fmt.Println("last nonce flag", r.nonce)
case err != nil:
return false, err
}

7
pkg/provider/crypto/writer.go
View file

@ -4,6 +4,7 @@ import (
"cachefs/pkg/chunk"
"crypto/cipher"
"errors"
"fmt"
"io"
"golang.org/x/crypto/chacha20poly1305"
@ -71,7 +72,9 @@ func (c *chunkWriter) full() bool {
func (c *chunkWriter) flush(last bool) (n int, err error) {
wb := c.buf[:0]
if last {
fmt.Println("last nonce", c.nonce)
setLastChunkFlag(&c.nonce)
fmt.Println("last nonce flag", c.nonce)
s := c.chunks.Size()
wb = c.buf[:s]
n = int(s)
@ -112,15 +115,19 @@ func (w *writer) Write(p []byte) (n int, err error) {
}
func (w *writer) Close() error {
fmt.Println("writer closed", w.err)
if w.err != nil {
return w.err
}
if w.last != nil {
fmt.Println("flushing last chunk")
_, w.err = w.last.flush(lastChunk)
if w.err != nil {
fmt.Println("error flushing last chunk", w.err)
return w.err
}
w.flush = false
}
if w.flush {

20
pkg/provider/parse/parse.go
View file

@ -20,7 +20,23 @@ var (
ErrUnsupportedScheme = errors.New("unsupported scheme")
)
func FS(url string, key []byte) (provider.FS, error) {
func TryParse(url string) error {
u, err := neturl.Parse(url)
if err != nil {
return err
}
if !filepath.IsAbs(u.Path) {
return ErrPathNotAbsolute
}
switch u.Scheme {
case "file", "sftp":
default:
return fmt.Errorf("%w: %s", ErrUnsupportedScheme, u.Scheme)
}
return nil
}
func FS(url string, key []byte, block bool) (provider.FS, error) {
u, err := neturl.Parse(url)
if err != nil {
return nil, err
@ -33,7 +49,7 @@ func FS(url string, key []byte) (provider.FS, error) {
case "file":
fs, err = os.NewFS(u.Path)
case "sftp":
fs, err = sftp.NewFS(u, klogr.New().WithName("sftp"))
fs, err = sftp.NewFS(u, block, klogr.New().WithName("sftp"))
default:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedScheme, u.Scheme)
}

17
pkg/provider/sftp/sftp.go
View file

@ -51,7 +51,7 @@ type FS struct {
keepalive bool
}
func NewFS(u *url.URL, log logr.Logger) (fs *FS, err error) {
func NewFS(u *url.URL, block bool, log logr.Logger) (fs *FS, err error) {
cfg, err := sshConfig(u)
if err != nil {
return
@ -66,6 +66,14 @@ func NewFS(u *url.URL, log logr.Logger) (fs *FS, err error) {
ctx, cancel := context.WithCancel(context.Background())
fs.cancel = cancel
go fs.connect(ctx)
if block {
for {
if _, e := fs.Stat("."); e != ErrNotConnected {
return
}
time.Sleep(time.Millisecond * 100)
}
}
return
}
@ -95,7 +103,12 @@ func (fs *FS) connect(ctx context.Context) {
go fs.keepAlive(ctx)
fs.keepalive = true
}
fs.client, err = sftp.NewClient(fs.c, sftp.UseFstat(true))
fs.client, err = sftp.NewClient(fs.c,
sftp.UseFstat(true),
sftp.UseConcurrentReads(true),
//sftp.UseConcurrentWrites(true),
//sftp.MaxPacket(23552),
)
select {
case <-ctx.Done():
return