ston1th/cachefs
0
0
Fork 0
Code Issues Releases Activity

added fs encryption overlay

Browse source
This commit is contained in:
ston1th 2022-04-06 01:08:19 +02:00
commit fd1e8118a4
28 changed files with 3994 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
/cachefs
/metadatagen
/run.sh

4
TODO.txt
View file

@ -1,5 +1,9 @@
* document openbsd pcap wireshark fix
* printf '\x6c' | dd seek=20 bs=1 count=1 conv=notrunc of=test.dump
* crypto
* flush last chunk in streaming mode
* add breadcrumb
* add copyright footer
* add stopped status?

15
cmd/cachefs/main.go
View file

@ -24,6 +24,8 @@ var (
src string
dst string
srckey string
dstkey string
metadata string
listenHttp string
listenWebdav string
@ -40,7 +42,9 @@ const gib = 1024 * 1024 * 1024
func main() {
klog.InitFlags(nil)
flag.StringVar(&src, "src", "", "url path to source files (example: file:///mnt/nfs)")
flag.StringVar(&srckey, "srckey", "", "hex encoded encryption key for src")
flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
flag.StringVar(&metadata, "data", "", "path to metadata file")
flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
@ -57,7 +61,16 @@ func main() {
log.V(2).Info("changed preload buffer size", "size", bs)
}
filesystem, err := fs.NewFS(quota*gib, max, src, dst, metadata, klogr.New().WithName("fs"))
filesystem, err := fs.NewFS(
quota*gib,
max,
src,
srckey,
dst,
dstkey,
metadata,
klogr.New().WithName("fs"),
)
if err != nil {
klog.Fatalf("init failed: %s", err)
}

14
cmd/metadatagen/main.go
View file

@ -3,6 +3,7 @@
package main
import (
"encoding/hex"
"flag"
"fmt"
"os"
@ -15,15 +16,26 @@ var (
version string
dst string
dstkey string
metadata string
)
func main() {
flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
flag.StringVar(&dstkey, "dstkey", "", "hex encoded encryption key for dst")
flag.StringVar(&metadata, "data", "", "path to metadata file")
flag.Parse()
dstfs, err := parse.FS(dst)
var (
dstk []byte
err error
)
if dstkey != "" {
dstk, err = hex.DecodeString(dstkey)
fmt.Println(err)
os.Exit(1)
}
dstfs, err := parse.FS(dst, dstk)
if err != nil {
fmt.Println(err)
os.Exit(1)

2
pkg/fs/chunk.go → pkg/chunk/chunk.go
View file

@ -1,6 +1,6 @@
// Copyright (C) 2022 Marius Schellenberger
package fs
package chunk
import (
"golang.org/x/exp/slices"

2
pkg/fs/file.go
View file

@ -49,6 +49,7 @@ func (p *preload) Read(data []byte) (n int, err error) {
func (f *File) Preload(ctx context.Context, unlock func()) {
log := f.log
defer f.Close()
defer unlock()
if f.offline {
log.V(2).Error(errors.New("no preload in offline mode"), "error preloading file")
@ -68,6 +69,7 @@ func (f *File) Preload(ctx context.Context, unlock func()) {
if err != nil && err != io.EOF {
log.Error(err, "error preloading file")
}
f.md.Close()
log.V(2).Info("preload finished", "skipped", p.skipped, "written", p.written)
}

20
pkg/fs/fs.go
View file

@ -6,6 +6,7 @@ import (
"cachefs/pkg/provider"
"cachefs/pkg/provider/parse"
"context"
"encoding/hex"
"errors"
"fmt"
stdfs "io/fs"
@ -31,15 +32,28 @@ type FS struct {
ph *PreloadHandler
}
func NewFS(quota int64, max int, src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, log logr.Logger) (fs *FS, err error) {
if src == dst {
return nil, errors.New("src and dst path can not be equal")
}
srcfs, err := parse.FS(src)
var srck, dstk []byte
if srckey != "" {
srck, err = hex.DecodeString(srckey)
if err != nil {
return nil, fmt.Errorf("srckey: %w", err)
}
}
if dstkey != "" {
dstk, err = hex.DecodeString(dstkey)
if err != nil {
return nil, fmt.Errorf("dstkey: %w", err)
}
}
srcfs, err := parse.FS(src, srck)
if err != nil {
return nil, fmt.Errorf("srcfs: %w", err)
}
dstfs, err := parse.FS(dst)
dstfs, err := parse.FS(dst, dstk)
if err != nil {
return nil, fmt.Errorf("dstfs: %w", err)
}

15
pkg/fs/metadata.go
View file

@ -3,6 +3,7 @@
package fs
import (
"cachefs/pkg/chunk"
"cachefs/pkg/provider"
"context"
"encoding/json"
@ -80,7 +81,7 @@ func MetadataGenerator(file string, dst provider.FS) error {
if size <= blocks*512 {
md[strings.TrimPrefix(path, dst.Root())] = &Metadata{
Size: size,
Chunks: Chunks{{0, size}},
Chunks: chunk.Chunks{{0, size}},
}
}
return nil
@ -328,13 +329,21 @@ type Metadata struct {
name string `json:"-"`
Size int64 `json:"s"`
Atime int64 `json:"a"`
Chunks Chunks `json:"c"`
Chunks chunk.Chunks `json:"c"`
}
func (md *Metadata) Close() error {
md.mu.Lock()
defer md.mu.Unlock()
err := md.f.Close()
md.f = nil
return err
}
func (md *Metadata) Delete() error {
md.mu.Lock()
defer md.mu.Unlock()
md.Chunks = Chunks{}
md.Chunks = chunk.Chunks{}
md.f.Close()
md.f = nil
atomic.StoreInt64(&md.Atime, now())

2
pkg/fs/preload.go
View file

@ -130,8 +130,6 @@ func (ph *PreloadHandler) preload(ctx context.Context) {
p.cancel = cancel
ph.mu.Unlock()
go f.Preload(ctx, func() {
file := f
file.Close()
ph.fin <- name
})
}

92
pkg/provider/crypto/crypto.go Normal file
View file

@ -0,0 +1,92 @@
package crypto
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/sha256"
"io"
"golang.org/x/crypto/chacha20poly1305"
"golang.org/x/crypto/hkdf"
)
const (
KDFNonceSize = 16
NonceSize = chacha20poly1305.NonceSize
Overhead = chacha20poly1305.Overhead
ChunkSize = 64 * 1024
FullSize = ChunkSize + Overhead
)
const (
infoCBC = "AES-CBC filename encryption key"
infoCBCIV = "AES-CBC filename encryption initialization vector"
infoChaPloy = "ChaCha20-Poly1305 file content encryption"
)
func cbcKey(key []byte) []byte {
return deriveKey(key, nil, infoCBC, 32)
}
func cbcIV(key []byte) []byte {
return deriveKey(key, nil, infoCBCIV, 16)
}
func streamKey(key, nonce []byte) []byte {
return deriveKey(key, nonce, infoChaPloy, chacha20poly1305.KeySize)
}
func deriveKey(key, nonce []byte, info string, length int) []byte {
h := hkdf.New(sha256.New, key, nonce, []byte(info))
k := make([]byte, length)
if _, err := io.ReadFull(h, k); err != nil {
panic("crypto: internal error: failed to read from HKDF: " + err.Error())
}
return k
}
type aesCbc struct {
key []byte
iv []byte
}
func newAesCbc(key []byte) (*aesCbc, error) {
cbc := &aesCbc{
key: cbcKey(key),
iv: cbcIV(key),
}
_, err := aes.NewCipher(cbc.key)
if err != nil {
return nil, err
}
return cbc, nil
}
func (cbc *aesCbc) encrypt(p []byte) (c []byte) {
block, _ := aes.NewCipher(cbc.key)
mode := cipher.NewCBCEncrypter(block, cbc.iv)
padded := padPKCS5(p, block.BlockSize())
c = make([]byte, len(padded))
mode.CryptBlocks(c, padded)
return
}
func (cbc *aesCbc) decrypt(c []byte) []byte {
block, _ := aes.NewCipher(cbc.key)
mode := cipher.NewCBCDecrypter(block, cbc.iv)
p := make([]byte, len(c))
mode.CryptBlocks(p, c)
return trimPKCS5(p)
}
func padPKCS5(b []byte, bs int) []byte {
n := bs - len(b)%bs
pad := bytes.Repeat([]byte{byte(n)}, n)
return append(b, pad...)
}
func trimPKCS5(b []byte) []byte {
pad := b[len(b)-1]
return b[:len(b)-int(pad)]
}

186
pkg/provider/crypto/file.go Normal file
View file

@ -0,0 +1,186 @@
package crypto
import (
"cachefs/pkg/provider"
"io"
"io/fs"
)
var _ provider.File = (*file)(nil)
type file struct {
provider.File
fs *FS
r *reader
w *writer
nonce [NonceSize]byte
offset int64
name string
}
func newFile(key []byte, name string, fs *FS, f provider.File) (ef provider.File, err error) {
nonce, err := initNonce(f)
if err != nil {
return
}
skey := streamKey(key, nonce)
w, err := newWriter(skey, f)
if err != nil {
return
}
r, err := newReader(skey, f)
if err != nil {
return
}
return &file{
File: f,
fs: fs,
w: w,
r: r,
name: name,
}, nil
}
func (f *file) Stat() (fs.FileInfo, error) {
fi, err := f.File.Stat()
return stat{FileInfo: fi, name: f.name}, err
}
func (f *file) Name() string {
return f.name
}
func (file) Fd() uintptr {
return 0
}
type stat struct {
fs.FileInfo
name string
}
func (s stat) Size() int64 {
return RealSize(s.FileInfo.Size())
}
func (s stat) Name() string {
return s.name
}
func (f *file) Close() error {
e1 := f.w.Close()
e2 := f.File.Close()
if e1 != nil {
return e1
}
return e2
}
func (f *file) Seek(offset int64, whence int) (n int64, err error) {
var cn, off, roff int64
switch whence {
case io.SeekStart:
cn, off, roff = align(offset)
f.offset = KDFNonceSize + off
n, err = f.File.Seek(f.offset, whence)
case io.SeekCurrent:
if f.offset == 0 {
f.offset = KDFNonceSize
}
cn, off, roff = align(f.offset + f.r.off + offset)
f.offset = off
_, err = f.File.Seek(f.offset, io.SeekStart)
n = offset
}
f.nonce = [NonceSize]byte{}
inc(&f.nonce, cn)
f.r.nonce = f.nonce
f.r.off = roff
f.r.err = nil
f.w.nonce = f.nonce
f.r.unread = f.r.unread[:0]
return
}
func (f *file) Read(p []byte) (n int, err error) {
n, err = f.r.Read(p)
return
}
func (f *file) Write(p []byte) (n int, err error) {
n, err = f.w.Write(p)
return
}
func (f *file) ReadAt(p []byte, pos int64) (n int, err error) {
_, _, roff := align(pos)
//cn, off, roff := align(pos)
f.Seek(pos, io.SeekStart)
//fmt.Println("readAt:", cn, off+KDFNonceSize, roff, pos)
_, err = f.r.readChunk()
if err != nil {
return
}
n = copy(p, f.r.unread[roff:])
return
}
func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
cn, off, woff := align(pos)
//fmt.Println("writeAt:", cn, off+KDFNonceSize, woff, pos)
c, ok := f.w.cm[cn]
if !ok {
c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
inc(&c.nonce, cn)
f.w.cm[cn] = c
if f.w.last == nil {
f.w.last = c
} else if cn > f.w.last.cn {
f.w.last = c
}
}
n, err = c.writeAt(data, woff)
if c.full() {
_, err = c.flush(notLastChunk)
//fmt.Println("flush:", cn)
delete(f.w.cm, cn)
}
return
}
func (f *file) Readdir(n int) ([]fs.FileInfo, error) {
fis, err := f.File.Readdir(n)
if err != nil {
return nil, err
}
fio := make([]fs.FileInfo, len(fis))
for i, fi := range fis {
p, err := f.fs.decPath(fi.Name())
if err != nil {
continue
}
fio[i] = dir{FileInfo: fi, name: p}
}
return fio, nil
}
func (f *file) ReadDir(n int) ([]fs.DirEntry, error) {
fis, err := f.Readdir(n)
if err != nil {
return nil, err
}
fio := make([]fs.DirEntry, len(fis))
for i, fi := range fis {
fio[i] = fs.FileInfoToDirEntry(fi)
}
return fio, nil
}
type dir struct {
fs.FileInfo
name string
}
func (d dir) Name() string {
return d.name
}

129
pkg/provider/crypto/fs.go Normal file
View file

@ -0,0 +1,129 @@
package crypto
import (
"cachefs/pkg/provider"
"encoding/base64"
"errors"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
)
var _ provider.FS = (*FS)(nil)
type FS struct {
fs provider.FS
cbc *aesCbc
key []byte
}
func NewFS(fs provider.FS, key []byte) (*FS, error) {
if len(key) != 32 {
return nil, errors.New("encryption key length must be 32 bytes")
}
cbc, err := newAesCbc(key)
if err != nil {
return nil, err
}
return &FS{
fs: fs,
cbc: cbc,
key: key,
}, nil
}
func (fs *FS) encPath(path string) (n string, err error) {
parts := strings.Split(path, "/")
cparts := make([]string, len(parts))
for i, p := range parts {
if p == "" {
continue
}
cparts[i] = base64.RawURLEncoding.EncodeToString(
fs.cbc.encrypt([]byte(p)),
)
}
n = filepath.Join(cparts...)
return
}
func (fs *FS) decPath(path string) (n string, err error) {
parts := strings.Split(path, "/")
pparts := make([]string, len(parts))
for i, p := range parts {
pparts[i] = base64.RawURLEncoding.EncodeToString(
fs.cbc.decrypt([]byte(p)),
)
}
n = filepath.Join(pparts...)
return
}
func (fs *FS) name(p string) string {
return filepath.Join(fs.Root(), p)
}
func (fs *FS) Stat(p string) (fs.FileInfo, error) {
c, err := fs.encPath(p)
if err != nil {
return nil, err
}
fi, err := fs.fs.Stat(c)
return stat{FileInfo: fi, name: fs.name(p)}, err
}
func (fs *FS) Remove(p string) error {
c, err := fs.encPath(p)
if err != nil {
return err
}
return fs.fs.Remove(c)
}
func (fs *FS) Open(p string) (provider.File, error) {
c, err := fs.encPath(p)
if err != nil {
return nil, err
}
f, err := fs.fs.Open(c)
if err != nil {
return nil, err
}
return newFile(fs.key, p, fs, f)
}
func (fs *FS) OpenFile(p string, flags int, mode os.FileMode) (provider.File, error) {
c, err := fs.encPath(p)
if err != nil {
return nil, err
}
f, err := fs.fs.OpenFile(c, flags, mode)
if err != nil {
return nil, err
}
return newFile(fs.key, p, fs, f)
}
func (fs *FS) MkdirAll(p string, mode os.FileMode) error {
c, err := fs.encPath(p)
if err != nil {
return err
}
return fs.fs.MkdirAll(c, mode)
}
func (fs *FS) Root() string {
return fs.fs.Root()
}
func (fs *FS) Fstat(_ uintptr) (int64, error) {
return 0, provider.ErrFstat
}
func (fs *FS) Close() error {
fs.key = nil
runtime.GC()
return nil
}

78
pkg/provider/crypto/helper.go Normal file
View file

@ -0,0 +1,78 @@
package crypto
import (
"cachefs/pkg/provider"
"crypto/rand"
"io"
"math"
)
func isZero(nonce []byte) bool {
for i := 0; i < len(nonce); i++ {
if nonce[i] != 0 {
return false
}
}
return true
}
func initNonce(f provider.File) (nonce []byte, err error) {
nonce = make([]byte, KDFNonceSize)
n, err := io.ReadFull(f, nonce)
if err != nil || n != len(nonce) || isZero(nonce) {
_, err = io.ReadFull(rand.Reader, nonce)
if err != nil {
return
}
_, err = f.Seek(0, io.SeekStart)
if err != nil {
return
}
_, err = f.Write(nonce)
return
}
return
}
func incNonce(nonce *[NonceSize]byte) {
for i := len(nonce) - 2; i >= 0; i-- {
nonce[i]++
if nonce[i] != 0 {
break
} else if i == 0 {
// The counter is 88 bits, this is unreachable.
panic("stream: chunk counter wrapped around")
}
}
}
func Size(s int64) int64 {
return KDFNonceSize + s + (int64(math.Ceil(float64(s)/ChunkSize)) * Overhead)
}
func RealSize(s int64) int64 {
s -= KDFNonceSize
return s - (((s / FullSize) + 1) * Overhead)
}
func chunkNum(o int64) int64 {
return int64(math.Ceil(float64(o) / ChunkSize))
}
func align(o int64) (cn, off, roff int64) {
cn = chunkNum(o)
off = cn * FullSize
roff = o - (cn * ChunkSize)
if roff < 0 {
cn--
off = cn * FullSize
roff = o - (cn * ChunkSize)
}
return
}
func inc(nonce *[NonceSize]byte, n int64) {
for i := int64(0); i < n; i++ {
incNonce(nonce)
}
}

107
pkg/provider/crypto/reader.go Normal file
View file

@ -0,0 +1,107 @@
package crypto
import (
"crypto/cipher"
"errors"
"io"
"golang.org/x/crypto/chacha20poly1305"
)
type cryptoReader interface {
io.Reader
io.ReaderAt
}
type reader struct {
a cipher.AEAD
src cryptoReader
unread []byte // decrypted but unread data, backed by buf
buf [FullSize]byte
off int64
err error
nonce [NonceSize]byte
}
func newReader(key []byte, src cryptoReader) (*reader, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
return &reader{
a: aead,
src: src,
}, nil
}
func (r *reader) Read(p []byte) (int, error) {
if len(r.unread) > 0 {
n := copy(p, r.unread)
r.unread = r.unread[n:]
return n, nil
}
if r.err != nil {
return 0, r.err
}
if len(p) == 0 {
return 0, nil
}
last, err := r.readChunk()
if err != nil {
r.err = err
return 0, err
}
if r.off > 0 && len(r.unread) >= int(r.off) {
r.unread = r.unread[r.off:]
r.off = 0
}
n := copy(p, r.unread)
r.unread = r.unread[n:]
if last {
r.err = io.EOF
}
return n, nil
}
func (r *reader) readChunk() (last bool, err error) {
if len(r.unread) != 0 {
panic("stream: internal error: readChunk called with dirty buffer")
}
in := r.buf[:]
n, err := io.ReadFull(r.src, in)
switch {
case err == io.EOF:
// A message can't end without a marked chunk. This message is truncated.
return false, io.ErrUnexpectedEOF
case err == io.ErrUnexpectedEOF:
// The last chunk can be short.
in = in[:n]
last = true
setLastChunkFlag(&r.nonce)
case err != nil:
return false, err
}
outBuf := make([]byte, 0, ChunkSize)
out, err := r.a.Open(outBuf, r.nonce[:], in, nil)
if err != nil && !last {
// Check if this was a full-length final chunk.
last = true
setLastChunkFlag(&r.nonce)
out, err = r.a.Open(outBuf, r.nonce[:], in, nil)
}
if err != nil {
return false, errors.New("failed to decrypt and authenticate payload chunk")
}
incNonce(&r.nonce)
r.unread = r.buf[:copy(r.buf[:], out)]
return last, nil
}

155
pkg/provider/crypto/writer.go Normal file
View file

@ -0,0 +1,155 @@
package crypto
import (
"cachefs/pkg/chunk"
"crypto/cipher"
"errors"
"io"
"golang.org/x/crypto/chacha20poly1305"
)
type cryptoWriter interface {
io.Writer
io.WriterAt
}
const lastChunkFlag = 0x01
func setLastChunkFlag(nonce *[NonceSize]byte) {
nonce[len(nonce)-1] = lastChunkFlag
}
type writer struct {
a cipher.AEAD
dst cryptoWriter
unwritten []byte // backed by buf
buf [FullSize]byte
nonce [NonceSize]byte
err error
cm map[int64]*chunkWriter
last *chunkWriter
flush bool
}
func newWriter(key []byte, dst cryptoWriter) (*writer, error) {
aead, err := chacha20poly1305.New(key)
if err != nil {
return nil, err
}
w := &writer{
a: aead,
dst: dst,
cm: make(map[int64]*chunkWriter),
}
w.unwritten = w.buf[:0]
return w, nil
}
type chunkWriter struct {
offset int64
cn int64
w *writer
buf [ChunkSize]byte
nonce [NonceSize]byte
chunks chunk.Chunks
}
func (c *chunkWriter) writeAt(p []byte, offset int64) (n int, err error) {
n = copy(c.buf[offset:], p)
c.chunks.Add(offset, n)
return
}
func (c *chunkWriter) full() bool {
if len(c.chunks) == 1 {
return c.chunks[0][1] == ChunkSize
}
return false
}
func (c *chunkWriter) flush(last bool) (n int, err error) {
wb := c.buf[:0]
if last {
setLastChunkFlag(&c.nonce)
s := c.chunks.Size()
wb = c.buf[:s]
n = int(s)
} else {
wb = c.buf[:]
n = len(c.buf)
}
buf := c.w.a.Seal(wb[:0], c.nonce[:], wb, nil)
_, err = c.w.dst.WriteAt(buf, c.offset)
return
}
func (w *writer) Write(p []byte) (n int, err error) {
// TODO: consider refactoring with a bytes.Buffer.
if w.err != nil {
return 0, w.err
}
if len(p) == 0 {
return 0, nil
}
total := len(p)
for len(p) > 0 {
freeBuf := w.buf[len(w.unwritten):ChunkSize]
n := copy(freeBuf, p)
p = p[n:]
w.unwritten = w.unwritten[:len(w.unwritten)+n]
w.flush = true
if len(w.unwritten) == ChunkSize && len(p) > 0 {
if err := w.flushChunk(notLastChunk); err != nil {
w.err = err
return 0, err
}
}
}
return total, nil
}
func (w *writer) Close() error {
if w.err != nil {
return w.err
}
if w.last != nil {
_, w.err = w.last.flush(lastChunk)
if w.err != nil {
return w.err
}
}
if w.flush {
w.err = w.flushChunk(lastChunk)
if w.err != nil {
return w.err
}
}
w.err = errors.New("stream.Writer is already closed")
return nil
}
const (
lastChunk = true
notLastChunk = false
)
func (w *writer) flushChunk(last bool) error {
if !last && len(w.unwritten) != ChunkSize {
panic("stream: internal error: flush called with partial chunk")
}
if last {
setLastChunkFlag(&w.nonce)
}
buf := w.a.Seal(w.buf[:0], w.nonce[:], w.unwritten, nil)
_, err := w.dst.Write(buf)
w.unwritten = w.buf[:0]
incNonce(&w.nonce)
return err
}

12
pkg/provider/interface.go
View file

@ -4,6 +4,7 @@ package provider
import (
"errors"
"io"
"io/fs"
"os"
)
@ -34,16 +35,17 @@ func WalkFS(fs FS) fs.FS {
}
type File interface {
io.Reader
io.ReaderAt
io.Writer
io.WriterAt
io.Seeker
io.Closer
Name() string
Stat() (fs.FileInfo, error)
Truncate(int64) error
Sync() error
Fd() uintptr
ReadDir(int) ([]fs.DirEntry, error)
Read([]byte) (int, error)
ReadAt([]byte, int64) (int, error)
WriteAt([]byte, int64) (int, error)
Seek(int64, int) (int64, error)
Readdir(int) ([]fs.FileInfo, error)
Close() error
}

2
pkg/provider/mount/mount.go → pkg/provider/os/os.go
View file

@ -1,6 +1,6 @@
// Copyright (C) 2022 Marius Schellenberger
package mount
package os
import (
"cachefs/pkg/provider"

20
pkg/provider/parse/parse.go
View file

@ -4,7 +4,8 @@ package parse
import (
"cachefs/pkg/provider"
"cachefs/pkg/provider/mount"
"cachefs/pkg/provider/crypto"
"cachefs/pkg/provider/os"
"cachefs/pkg/provider/sftp"
"errors"
"fmt"
@ -19,7 +20,7 @@ var (
ErrUnsupportedScheme = errors.New("unsupported scheme")
)
func FS(url string) (provider.FS, error) {
func FS(url string, key []byte) (provider.FS, error) {
u, err := neturl.Parse(url)
if err != nil {
return nil, err
@ -27,11 +28,20 @@ func FS(url string) (provider.FS, error) {
if !filepath.IsAbs(u.Path) {
return nil, ErrPathNotAbsolute
}
var fs provider.FS
switch u.Scheme {
case "file":
return mount.NewFS(u.Path)
fs, err = os.NewFS(u.Path)
case "sftp":
return sftp.NewFS(u, klogr.New().WithName("sftp"))
}
fs, err = sftp.NewFS(u, klogr.New().WithName("sftp"))
default:
return nil, fmt.Errorf("%w: %s", ErrUnsupportedScheme, u.Scheme)
}
if err != nil {
return nil, err
}
if key != nil {
fs, err = crypto.NewFS(fs, key)
}
return fs, err
}

2
pkg/provider/sftp/sftp.go
View file

@ -160,7 +160,7 @@ func (fs *FS) OpenFile(p string, flags int, _ os.FileMode) (provider.File, error
return &file{f, fs}, err
}
func (fs *FS) Fstat(uintptr) (int64, error) {
func (fs *FS) Fstat(_ uintptr) (int64, error) {
return 0, provider.ErrFstat
}

7
run.sh
View file

@ -1,7 +0,0 @@
#!/bin/sh
./cachefs \
-src /home/marius/Desktop \
-dst /home/marius/mirrortmp \
-data /home/marius/mirrortmp/md.json \
-listen :8080 \
-v 2

98
vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305.go generated vendored Normal file
View file

@ -0,0 +1,98 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package chacha20poly1305 implements the ChaCha20-Poly1305 AEAD and its
// extended nonce variant XChaCha20-Poly1305, as specified in RFC 8439 and
// draft-irtf-cfrg-xchacha-01.
package chacha20poly1305 // import "golang.org/x/crypto/chacha20poly1305"
import (
"crypto/cipher"
"errors"
)
const (
// KeySize is the size of the key used by this AEAD, in bytes.
KeySize = 32
// NonceSize is the size of the nonce used with the standard variant of this
// AEAD, in bytes.
//
// Note that this is too short to be safely generated at random if the same
// key is reused more than 2³² times.
NonceSize = 12
// NonceSizeX is the size of the nonce used with the XChaCha20-Poly1305
// variant of this AEAD, in bytes.
NonceSizeX = 24
// Overhead is the size of the Poly1305 authentication tag, and the
// difference between a ciphertext length and its plaintext.
Overhead = 16
)
type chacha20poly1305 struct {
key [KeySize]byte
}
// New returns a ChaCha20-Poly1305 AEAD that uses the given 256-bit key.
func New(key []byte) (cipher.AEAD, error) {
if len(key) != KeySize {
return nil, errors.New("chacha20poly1305: bad key length")
}
ret := new(chacha20poly1305)
copy(ret.key[:], key)
return ret, nil
}
func (c *chacha20poly1305) NonceSize() int {
return NonceSize
}
func (c *chacha20poly1305) Overhead() int {
return Overhead
}
func (c *chacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
if len(nonce) != NonceSize {
panic("chacha20poly1305: bad nonce length passed to Seal")
}
if uint64(len(plaintext)) > (1<<38)-64 {
panic("chacha20poly1305: plaintext too large")
}
return c.seal(dst, nonce, plaintext, additionalData)
}
var errOpen = errors.New("chacha20poly1305: message authentication failed")
func (c *chacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
if len(nonce) != NonceSize {
panic("chacha20poly1305: bad nonce length passed to Open")
}
if len(ciphertext) < 16 {
return nil, errOpen
}
if uint64(len(ciphertext)) > (1<<38)-48 {
panic("chacha20poly1305: ciphertext too large")
}
return c.open(dst, nonce, ciphertext, additionalData)
}
// sliceForAppend takes a slice and a requested number of bytes. It returns a
// slice with the contents of the given slice followed by that many bytes and a
// second slice that aliases into it and contains only the extra bytes. If the
// original slice has sufficient capacity then no allocation is performed.
func sliceForAppend(in []byte, n int) (head, tail []byte) {
if total := len(in) + n; cap(in) >= total {
head = in[:total]
} else {
head = make([]byte, total)
copy(head, in)
}
tail = head[len(in):]
return
}

87
vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go generated vendored Normal file
View file

@ -0,0 +1,87 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build gc && !purego
// +build gc,!purego
package chacha20poly1305
import (
"encoding/binary"
"golang.org/x/crypto/internal/subtle"
"golang.org/x/sys/cpu"
)
//go:noescape
func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool
//go:noescape
func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte)
var (
useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2
)
// setupState writes a ChaCha20 input matrix to state. See
// https://tools.ietf.org/html/rfc7539#section-2.3.
func setupState(state *[16]uint32, key *[32]byte, nonce []byte) {
state[0] = 0x61707865
state[1] = 0x3320646e
state[2] = 0x79622d32
state[3] = 0x6b206574
state[4] = binary.LittleEndian.Uint32(key[0:4])
state[5] = binary.LittleEndian.Uint32(key[4:8])
state[6] = binary.LittleEndian.Uint32(key[8:12])
state[7] = binary.LittleEndian.Uint32(key[12:16])
state[8] = binary.LittleEndian.Uint32(key[16:20])
state[9] = binary.LittleEndian.Uint32(key[20:24])
state[10] = binary.LittleEndian.Uint32(key[24:28])
state[11] = binary.LittleEndian.Uint32(key[28:32])
state[12] = 0
state[13] = binary.LittleEndian.Uint32(nonce[0:4])
state[14] = binary.LittleEndian.Uint32(nonce[4:8])
state[15] = binary.LittleEndian.Uint32(nonce[8:12])
}
func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
if !cpu.X86.HasSSSE3 {
return c.sealGeneric(dst, nonce, plaintext, additionalData)
}
var state [16]uint32
setupState(&state, &c.key, nonce)
ret, out := sliceForAppend(dst, len(plaintext)+16)
if subtle.InexactOverlap(out, plaintext) {
panic("chacha20poly1305: invalid buffer overlap")
}
chacha20Poly1305Seal(out[:], state[:], plaintext, additionalData)
return ret
}
func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
if !cpu.X86.HasSSSE3 {
return c.openGeneric(dst, nonce, ciphertext, additionalData)
}
var state [16]uint32
setupState(&state, &c.key, nonce)
ciphertext = ciphertext[:len(ciphertext)-16]
ret, out := sliceForAppend(dst, len(ciphertext))
if subtle.InexactOverlap(out, ciphertext) {
panic("chacha20poly1305: invalid buffer overlap")
}
if !chacha20Poly1305Open(out, state[:], ciphertext, additionalData) {
for i := range out {
out[i] = 0
}
return nil, errOpen
}
return ret, nil
}

2696
vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s generated vendored Normal file
View file

File diff suppressed because it is too large Load diff

81
vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_generic.go generated vendored Normal file
View file

@ -0,0 +1,81 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package chacha20poly1305
import (
"encoding/binary"
"golang.org/x/crypto/chacha20"
"golang.org/x/crypto/internal/poly1305"
"golang.org/x/crypto/internal/subtle"
)
func writeWithPadding(p *poly1305.MAC, b []byte) {
p.Write(b)
if rem := len(b) % 16; rem != 0 {
var buf [16]byte
padLen := 16 - rem
p.Write(buf[:padLen])
}
}
func writeUint64(p *poly1305.MAC, n int) {
var buf [8]byte
binary.LittleEndian.PutUint64(buf[:], uint64(n))
p.Write(buf[:])
}
func (c *chacha20poly1305) sealGeneric(dst, nonce, plaintext, additionalData []byte) []byte {
ret, out := sliceForAppend(dst, len(plaintext)+poly1305.TagSize)
ciphertext, tag := out[:len(plaintext)], out[len(plaintext):]
if subtle.InexactOverlap(out, plaintext) {
panic("chacha20poly1305: invalid buffer overlap")
}
var polyKey [32]byte
s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
s.XORKeyStream(polyKey[:], polyKey[:])
s.SetCounter(1) // set the counter to 1, skipping 32 bytes
s.XORKeyStream(ciphertext, plaintext)
p := poly1305.New(&polyKey)
writeWithPadding(p, additionalData)
writeWithPadding(p, ciphertext)
writeUint64(p, len(additionalData))
writeUint64(p, len(plaintext))
p.Sum(tag[:0])
return ret
}
func (c *chacha20poly1305) openGeneric(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
tag := ciphertext[len(ciphertext)-16:]
ciphertext = ciphertext[:len(ciphertext)-16]
var polyKey [32]byte
s, _ := chacha20.NewUnauthenticatedCipher(c.key[:], nonce)
s.XORKeyStream(polyKey[:], polyKey[:])
s.SetCounter(1) // set the counter to 1, skipping 32 bytes
p := poly1305.New(&polyKey)
writeWithPadding(p, additionalData)
writeWithPadding(p, ciphertext)
writeUint64(p, len(additionalData))
writeUint64(p, len(ciphertext))
ret, out := sliceForAppend(dst, len(ciphertext))
if subtle.InexactOverlap(out, ciphertext) {
panic("chacha20poly1305: invalid buffer overlap")
}
if !p.Verify(tag) {
for i := range out {
out[i] = 0
}
return nil, errOpen
}
s.XORKeyStream(out, ciphertext)
return ret, nil
}

16
vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_noasm.go generated vendored Normal file
View file

@ -0,0 +1,16 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !amd64 || !gc || purego
// +build !amd64 !gc purego
package chacha20poly1305
func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
return c.sealGeneric(dst, nonce, plaintext, additionalData)
}
func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
return c.openGeneric(dst, nonce, ciphertext, additionalData)
}

86
vendor/golang.org/x/crypto/chacha20poly1305/xchacha20poly1305.go generated vendored Normal file
View file

@ -0,0 +1,86 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package chacha20poly1305
import (
"crypto/cipher"
"errors"
"golang.org/x/crypto/chacha20"
)
type xchacha20poly1305 struct {
key [KeySize]byte
}
// NewX returns a XChaCha20-Poly1305 AEAD that uses the given 256-bit key.
//
// XChaCha20-Poly1305 is a ChaCha20-Poly1305 variant that takes a longer nonce,
// suitable to be generated randomly without risk of collisions. It should be
// preferred when nonce uniqueness cannot be trivially ensured, or whenever
// nonces are randomly generated.
func NewX(key []byte) (cipher.AEAD, error) {
if len(key) != KeySize {
return nil, errors.New("chacha20poly1305: bad key length")
}
ret := new(xchacha20poly1305)
copy(ret.key[:], key)
return ret, nil
}
func (*xchacha20poly1305) NonceSize() int {
return NonceSizeX
}
func (*xchacha20poly1305) Overhead() int {
return Overhead
}
func (x *xchacha20poly1305) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
if len(nonce) != NonceSizeX {
panic("chacha20poly1305: bad nonce length passed to Seal")
}
// XChaCha20-Poly1305 technically supports a 64-bit counter, so there is no
// size limit. However, since we reuse the ChaCha20-Poly1305 implementation,
// the second half of the counter is not available. This is unlikely to be
// an issue because the cipher.AEAD API requires the entire message to be in
// memory, and the counter overflows at 256 GB.
if uint64(len(plaintext)) > (1<<38)-64 {
panic("chacha20poly1305: plaintext too large")
}
c := new(chacha20poly1305)
hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16])
copy(c.key[:], hKey)
// The first 4 bytes of the final nonce are unused counter space.
cNonce := make([]byte, NonceSize)
copy(cNonce[4:12], nonce[16:24])
return c.seal(dst, cNonce[:], plaintext, additionalData)
}
func (x *xchacha20poly1305) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
if len(nonce) != NonceSizeX {
panic("chacha20poly1305: bad nonce length passed to Open")
}
if len(ciphertext) < 16 {
return nil, errOpen
}
if uint64(len(ciphertext)) > (1<<38)-48 {
panic("chacha20poly1305: ciphertext too large")
}
c := new(chacha20poly1305)
hKey, _ := chacha20.HChaCha20(x.key[:], nonce[0:16])
copy(c.key[:], hKey)
// The first 4 bytes of the final nonce are unused counter space.
cNonce := make([]byte, NonceSize)
copy(cNonce[4:12], nonce[16:24])
return c.open(dst, cNonce[:], ciphertext, additionalData)
}

93
vendor/golang.org/x/crypto/hkdf/hkdf.go generated vendored Normal file
View file

@ -0,0 +1,93 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package hkdf implements the HMAC-based Extract-and-Expand Key Derivation
// Function (HKDF) as defined in RFC 5869.
//
// HKDF is a cryptographic key derivation function (KDF) with the goal of
// expanding limited input keying material into one or more cryptographically
// strong secret keys.
package hkdf // import "golang.org/x/crypto/hkdf"
import (
"crypto/hmac"
"errors"
"hash"
"io"
)
// Extract generates a pseudorandom key for use with Expand from an input secret
// and an optional independent salt.
//
// Only use this function if you need to reuse the extracted key with multiple
// Expand invocations and different context values. Most common scenarios,
// including the generation of multiple keys, should use New instead.
func Extract(hash func() hash.Hash, secret, salt []byte) []byte {
if salt == nil {
salt = make([]byte, hash().Size())
}
extractor := hmac.New(hash, salt)
extractor.Write(secret)
return extractor.Sum(nil)
}
type hkdf struct {
expander hash.Hash
size int
info []byte
counter byte
prev []byte
buf []byte
}
func (f *hkdf) Read(p []byte) (int, error) {
// Check whether enough data can be generated
need := len(p)
remains := len(f.buf) + int(255-f.counter+1)*f.size
if remains < need {
return 0, errors.New("hkdf: entropy limit reached")
}
// Read any leftover from the buffer
n := copy(p, f.buf)
p = p[n:]
// Fill the rest of the buffer
for len(p) > 0 {
f.expander.Reset()
f.expander.Write(f.prev)
f.expander.Write(f.info)
f.expander.Write([]byte{f.counter})
f.prev = f.expander.Sum(f.prev[:0])
f.counter++
// Copy the new batch into p
f.buf = f.prev
n = copy(p, f.buf)
p = p[n:]
}
// Save leftovers for next run
f.buf = f.buf[n:]
return need, nil
}
// Expand returns a Reader, from which keys can be read, using the given
// pseudorandom key and optional context info, skipping the extraction step.
//
// The pseudorandomKey should have been generated by Extract, or be a uniformly
// random or pseudorandom cryptographically strong key. See RFC 5869, Section
// 3.3. Most common scenarios will want to use New instead.
func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader {
expander := hmac.New(hash, pseudorandomKey)
return &hkdf{expander, expander.Size(), info, 1, nil, nil}
}
// New returns a Reader, from which keys can be read, using the given hash,
// secret, salt and context info. Salt and info can be nil.
func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader {
prk := Extract(hash, secret, salt)
return Expand(hash, prk, info)
}

2
vendor/modules.txt vendored
View file

@ -12,9 +12,11 @@ github.com/pkg/sftp/internal/encoding/ssh/filexfer
## explicit; go 1.17
golang.org/x/crypto/blowfish
golang.org/x/crypto/chacha20
golang.org/x/crypto/chacha20poly1305
golang.org/x/crypto/curve25519
golang.org/x/crypto/curve25519/internal/field
golang.org/x/crypto/ed25519
golang.org/x/crypto/hkdf
golang.org/x/crypto/internal/poly1305
golang.org/x/crypto/internal/subtle
golang.org/x/crypto/ssh