small fixes and dependency updates

Makefile

@@ -9,7 +9,7 @@ MD_PROGRAM=metadatagen
 MD_CMD=-o $(MD_PROGRAM) cmd/$(MD_PROGRAM)/main.go
 CTL_PROGRAM=cachectl
 CTL_CMD=-o $(CTL_PROGRAM) cmd/$(CTL_PROGRAM)/main.go
-ENV=CGO_ENABLED=0 GO111MODULE=on
+ENV=GOGC=0 CGO_ENABLED=0 GO111MODULE=on
 
 all: $(PROGRAM) $(MD_PROGRAM) $(CTL_PROGRAM)
 
@@ -46,6 +46,9 @@ codeqa: govet misspell test
 govet:
 	$(CC) vet ./...
 
+update:
+	$(ENV) $(CC) get -u ./...
+
 misspell:
 	$(GOPATH)/bin/misspell cmd/* pkg/* Makefile README.md
 
@@ -63,4 +66,4 @@ else
 	$(info skipping tests of other platforms)
 endif
 
-.PHONY: setup release release-vendor vendor build clean codeqa gofmt govet misspell staticcheck test
+.PHONY: setup release release-vendor vendor build clean codeqa gofmt govet update misspell staticcheck test

TODO.txt

@@ -2,11 +2,14 @@
   * printf '\x6c' | dd seek=20 bs=1 count=1 conv=notrunc of=test.dump
 
 * crypto
-  * flush last chunk in streaming mode
+  * flush last chunk in streaming mode - check if implemented
+
+* web
+  * listing of all files in cache
+  * add breadcrumb
+
+* tuning
+  * debug logger for writer queue size
 
 * sftp
   * implement read ahead buffer?
-
-* add breadcrumb
-* config file?
-  * whitelist allowed src paths

cmd/cachectl/main.go

@@ -111,7 +111,7 @@ func ls(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	fs, err := parse.FS(args[0], k, true)
+	fs, err := parse.FS(args[0], k, nil, true)
 	if err != nil {
 		return
 	}
@@ -153,12 +153,12 @@ func cp(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	sfs, err := parse.FS(src, sk, true)
+	sfs, err := parse.FS(src, sk, nil, true)
 	if err != nil {
 		return
 	}
 	defer sfs.Close()
-	dfs, err := parse.FS(dst, dk, true)
+	dfs, err := parse.FS(dst, dk, nil, true)
 	if err != nil {
 		return
 	}
@@ -248,7 +248,7 @@ func rm(args []string) (err error) {
 	if err != nil {
 		return
 	}
-	fs, err := parse.FS(src, k, true)
+	fs, err := parse.FS(src, k, nil, true)
 	if err != nil {
 		return
 	}

cmd/cachefs/main.go

@@ -7,15 +7,17 @@ import (
 	"flag"
 	"net/http"
 
-	//_ "net/http/pprof"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"cachefs/pkg/config"
 	"cachefs/pkg/fs"
+	"cachefs/pkg/provider/parse"
 	"cachefs/pkg/srv"
 
+	"git.giftfish.de/ston1th/godrop/v2"
 	"github.com/go-logr/logr"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
@@ -23,71 +25,47 @@ import (
 
 var (
 	version string
-
+	cfg     string
-	src          string
-	dst          string
-	metadata     string
-	listenHttp   string
-	listenWebdav string
-	listenCache  string
-	quota        int64
-	max          int
-	bs           int
-	block        bool
 
 	log logr.Logger
 )
 
-const gib = 1024 * 1024 * 1024
-
 func main() {
 	klog.InitFlags(nil)
-	flag.StringVar(&src, "src", "", "url path to source files (example: file:///mnt/nfs)")
+	flag.StringVar(&cfg, "config", "", "path to config file")
-	flag.StringVar(&dst, "dst", "", "url path to cache files (example: file:///mnt/cache)")
-	flag.StringVar(&metadata, "data", "", "path to metadata file")
-	flag.StringVar(&listenHttp, "listen", "127.0.0.1:8080", "listen addr:port")
-	flag.StringVar(&listenWebdav, "webdav", "", "listen addr:port for webdav")
-	flag.StringVar(&listenCache, "cache", "", "listen addr:port for cache only")
-	flag.IntVar(&max, "max", 1, "max parallel preloads")
-	flag.IntVar(&bs, "bs", -1, "tune preload buffer size in bytes (default: 8192)")
-	flag.Int64Var(&quota, "quota", 1, "max disk usage quota for the dst cache in GiB")
-	flag.BoolVar(&block, "block", true, "block until sftp is connected")
 	flag.Parse()
-	//go func() {
-	//	http.ListenAndServe("127.0.0.1:7777", nil)
-	//}()
 
 	log = klogr.New().WithName("main")
 	log.Info("starting cachefs", "version", version)
 
-	if fs.SetBufferSize(bs) {
+	c, err := config.ParseFile(cfg)
-		log.V(2).Info("changed preload buffer size", "size", bs)
+	if err != nil {
+		klog.Fatalf("init failed: %s", err)
 	}
 
-	filesystem, err := fs.NewFS(
+	err = dropPrivs(c)
-		quota*gib,
+	if err != nil {
-		max,
+		klog.Fatalf("init failed: %s", err)
-		src,
+	}
-		os.Getenv("SRC_KEY"),
+
-		dst,
+	if fs.SetBufferSize(c.Cache.Buffer) {
-		os.Getenv("DST_KEY"),
+		log.V(2).Info("changed preload buffer size", "size", c.Cache.Buffer)
-		metadata,
+	}
-		block,
+
-		klogr.New().WithName("fs"),
+	filesystem, err := fs.NewFSFromConfig(c, klogr.New().WithName("fs"))
-	)
 	if err != nil {
 		klog.Fatalf("init failed: %s", err)
 	}
 	slog := klogr.New().WithName("srv")
 	s := &http.Server{
-		Addr: listenHttp,
+		Addr: c.Server.HTTP.Addr,
 		Handler: srv.NewFileServer(
 			filesystem,
 			slog,
 		),
 	}
+	log.Info("starting main server", "addr", c.Server.HTTP.Addr)
 	go func() {
-		log.Info("starting main server", "addr", listenHttp)
 		err := s.ListenAndServe()
 		if err != nil && err != http.ErrServerClosed {
 			klog.Fatalf("init failed: %s", err)
@@ -96,16 +74,18 @@ func main() {
 	var (
 		dav   *http.Server
 		cache *http.Server
+		plain *http.Server
 	)
-	if listenWebdav != "" {
+	davAddr := c.Server.WebDav.Addr
-		log.Info("starting webdav server", "addr", listenWebdav)
+	if davAddr != "" {
 		dav = &http.Server{
-			Addr: listenWebdav,
+			Addr: davAddr,
 			Handler: srv.NewWebDavServer(
 				fs.NewWebDavFS(filesystem),
 				slog.WithName("webdav"),
 			),
 		}
+		log.Info("starting webdav server", "addr", davAddr)
 		go func() {
 			err := dav.ListenAndServe()
 			if err != nil && err != http.ErrServerClosed {
@@ -113,15 +93,16 @@ func main() {
 			}
 		}()
 	}
-	if listenCache != "" {
+	cacheAddr := c.Server.Cache.Addr
-		log.Info("starting cache server", "addr", listenCache)
+	if cacheAddr != "" {
 		cache = &http.Server{
-			Addr: listenCache,
+			Addr: cacheAddr,
 			Handler: srv.NewCacheServer(
 				filesystem,
 				slog.WithName("cache"),
 			),
 		}
+		log.Info("starting cache server", "addr", cacheAddr)
 		go func() {
 			err := cache.ListenAndServe()
 			if err != nil && err != http.ErrServerClosed {
@@ -129,6 +110,23 @@ func main() {
 			}
 		}()
 	}
+	plainAddr := c.Server.Plain.Addr
+	if plainAddr != "" {
+		plain = &http.Server{
+			Addr: plainAddr,
+			Handler: srv.NewPlainServer(
+				filesystem,
+				slog.WithName("plain"),
+			),
+		}
+		log.Info("starting plain server", "addr", plainAddr)
+		go func() {
+			err := plain.ListenAndServe()
+			if err != nil && err != http.ErrServerClosed {
+				klog.Fatalf("init failed: %s", err)
+			}
+		}()
+	}
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	<-sigs
@@ -146,7 +144,42 @@ func main() {
 		cache.Shutdown(ctx)
 		cancel()
 	}
+	if plain != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		plain.Shutdown(ctx)
+		cancel()
+	}
 	filesystem.Close()
+	<-filesystem.Done()
 	log.Info("cachefs shutdown completed")
 	os.Exit(0)
 }
+
+func dropPrivs(cfg *config.Config) (err error) {
+	// flock fattr
+	err = godrop.PledgePromises("stdio rpath wpath cpath inet dns unveil")
+	if err != nil {
+		return
+	}
+	err = godrop.Unveil(cfg.Cache.Metadata, "rwc")
+	if err != nil {
+		return
+	}
+	for _, p := range parse.Paths(cfg.Cache.Src.Path) {
+		err = godrop.Unveil(p, "r")
+		if err != nil {
+			return
+		}
+	}
+	for i, p := range parse.Paths(cfg.Cache.Dst.Path) {
+		if i == 0 {
+			err = godrop.Unveil(p, "rwc")
+		} else {
+			err = godrop.Unveil(p, "r")
+		}
+		if err != nil {
+			return
+		}
+	}
+	return godrop.UnveilBlock()
+}

cmd/metadatagen/main.go

@@ -44,7 +44,7 @@ func main() {
 		}
 		encrypted = true
 	}
-	dstfs, err := parse.FS(dst, dstk, true)
+	dstfs, err := parse.FS(dst, dstk, nil, true)
 	if err != nil {
 		fatal(err)
 	}

config.yaml

@@ -0,0 +1,40 @@
+server:
+# listen addr:port for default http interface
+  http:
+    addr: "127.0.0.1:8080"
+# listen addr:port for webdav
+  #webdav:
+  #  addr: ""
+# listen addr:port for cache only
+  #cache:
+  #  addr: ""
+# listen addr:port for plain directory listings over http
+  #plain:
+  #  addr: ""
+
+# regex whitelist for allowed filesystem paths
+#path: ""
+
+cache:
+# path to metadata file
+  metadata: "/path/to/metadata.json"
+  src:
+# url path to source files
+    path: "file:///mnt/nfs"
+    #path: "sftp://user@src-host:/home/user"
+# optional hex encoded encryption key (32 bytes)
+    #key: ""
+  dst:
+# url path to cache files
+    path: "file:///mnt/cache"
+    #path: "sftp://user@dst-host:/home/user"
+# optional hex encoded encryption key (32 bytes)
+    #key: ""
+# max parallel preloads
+  #preloads: 1
+# preload buffer size in bytes
+  #buffer: 8192
+# max disk usage quota for the dst cache in GiB
+  #quota: 1
+# block until sftp is connected
+  #blockSFTP: true

go.mod

@@ -1,15 +1,16 @@
 module cachefs
 
-go 1.18
+go 1.21
 
 require (
-	github.com/go-logr/logr v1.2.2
+	git.giftfish.de/ston1th/godrop/v2 v2.1.1
-	github.com/pkg/sftp v1.13.4
+	github.com/go-logr/logr v1.2.4
-	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29
+	github.com/pkg/sftp v1.13.6
-	golang.org/x/exp v0.0.0-20220314205449-43aec2f8a4e7
+	golang.org/x/crypto v0.12.0
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
+	golang.org/x/net v0.14.0
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sys v0.11.0
-	k8s.io/klog/v2 v2.40.1
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/klog/v2 v2.100.1
 )
 
 require github.com/kr/fs v0.1.0 // indirect

go.sum

@@ -1,35 +1,62 @@
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+git.giftfish.de/ston1th/godrop/v2 v2.1.1 h1:jCzQPFpRImr2N4kz+4lC7wGZJAWGx9pH5X09nUsRQDU=
+git.giftfish.de/ston1th/godrop/v2 v2.1.1/go.mod h1:DWC4iM+u/bpdJsauL00h0V9tZ97d6Bzi2RGaFKlvZoQ=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/pkg/sftp v1.13.4 h1:Lb0RYJCmgUcBgZosfoi9Y9sbl6+LJgOIgk/2Y4YjMFg=
+github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
-github.com/pkg/sftp v1.13.4/go.mod h1:LzqnAvaD5TWeNBsZpfKxSYn1MbjWwOsCIAFFJbpIsK8=
+github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
-golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/exp v0.0.0-20220314205449-43aec2f8a4e7 h1:jynE66seADJbyWMUdeOyVTvPtBZt7L6LJHupGwxPZRM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20220314205449-43aec2f8a4e7/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
+golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=

pkg/chunk/chunk.go

@@ -3,14 +3,15 @@
 package chunk
 
 import (
-	"golang.org/x/exp/slices"
+	"cmp"
+	"slices"
 )
 
 type Chunk [2]int64
 
 type Chunks []Chunk
 
-func (Chunks) Less(i, j Chunk) bool { return i[0] < j[0] }
+func (Chunks) Comp(a, b Chunk) int { return cmp.Compare(a[0], b[0]) }
 
 func (cs Chunks) Exists(off int64, n int, size int64) bool {
 	end := off + int64(n)
@@ -42,7 +43,7 @@ func (cs *Chunks) merge() {
 	if len(c) < 2 {
 		return
 	}
-	slices.SortFunc(c, c.Less)
+	slices.SortFunc(c, c.Comp)
 	for i := 0; i < len(c); i++ {
 		if i+1 == len(c) {
 			break

pkg/config/config.go

@@ -0,0 +1,106 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package config
+
+import (
+	"errors"
+	"os"
+	"regexp"
+
+	"gopkg.in/yaml.v3"
+)
+
+func ParseFile(file string) (cfg *Config, err error) {
+	if file == "" {
+		return nil, errors.New("missing config file")
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	cfg = new(Config)
+	err = yaml.NewDecoder(f).Decode(cfg)
+	if err != nil {
+		return
+	}
+	err = Validate(cfg)
+	return
+}
+
+type Config struct {
+	Server Server         `yaml:"server"`
+	Cache  Cache          `yaml:"cache"`
+	Path   string         `yaml:"path"`
+	REPath *regexp.Regexp `yaml:"-"`
+}
+
+type Server struct {
+	HTTP   Listen `yaml:"http"`
+	WebDav Listen `yaml:"webdav"`
+	Cache  Listen `yaml:"cache"`
+	Plain  Listen `yaml:"plain"`
+}
+
+type Listen struct {
+	Addr string `yaml:"addr"`
+}
+
+type Cache struct {
+	Metadata  string `yaml:"metadata"`
+	Src       Path   `yaml:"src"`
+	Dst       Path   `yaml:"dst"`
+	Preloads  int    `yaml:"preloads"`
+	Buffer    int    `yaml:"buffer"`
+	Quota     int64  `yaml:"quota"`
+	BlockSFTP *bool  `yaml:"blockSFTP"`
+}
+
+type Path struct {
+	Path string `yaml:"path"`
+	Key  string `yaml:"key"`
+}
+
+const (
+	defListenHTTP = "127.0.0.1:8080"
+	defPreloads   = 1
+	defBuffer     = 8192
+	defQuota      = 1
+	defBlockSFTP  = true
+	gib           = 1024 * 1024 * 1024
+)
+
+func Validate(cfg *Config) error {
+	if cfg.Server.HTTP.Addr == "" {
+		cfg.Server.HTTP.Addr = defListenHTTP
+	}
+	c := &cfg.Cache
+	if c.Metadata == "" {
+		return errors.New("cache.metadata is empty")
+	}
+	if c.Src.Path == "" {
+		return errors.New("cache.src is empty")
+	}
+	if c.Dst.Path == "" {
+		return errors.New("cache.dst is empty")
+	}
+	if c.Preloads <= 0 {
+		c.Preloads = defPreloads
+	}
+	if c.Buffer < 1024 {
+		c.Buffer = defBuffer
+	}
+	if c.Quota < 1 {
+		c.Quota = defQuota
+	}
+	c.Quota *= gib
+	if c.BlockSFTP == nil {
+		c.BlockSFTP = new(bool)
+		*c.BlockSFTP = defBlockSFTP
+	}
+	var err error
+	if cfg.Path != "" {
+		cfg.REPath, err = regexp.Compile(cfg.Path)
+	}
+	return err
+}

pkg/fs/discard.go

@@ -4,7 +4,6 @@ package fs
 
 import (
 	"io"
-	"sync"
 )
 
 var Discard io.Writer = discard{}
@@ -21,30 +20,33 @@ func (discard) WriteString(s string) (int, error) {
 	return len(s), nil
 }
 
-const DefaultBufferSize = 8192
+const (
+	StreamingBufferSize = 32768
+	DefaultBufferSize   = 8192
+)
 
-var blackHolePool = sync.Pool{
+func poolNewFunc(size int) func() *[]byte {
-	New: func() any {
+	return func() *[]byte {
-		b := make([]byte, DefaultBufferSize)
+		b := make([]byte, size)
 		return &b
-	},
+	}
 }
 
+var (
+	streamingPool = newPool(poolNewFunc(StreamingBufferSize))
+	blackHolePool = newPool(poolNewFunc(DefaultBufferSize))
+)
+
 func SetBufferSize(size int) bool {
 	if size < 0 || size == DefaultBufferSize {
 		return false
 	}
-	blackHolePool = sync.Pool{
+	blackHolePool = newPool(poolNewFunc(size))
-		New: func() any {
-			b := make([]byte, size)
-			return &b
-		},
-	}
 	return true
 }
 
 func (discard) ReadFrom(r io.Reader) (n int64, err error) {
-	bufp := blackHolePool.Get().(*[]byte)
+	bufp := blackHolePool.Get()
 	readSize := 0
 	for {
 		readSize, err = r.Read(*bufp)

pkg/fs/file.go

@@ -4,6 +4,7 @@ package fs
 
 import (
 	"cachefs/pkg/provider"
+	"cachefs/pkg/provider/crypto"
 	"context"
 	"errors"
 	"io"
@@ -41,6 +42,7 @@ func (p *preload) Read(data []byte) (n int, err error) {
 	if p.f.hasChunk(n) {
 		p.skipped++
 		_, err = p.f.Seek(int64(n), io.SeekCurrent)
+		n = 0
 		return
 	}
 	n, err = p.f.readToCache(data)
@@ -48,37 +50,45 @@ func (p *preload) Read(data []byte) (n int, err error) {
 	return
 }
 
-func (f *File) Preload(ctx context.Context, unlock func()) {
+func (f *File) Preload(ctx context.Context, r *Rate, fin, errf, stop func()) {
 	log := f.log
+	defer r.Stop()
 	defer f.Close()
 	if f.offline {
 		log.V(2).Error(errors.New("no preload in offline mode"), "error preloading file")
-		unlock()
+		errf()
 		return
 	}
 	if f.md.FullyCached() {
 		log.V(2).Info("skipped preload for fully cached file")
-		unlock()
+		fin()
 		return
 	}
 
 	log.V(2).Info("preload started")
 	p := &preload{f: f, ctx: ctx}
-	_, err := io.Copy(Discard, p)
+	r.SetReader(p)
+	_, err := io.Copy(Discard, r)
 	if err == context.Canceled {
 		log.V(2).Info("preload canceled", "skipped", p.skipped, "written", p.written)
-		// do not call unlock() to keep preload in list
+		stop()
 		return
 	}
+	perr := false
 	if err != nil && err != io.EOF {
 		log.Error(err, "error preloading file")
+		perr = true
 	}
 	err = f.md.Close()
 	if err != nil {
 		log.Error(err, "error closing cache file")
 	}
+	if perr {
+		errf()
+		return
+	}
 	log.V(2).Info("preload finished", "skipped", p.skipped, "written", p.written)
-	unlock()
+	fin()
 }
 
 func (f *File) Read(p []byte) (n int, err error) {
@@ -88,6 +98,10 @@ func (f *File) Read(p []byte) (n int, err error) {
 		if err == io.EOF {
 			return
 		}
+		if err == crypto.ErrDecrypt {
+			log.V(2).Info("rereading to cache file", "err", err)
+			return f.readToCache(p)
+		}
 		if err != nil {
 			if !IsIOErr(err) {
 				log.Error(err, "error reading cache file")
@@ -95,7 +109,7 @@ func (f *File) Read(p []byte) (n int, err error) {
 			}
 			n, err = f.readSource(p)
 			if err != nil {
-				log.Error(err, "error reading source file")
+				log.Error(err, "error reading fallback source file")
 				return
 			}
 		}
@@ -127,7 +141,8 @@ func (f *File) readToCache(p []byte) (n int, err error) {
 	log := f.log
 	n, err = f.readSource(p)
 	if n > 0 {
-		if n, err = f.md.WriteAt(p[:n], f.offset); err != nil {
+		// async cache write
+		if _, err = f.md.WriteAt(p[:n], f.offset); err != nil {
 			log.Error(err, "error writing cache file")
 			return
 		}
@@ -136,7 +151,6 @@ func (f *File) readToCache(p []byte) (n int, err error) {
 		log.Error(err, "error reading source file")
 		return
 	}
-	f.md.AddChunk(f.offset, n)
 	f.offset += int64(n)
 	if n > 0 && n < len(p) {
 		_, err = f.Seek(f.offset, io.SeekStart)

pkg/fs/fs.go

@@ -3,6 +3,7 @@
 package fs
 
 import (
+	"cachefs/pkg/config"
 	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/parse"
 	"context"
@@ -12,6 +13,8 @@ import (
 	stdfs "io/fs"
 	"net/http"
 	"os"
+	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/go-logr/logr"
@@ -19,8 +22,8 @@ import (
 
 type FS struct {
 	log     logr.Logger
-	done    chan struct{}
 	cancel  func()
+	pCancel func()
 	NoCache http.Handler
 	src     provider.FS
 	dst     provider.FS
@@ -32,7 +35,22 @@ type FS struct {
 	ph *PreloadHandler
 }
 
-func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, block bool, log logr.Logger) (fs *FS, err error) {
+func NewFSFromConfig(cfg *config.Config, log logr.Logger) (fs *FS, err error) {
+	c := cfg.Cache
+	return NewFS(
+		c.Quota,
+		c.Preloads,
+		c.Src.Path,
+		c.Src.Key,
+		c.Dst.Path,
+		c.Dst.Key,
+		c.Metadata,
+		cfg.REPath,
+		*c.BlockSFTP,
+		log,
+	)
+}
+func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, re *regexp.Regexp, block bool, log logr.Logger) (fs *FS, err error) {
 	if src == dst {
 		return nil, errors.New("src and dst path can not be equal")
 	}
@@ -51,21 +69,22 @@ func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, bloc
 		}
 		log.V(2).Info("using destination encryption")
 	}
-	srcfs, err := parse.FS(src, srck, block)
+	srcfs, err := parse.FS(src, srck, re, block)
 	if err != nil {
 		return nil, fmt.Errorf("srcfs: %w", err)
 	}
-	dstfs, err := parse.FS(dst, dstk, block)
+	dstfs, err := parse.FS(dst, dstk, nil, block)
 	if err != nil {
 		return nil, fmt.Errorf("dstfs: %w", err)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
+	pCtx, pCancel := context.WithCancel(context.Background())
 
 	fs = &FS{
 		log:     log,
-		done:    make(chan struct{}),
 		cancel:  cancel,
+		pCancel: pCancel,
 		NoCache: http.FileServer(http.Dir(src)),
 		src:     srcfs,
 		dst:     dstfs,
@@ -80,7 +99,7 @@ func NewFS(quota int64, max int, src, srckey, dst, dstkey, metadata string, bloc
 	if err != nil {
 		return nil, fmt.Errorf("quota: %w", err)
 	}
-	fs.ph = NewPreloadHandler(ctx, fs, max, log.WithName("preload"))
+	fs.ph = NewPreloadHandler(pCtx, fs, max, log.WithName("preload"))
 	fs.sc = NewStatCache(ctx)
 	return
 }
@@ -99,6 +118,36 @@ func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
 	return
 }
 
+const (
+	_ = 1 << (iota * 10)
+	KiB
+	MiB
+	GiB
+	TiB
+)
+
+func FileSize(s int64) string {
+	switch {
+	case s >= TiB:
+		return strconv.FormatFloat(float64(s)/TiB, 'f', 2, 64) + " TiB"
+	case s >= GiB:
+		return strconv.FormatFloat(float64(s)/GiB, 'f', 2, 64) + " GiB"
+	case s >= MiB:
+		return strconv.FormatFloat(float64(s)/MiB, 'f', 2, 64) + " MiB"
+	case s >= KiB:
+		return strconv.FormatFloat(float64(s)/KiB, 'f', 2, 64) + " KiB"
+	}
+	return strconv.Itoa(int(s)) + " B"
+}
+
+func (fs *FS) FileSize(name string) string {
+	fi, err := fs.Stat(name)
+	if err != nil {
+		return ""
+	}
+	return FileSize(fi.Size())
+}
+
 func (fs *FS) RemoveDst(name string) error {
 	return fs.dst.Remove(name)
 }
@@ -265,8 +314,14 @@ func (fs *FS) CacheStatus(name string) int {
 }
 
 func (fs *FS) Close() {
+	fs.pCancel()
+	<-fs.ph.Done()
+
 	fs.cancel()
 	fs.src.Close()
 	fs.dst.Close()
-	<-fs.done
+}
+
+func (fs *FS) Done() <-chan struct{} {
+	return fs.mh.Done()
 }

pkg/fs/metadata.go

@@ -3,15 +3,13 @@
 package fs
 
 import (
-	"cachefs/pkg/chunk"
-	"cachefs/pkg/provider"
-	"cachefs/pkg/provider/sftp"
 	"context"
 	"encoding/json"
 	"errors"
 	"io"
 	"io/fs"
 	stdfs "io/fs"
+	"maps"
 	"math"
 	"os"
 	"path/filepath"
@@ -20,8 +18,11 @@ import (
 	"sync/atomic"
 	"time"
 
+	"cachefs/pkg/chunk"
+	"cachefs/pkg/provider"
+	"cachefs/pkg/provider/sftp"
+
 	"github.com/go-logr/logr"
-	"golang.org/x/exp/maps"
 )
 
 func now() int64 {
@@ -29,13 +30,14 @@ func now() int64 {
 }
 
 type MetadataHandler struct {
-	mu  sync.RWMutex
+	mu   sync.RWMutex
-	log logr.Logger
+	done chan struct{}
-	md  map[string]*Metadata
+	log  logr.Logger
-	fs  *FS
+	md   map[string]*Metadata
-	dst string
+	fs   *FS
-	f   *os.File
+	dst  string
-	enc *json.Encoder
+	f    *os.File
+	enc  *json.Encoder
 }
 
 func MetadataGenerator(file string, dst provider.FS, fstat, encrypted bool) error {
@@ -109,10 +111,11 @@ func NewMetadataHandler(ctx context.Context, fs *FS, file, dst string, log logr.
 		return nil, errors.New("metadata path is not absolute")
 	}
 	mh = &MetadataHandler{
-		log: log,
+		log:  log,
-		md:  make(map[string]*Metadata),
+		done: make(chan struct{}),
-		fs:  fs,
+		md:   make(map[string]*Metadata),
-		dst: dst,
+		fs:   fs,
+		dst:  dst,
 	}
 	mh.f, err = os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o640)
 	if err != nil {
@@ -129,13 +132,13 @@ func NewMetadataHandler(ctx context.Context, fs *FS, file, dst string, log logr.
 	return
 }
 
-func (mh *MetadataHandler) DeleteOldest() (s int64) {
+func (mh *MetadataHandler) DeleteOldest() (free int64) {
 	mh.mu.Lock()
 	defer mh.mu.Unlock()
 	atime := int64(math.MaxInt64)
 	var m *Metadata
 	for _, v := range mh.md {
-		if v.Atime != 0 && v.f != nil && v.Atime < atime {
+		if v.Atime != 0 && v.Atime < atime {
 			atime = v.Atime
 			m = v
 		}
@@ -143,16 +146,16 @@ func (mh *MetadataHandler) DeleteOldest() (s int64) {
 	if m != nil {
 		log := mh.log.WithValues("file", m.name)
 		log.Info("deleting oldest file")
-		cs := m.Size
+		s := m.Size
 		if !m.FullyCached() {
-			cs = m.ChunkSize()
+			s = m.ChunkSize()
 		}
 		err := m.Delete()
 		if err != nil {
 			log.Error(err, "error deleting oldest file")
 			return
 		}
-		s = cs
+		free = s
 	}
 	return
 }
@@ -197,7 +200,14 @@ func (mh *MetadataHandler) Metadata(name string, size int64) (md *Metadata) {
 		}
 		return md
 	}
-	md = &Metadata{Size: size, fs: mh.fs, name: name}
+	ctx, cancel := context.WithCancel(context.Background())
+	md = &Metadata{
+		Size:   size,
+		fs:     mh.fs,
+		name:   name,
+		ctx:    ctx,
+		cancel: cancel,
+	}
 	mh.md[name] = md
 	return
 }
@@ -219,6 +229,7 @@ func (mh *MetadataHandler) init() {
 			}
 			return true
 		}
+		v.ctx, v.cancel = context.WithCancel(context.Background())
 		v.fs = mh.fs
 		v.name = k
 		return false
@@ -226,12 +237,15 @@ func (mh *MetadataHandler) init() {
 }
 
 func (mh *MetadataHandler) flusher(ctx context.Context) {
+	t := time.NewTicker(time.Minute * 5)
 	for {
 		select {
 		case <-ctx.Done():
+			t.Stop()
 			mh.close()
+			close(mh.done)
 			return
-		case <-time.After(time.Minute * 5):
+		case <-t.C:
 		}
 		mh.cleanupEmptyDirs()
 		mh.flush()
@@ -301,15 +315,20 @@ func (mh *MetadataHandler) flush() {
 func (mh *MetadataHandler) close() {
 	mh.mu.Lock()
 	for _, md := range mh.md {
-		if md.f != nil {
+		md.Close()
-			md.f.Sync()
+		//if md.f != nil {
-			md.f.Close()
+		//	close(md.wc)
-		}
+		//	md.f.Sync()
+		//	md.f.Close()
+		//}
 	}
 	mh.mu.Unlock()
 	mh.flush()
 	mh.f.Close()
-	close(mh.fs.done)
+}
+
+func (mh *MetadataHandler) Done() <-chan struct{} {
+	return mh.done
 }
 
 type Error string
@@ -338,54 +357,74 @@ func IsIOErr(err error) bool {
 }
 
 type Metadata struct {
-	mu     sync.RWMutex  `json:"-"`
+	mu     sync.Mutex            `json:"-"`
-	fs     *FS           `json:"-"`
+	cmu    sync.RWMutex          `json:"-"`
-	f      provider.File `json:"-"`
+	fs     *FS                   `json:"-"`
-	name   string        `json:"-"`
+	f      provider.File         `json:"-"`
-	Size   int64         `json:"s"`
+	wc     chan writeAt          `json:"-"`
-	Atime  int64         `json:"a"`
+	done   chan struct{}         `json:"-"`
-	Chunks chunk.Chunks  `json:"c"`
+	ctx    context.Context       `json:"-"`
+	cancel func()                `json:"-"`
+	err    atomic.Pointer[mdErr] `json:"-"`
+	name   string                `json:"-"`
+	Size   int64                 `json:"s"`
+	Atime  int64                 `json:"a"`
+	Chunks chunk.Chunks          `json:"c"`
 }
 
-func (md *Metadata) Close() error {
+func (md *Metadata) Close() (err error) {
 	md.mu.Lock()
 	defer md.mu.Unlock()
-	err := md.f.Close()
+	return md.close()
-	md.f = nil
+}
-	return err
+
+func (md *Metadata) close() (err error) {
+	md.cancel()
+	if md.wc != nil {
+		close(md.wc)
+		<-md.done
+		md.wc = nil
+	}
+	if md.f != nil {
+		md.f.Sync()
+		err = md.f.Close()
+		md.f = nil
+	}
+	return
 }
 
 func (md *Metadata) Delete() error {
 	md.mu.Lock()
 	defer md.mu.Unlock()
+	md.cmu.Lock()
 	md.Chunks = chunk.Chunks{}
-	md.f.Close()
+	md.cmu.Unlock()
-	md.f = nil
+	md.close()
 	atomic.StoreInt64(&md.Atime, now())
 	return md.fs.RemoveDst(md.name)
 }
 
 func (md *Metadata) FullyCached() bool {
-	md.mu.RLock()
+	md.cmu.RLock()
-	defer md.mu.RUnlock()
+	defer md.cmu.RUnlock()
 	return len(md.Chunks) == 1 && md.Size == md.Chunks[0][1]
 }
 
 func (md *Metadata) HasChunk(off int64, n int) bool {
-	md.mu.RLock()
+	md.cmu.RLock()
-	defer md.mu.RUnlock()
+	defer md.cmu.RUnlock()
 	return md.Chunks.Exists(off, n, md.Size)
 }
 
 func (md *Metadata) ChunkSize() int64 {
-	md.mu.RLock()
+	md.cmu.RLock()
-	defer md.mu.RUnlock()
+	defer md.cmu.RUnlock()
 	return md.Chunks.Size()
 }
 
-func (md *Metadata) AddChunk(off int64, n int) {
+func (md *Metadata) addChunk(off int64, n int) {
-	md.mu.Lock()
+	md.cmu.Lock()
-	defer md.mu.Unlock()
+	defer md.cmu.Unlock()
 	md.Chunks.Add(off, n)
 }
 
@@ -400,27 +439,73 @@ func (md *Metadata) AddChunk(off int64, n int) {
 //	return md.f.ReadAt(p, pos)
 //}
 
+type mdErr struct {
+	err error
+}
+
+type writeAt struct {
+	pos  int64
+	data []byte
+	ret  *[]byte
+}
+
+func (md *Metadata) resetChans() {
+	md.wc = make(chan writeAt, 10)
+	md.done = make(chan struct{})
+}
+
+func (md *Metadata) writer() {
+	md.resetChans()
+	md.err.Store(nil)
+	go func() {
+		for wa := range md.wc {
+			atomic.StoreInt64(&md.Atime, now())
+			n, err := md.f.WriteAt(wa.data, wa.pos)
+			if err == nil {
+				md.addChunk(wa.pos, n)
+				md.fs.q.Add(n)
+			} else {
+				md.err.Store(&mdErr{err})
+			}
+			if wa.ret != nil {
+				streamingPool.Put(wa.ret)
+			}
+		}
+		close(md.done)
+	}()
+}
+
 func (md *Metadata) WriteAt(data []byte, pos int64) (int, error) {
+	err := md.err.Load()
+	if err != nil && err.err != nil {
+		return 0, err.err
+	}
 	if md.f == nil {
 		err := md.openCacheFile()
 		if err != nil {
 			return 0, err
 		}
 	}
-	atomic.StoreInt64(&md.Atime, now())
+	if md.ctx.Err() == context.Canceled {
-	md.fs.q.Add(len(data))
+		return 0, context.Canceled
-	return md.f.WriteAt(data, pos)
-}
-
-func (md *Metadata) openCacheFile() error {
-	md.mu.Lock()
-	defer md.mu.Unlock()
-	if md.f != nil {
-		return nil
 	}
-	f, err := md.fs.openCacheFile(md.name, md.Size)
+	wa := writeAt{pos: pos}
-	md.f = f
+	ret := streamingPool.Get()
-	return err
+	n := len(data)
+	buf := *ret
+	if n > len(buf) {
+		streamingPool.Put(ret)
+		buf = make([]byte, n)
+	} else {
+		wa.ret = ret
+	}
+	copy(buf, data)
+	wa.data = buf[:n]
+	select {
+	case <-md.ctx.Done():
+	case md.wc <- wa:
+	}
+	return n, nil
 }
 
 func (md *Metadata) Stat() (os.FileInfo, error) {
@@ -432,3 +517,22 @@ func (md *Metadata) Stat() (os.FileInfo, error) {
 	}
 	return md.f.Stat()
 }
+
+func (md *Metadata) openCacheFile() error {
+	md.mu.Lock()
+	defer md.mu.Unlock()
+	if md.f != nil {
+		return nil
+	}
+	f, err := md.fs.openCacheFile(md.name, md.Size)
+	if err != nil {
+		return err
+	}
+	md.f = f
+	if md.ctx.Err() != nil {
+		md.cancel()
+		md.ctx, md.cancel = context.WithCancel(context.Background())
+	}
+	md.writer()
+	return nil
+}

pkg/fs/pool.go

@@ -0,0 +1,27 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package fs
+
+import (
+	"sync"
+)
+
+type pool[T any] struct {
+	p sync.Pool
+}
+
+func newPool[T any](newf func() T) *pool[T] {
+	return &pool[T]{
+		p: sync.Pool{
+			New: func() any { return newf() },
+		},
+	}
+}
+
+func (p *pool[T]) Get() T {
+	return p.p.Get().(T)
+}
+
+func (p *pool[T]) Put(v T) {
+	p.p.Put(v)
+}

pkg/fs/preload.go

@@ -3,19 +3,24 @@
 package fs
 
 import (
+	"cmp"
 	"context"
+	"io"
+	"slices"
+	"strconv"
 	"sync"
+	"sync/atomic"
+	"time"
 
 	"github.com/go-logr/logr"
-	"golang.org/x/exp/slices"
 )
 
 type queue []*Preload
 
-func (queue) LessReverse(i, j *Preload) bool { return i.Prio > j.Prio }
+func (queue) CompReverse(a, b *Preload) int { return cmp.Compare(b.Prio, a.Prio) }
 
 func (q *queue) Sort() {
-	slices.SortFunc(*q, q.LessReverse)
+	slices.SortFunc(*q, q.CompReverse)
 }
 
 func (q *queue) Add(ph *PreloadHandler, name string, prio int) {
@@ -29,14 +34,23 @@ func (q *queue) Add(ph *PreloadHandler, name string, prio int) {
 			return
 		}
 	}
-	*q = append(*q, &Preload{Name: name, ph: ph})
+	p := &Preload{Name: name, ph: ph}
+	*q = append(*q, p)
+	file, err := ph.fs.Open(name)
+	if err == nil {
+		defer file.Close()
+		f, ok := file.(*File)
+		if ok {
+			p.Size = f.size()
+		}
+	}
 	q.Sort()
 }
 
-func (q *queue) Remove(name string) {
+func (q *queue) Remove(name string, collect bool) {
 	for i, p := range *q {
 		if p.Name == name {
-			p.stop()
+			p.stop(collect)
 			*q = slices.Delete(*q, i, i+1)
 			q.Sort()
 			return
@@ -44,24 +58,115 @@ func (q *queue) Remove(name string) {
 	}
 }
 
+func (q *queue) GetPreload(name string) *Preload {
+	for _, p := range *q {
+		if p.Name == name {
+			return p
+		}
+	}
+	return nil
+}
+
+type Rate struct {
+	r        io.Reader
+	done     chan struct{}
+	rate     atomic.Pointer[string]
+	num      int64
+	last     int64
+	interval int
+}
+
+var zeroRate = fmtRate(0)
+
+func newRate() *Rate {
+	r := &Rate{
+		done:     make(chan struct{}),
+		interval: 10,
+	}
+	r.rate.Store(&zeroRate)
+	return r
+}
+
+func (r *Rate) SetReader(reader io.Reader) {
+	r.r = reader
+	go r.sampler()
+}
+
+func (r *Rate) Read(b []byte) (n int, err error) {
+	n, err = r.r.Read(b)
+	if n > 0 {
+		atomic.AddInt64(&r.num, int64(n))
+	}
+	return
+}
+
+func fmtRate(r int) string {
+	switch {
+	case r >= GiB:
+		return strconv.FormatFloat(float64(r)/GiB, 'f', 2, 64) + " GiB/s"
+	case r >= MiB:
+		return strconv.FormatFloat(float64(r)/MiB, 'f', 2, 64) + " MiB/s"
+	case r >= KiB:
+		return strconv.FormatFloat(float64(r)/KiB, 'f', 2, 64) + " KiB/s"
+	}
+	return strconv.Itoa(r) + " B/s"
+}
+
+func (r *Rate) sampler() {
+	t := time.NewTicker(time.Duration(r.interval) * time.Second)
+	for {
+		select {
+		case <-r.done:
+			t.Stop()
+			return
+		case <-t.C:
+			num := atomic.LoadInt64(&r.num)
+			rate := int(num-r.last) / r.interval
+			r.last = num
+			s := fmtRate(rate)
+			r.rate.Store(&s)
+		}
+	}
+}
+
+func (r *Rate) String() string {
+	s := r.rate.Load()
+	if s == nil {
+		return ""
+	}
+	return *s
+}
+
+func (r *Rate) Stop() {
+	close(r.done)
+	r.rate.Store(&zeroRate)
+}
+
 type Preload struct {
 	ph      *PreloadHandler
+	rate    *Rate
+	delay   time.Duration
 	Name    string
 	cancel  func()
 	Prio    int
 	Status  int
+	Errc    int
+	Size    int64
 	Running bool
 }
 
 type Preloads []Preload
 
 type PreloadHandler struct {
-	mu  sync.RWMutex
+	mu   sync.RWMutex
-	log logr.Logger
+	log  logr.Logger
-	fs  *FS
+	fs   *FS
-	q   queue
+	q    queue
-	fin chan string
+	fin  chan string
-	max int
+	err  chan string
+	stop chan struct{}
+	done chan struct{}
+	max  int
 }
 
 func NewPreloadHandler(ctx context.Context, fs *FS, max int, log logr.Logger) (ph *PreloadHandler) {
@@ -69,20 +174,23 @@ func NewPreloadHandler(ctx context.Context, fs *FS, max int, log logr.Logger) (p
 		max = 1
 	}
 	ph = &PreloadHandler{
-		log: log,
+		log:  log,
-		fs:  fs,
+		fs:   fs,
-		q:   make(queue, 0),
+		q:    make(queue, 0),
-		fin: make(chan string, 1),
+		fin:  make(chan string, 1),
-		max: max,
+		err:  make(chan string, 1),
+		stop: make(chan struct{}, 1),
+		done: make(chan struct{}),
+		max:  max,
 	}
-	go ph.preloadFinish(ctx)
+	go ph.preloadStatus(ctx)
 	return
 }
 
 func (ph *PreloadHandler) RemovePreload(name string) {
 	ph.mu.RLock()
 	defer ph.mu.RUnlock()
-	ph.q.Remove(name)
+	ph.q.Remove(name, true)
 	ph.schedule()
 }
 
@@ -107,21 +215,32 @@ func (ph *PreloadHandler) Preload(name string, prio int) {
 func (ph *PreloadHandler) schedule() {
 	for i := 0; i < len(ph.q); i++ {
 		if i < ph.max {
-			ph.q[i].start()
+			go ph.q[i].start()
 		} else {
-			ph.q[i].stop()
+			ph.q[i].stop(true)
 		}
 	}
 }
 
+func (p *Preload) Rate() string {
+	if p.rate != nil {
+		return p.rate.String()
+	}
+	return ""
+}
+
 func (p *Preload) start() {
 	if p.Running {
 		return
 	}
+	if p.delay != 0 {
+		time.Sleep(p.delay)
+	}
 	name := p.Name
 	file, err := p.ph.fs.Open(name)
 	if err != nil {
 		p.ph.log.Error(err, "error staring next preload", "file", name)
+		p.ph.err <- name
 		return
 	}
 	f, ok := file.(*File)
@@ -131,19 +250,33 @@ func (p *Preload) start() {
 	ctx, cancel := context.WithCancel(context.Background())
 	p.Running = true
 	p.cancel = cancel
-	go f.Preload(ctx, func() {
+	p.rate = newRate()
-		p.ph.fin <- name
+	p.Size = f.size()
-	})
+	go f.Preload(ctx,
+		p.rate,
+		func() {
+			p.ph.fin <- name
+		},
+		func() {
+			p.ph.err <- name
+		},
+		func() {
+			p.ph.stop <- struct{}{}
+		},
+	)
 }
 
-func (p *Preload) stop() {
+func (p *Preload) stop(collect bool) {
 	if p.Running {
-		p.cancel()
 		p.Running = false
+		p.cancel()
+		if collect {
+			<-p.ph.stop
+		}
 	}
 }
 
-func (ph *PreloadHandler) preloadFinish(ctx context.Context) {
+func (ph *PreloadHandler) preloadStatus(ctx context.Context) {
 	for {
 		select {
 		case <-ctx.Done():
@@ -151,22 +284,45 @@ func (ph *PreloadHandler) preloadFinish(ctx context.Context) {
 			return
 		case name := <-ph.fin:
 			ph.mu.Lock()
-			ph.q.Remove(name)
+			ph.q.Remove(name, false)
 			ph.schedule()
 			ph.mu.Unlock()
+		case name := <-ph.err:
+			ph.mu.Lock()
+			p := ph.q.GetPreload(name)
+			if p != nil {
+				p.stop(false)
+				p.delay = time.Second * 2
+				if p.Errc >= 10 {
+					p.Prio = -10
+					ph.schedule()
+				} else {
+					p.Errc += 1
+					go p.start()
+				}
+			}
+			ph.mu.Unlock()
 		}
 	}
 }
 
 func (ph *PreloadHandler) close() {
+	ph.mu.Lock()
+	defer ph.mu.Unlock()
+	n := len(ph.q)
 	for _, p := range ph.q {
-		p.stop()
+		p.stop(false)
 	}
-	for {
+	for i := 0; i < n; i++ {
 		select {
 		case <-ph.fin:
-		default:
+		case <-ph.err:
-			return
+		case <-ph.stop:
 		}
 	}
+	close(ph.done)
+}
+
+func (ph *PreloadHandler) Done() <-chan struct{} {
+	return ph.done
 }

pkg/fs/quota.go

@@ -42,8 +42,8 @@ func (q *Quota) cleanup() {
 	}
 	q.log.Info("quota usage", "current", atomic.LoadInt64(&q.cur), "max", q.max)
 	defer q.mu.Unlock()
-	size := q.fs.mh.DeleteOldest()
+	free := q.fs.mh.DeleteOldest()
-	atomic.AddInt64(&q.cur, -size)
+	atomic.AddInt64(&q.cur, -free)
 }
 
 func (q *Quota) Add(n int) {

pkg/provider/crypto/crypto.go

@@ -1,3 +1,6 @@
+// Copyright (C) 2022 Marius Schellenberger
+// crypto parts taken from https://github.com/filosottile/age
+
 package crypto
 
 import (

pkg/provider/crypto/file.go

@@ -1,3 +1,5 @@
+// Copyright (C) 2022 Marius Schellenberger
+
 package crypto
 
 import (
@@ -139,7 +141,6 @@ func (f *file) Seek(offset int64, whence int) (n int64, err error) {
 		return
 	}
 	f.nonce = [NonceSize]byte{}
-	//inc(&f.nonce, cn)
 	put(&f.nonce, cn)
 	f.r.nonce = f.nonce
 	f.w.nonce = f.nonce
@@ -164,7 +165,7 @@ func (f *file) ReadAt(p []byte, pos int64) (n int, err error) {
 	defer f.rmu.Unlock()
 	cn, _, roff := align(pos)
 	var last bool
-	if f.r.cn != cn {
+	if f.r.cn != cn || roff >= int64(len(f.r.unread)) {
 		_, err = f.Seek(pos, io.SeekStart)
 		if err != nil {
 			return
@@ -184,26 +185,35 @@ func (f *file) ReadAt(p []byte, pos int64) (n int, err error) {
 func (f *file) WriteAt(data []byte, pos int64) (n int, err error) {
 	f.wmu.Lock()
 	defer f.wmu.Unlock()
-	cn, off, woff := align(pos)
+	for n != len(data) {
-	c, ok := f.w.cm[cn]
+		cn, off, woff := align(pos + int64(n))
-	if !ok {
+		c, ok := f.w.cm[cn]
-		c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
+		if !ok {
-		//inc(&c.nonce, cn)
+			c = &chunkWriter{offset: off + KDFNonceSize, w: f.w, cn: cn}
-		put(&c.nonce, cn)
+			put(&c.nonce, cn)
-		f.w.cm[cn] = c
+			f.w.cm[cn] = c
-		if f.w.last == nil {
+			if f.w.last == nil {
-			f.w.last = c
+				f.w.last = c
-		} else if cn > f.w.last.cn {
+			} else if cn > f.w.last.cn {
-			f.w.last = c
+				f.w.last = c
+			}
+		}
+		w, e := c.writeAt(data[n:], woff)
+		n += w
+		if c.full() {
+			if f.w.last != nil && f.w.last.cn == cn {
+				f.w.last = nil
+			}
+			// async flush
+			go func() {
+				c.flush(notLastChunk)
+			}()
+			delete(f.w.cm, cn)
+		}
+		if e != nil {
+			err = e
+			return
 		}
-	}
-	n, err = c.writeAt(data, woff)
-	if c.full() {
-		// async flush
-		go func() {
-			c.flush(notLastChunk)
-		}()
-		delete(f.w.cm, cn)
 	}
 	return
 }

pkg/provider/crypto/fs.go

@@ -1,3 +1,5 @@
+// Copyright (C) 2022 Marius Schellenberger
+
 package crypto
 
 import (

pkg/provider/crypto/helper.go

@@ -1,3 +1,6 @@
+// Copyright (C) 2022 Marius Schellenberger
+// crypto parts taken from https://github.com/filosottile/age
+
 package crypto
 
 import (

pkg/provider/crypto/helper_test.go

@@ -1,3 +1,5 @@
+// Copyright (C) 2022 Marius Schellenberger
+
 package crypto
 
 import (

pkg/provider/crypto/reader.go

@@ -1,3 +1,6 @@
+// Copyright (C) 2022 Marius Schellenberger
+// crypto parts taken from https://github.com/filosottile/age
+
 package crypto
 
 import (
@@ -77,6 +80,8 @@ func (r *reader) Read(p []byte) (int, error) {
 	return n, nil
 }
 
+var ErrDecrypt = errors.New("failed to decrypt and authenticate payload chunk")
+
 func (r *reader) readChunk() (last bool, err error) {
 	if len(r.unread) != 0 {
 		panic("stream: internal error: readChunk called with dirty buffer")
@@ -97,7 +102,6 @@ func (r *reader) readChunk() (last bool, err error) {
 		return false, err
 	}
 
-	//outBuf := make([]byte, 0, ChunkSize)
 	out, err := r.a.Open(r.outBuf, r.nonce[:], in, nil)
 	if err != nil && !last {
 		// Check if this was a full-length final chunk.
@@ -106,11 +110,10 @@ func (r *reader) readChunk() (last bool, err error) {
 		out, err = r.a.Open(r.outBuf, r.nonce[:], in, nil)
 	}
 	if err != nil {
-		return false, errors.New("failed to decrypt and authenticate payload chunk")
+		return false, ErrDecrypt
 	}
 
 	incNonce(&r.nonce)
-	//r.unread = r.buf[:copy(r.buf[:], out)]
 	size := FullSize
 	if len(out) < FullSize {
 		size = len(out)

pkg/provider/crypto/writer.go

@@ -1,3 +1,6 @@
+// Copyright (C) 2022 Marius Schellenberger
+// crypto parts taken from https://github.com/filosottile/age
+
 package crypto
 
 import (

pkg/provider/filter/file.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package filter
+
+import (
+	"cachefs/pkg/provider"
+	"io/fs"
+	"path/filepath"
+)
+
+var _ provider.File = (*file)(nil)
+
+type file struct {
+	provider.File
+	fs   *FS
+	path string
+}
+
+func newFile(f provider.File, fs *FS, path string) (nf provider.File, err error) {
+	nf = &file{
+		File: f,
+		fs:   fs,
+		path: path,
+	}
+	return nf, nil
+}
+
+func (f *file) Readdir(n int) ([]fs.FileInfo, error) {
+	fis, err := f.File.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.FileInfo, len(fis))
+	i := 0
+	for _, fi := range fis {
+		file := filepath.Join(f.path, fi.Name())
+		if f.fs.match(file) == nil {
+			fio[i] = fi
+			i++
+		}
+	}
+	return fio[:i], nil
+}
+
+func (f *file) ReadDir(n int) ([]fs.DirEntry, error) {
+	fis, err := f.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	fio := make([]fs.DirEntry, len(fis))
+	for i, fi := range fis {
+		fio[i] = fs.FileInfoToDirEntry(fi)
+	}
+	return fio, nil
+}

pkg/provider/filter/fs.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package filter
+
+import (
+	stdfs "io/fs"
+	"os"
+	"regexp"
+
+	"cachefs/pkg/provider"
+)
+
+var _ provider.FS = (*FS)(nil)
+
+type FS struct {
+	provider.FS
+	re *regexp.Regexp
+}
+
+func NewFS(fs provider.FS, re *regexp.Regexp) (*FS, error) {
+	return &FS{
+		FS: fs,
+		re: re,
+	}, nil
+}
+
+func (fs *FS) match(p string) error {
+	if fs.re.MatchString(p) {
+		return nil
+	}
+	return stdfs.ErrNotExist
+}
+
+func (fs *FS) Stat(p string) (stdfs.FileInfo, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	return fs.FS.Stat(p)
+}
+
+func (fs *FS) Remove(p string) error {
+	err := fs.match(p)
+	if err != nil {
+		return err
+	}
+	return fs.FS.Remove(p)
+}
+
+func (fs *FS) Open(p string) (provider.File, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.FS.Open(p)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(f, fs, p)
+}
+
+func (fs *FS) OpenFile(p string, flags int, mode os.FileMode) (provider.File, error) {
+	err := fs.match(p)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.FS.OpenFile(p, flags, mode)
+	if err != nil {
+		return nil, err
+	}
+	return newFile(f, fs, p)
+}
+
+func (fs *FS) MkdirAll(p string, mode os.FileMode) error {
+	err := fs.match(p)
+	if err != nil {
+		return err
+	}
+	return fs.FS.MkdirAll(p, mode)
+}

pkg/provider/parse/parse.go

@@ -5,12 +5,14 @@ package parse
 import (
 	"cachefs/pkg/provider"
 	"cachefs/pkg/provider/crypto"
+	"cachefs/pkg/provider/filter"
 	"cachefs/pkg/provider/os"
 	"cachefs/pkg/provider/sftp"
 	"errors"
 	"fmt"
 	neturl "net/url"
 	"path/filepath"
+	"regexp"
 
 	"k8s.io/klog/v2/klogr"
 )
@@ -36,7 +38,32 @@ func TryParse(url string) error {
 	return nil
 }
 
-func FS(url string, key []byte, block bool) (provider.FS, error) {
+func Paths(url string) (paths []string) {
+	u, err := neturl.Parse(url)
+	if err != nil {
+		return
+	}
+	switch u.Scheme {
+	case "file":
+		paths = append(paths, u.Path)
+	case "sftp":
+		q, err := neturl.ParseQuery(u.RawQuery)
+		if err != nil {
+			return
+		}
+		known, err := sftp.KnownHosts(q)
+		if err != nil {
+			return
+		}
+		paths = append(paths, known)
+		if key := q.Get(sftp.KeyParam); key != "" {
+			paths = append(paths, key)
+		}
+	}
+	return
+}
+
+func FS(url string, key []byte, re *regexp.Regexp, block bool) (provider.FS, error) {
 	u, err := neturl.Parse(url)
 	if err != nil {
 		return nil, err
@@ -58,6 +85,12 @@ func FS(url string, key []byte, block bool) (provider.FS, error) {
 	}
 	if key != nil {
 		fs, err = crypto.NewFS(fs, key)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if re != nil {
+		fs, err = filter.NewFS(fs, re)
 	}
 	return fs, err
 }

pkg/provider/sftp/error.go

@@ -1,4 +1,5 @@
 // Copyright (C) 2022 Marius Schellenberger
+
 package sftp
 
 import (

pkg/provider/sftp/sftp.go

@@ -197,14 +197,19 @@ func (fs *FS) MkdirAll(p string, _ os.FileMode) error {
 
 func (fs *FS) Close() error {
 	fs.cancel()
-	fs.client.Close()
+	err := fs.client.Close()
-	return fs.c.Close()
+	if fs.c != nil {
+		return fs.c.Close()
+	}
+	return err
 }
 
 func (fs *FS) path(p string) string {
 	return filepath.Join(fs.root, filepath.FromSlash(path.Clean("/"+p)))
 }
 
+const KeyParam = "key"
+
 func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 	if u.User == nil {
 		return nil, errors.New("missing username")
@@ -223,13 +228,9 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 		User:    u.User.Username(),
 		Timeout: time.Second * 30,
 	}
-	known := q.Get("known_hosts")
+	known, err := KnownHosts(q)
-	if known == "" {
+	if err != nil {
-		home, err := os.UserHomeDir()
+		return nil, err
-		if err != nil {
-			return nil, fmt.Errorf("error reading user home: %w", err)
-		}
-		known = filepath.Join(home, ".ssh", "known_hosts")
 	}
 	c.HostKeyCallback, err = knownhosts.New(known)
 	if err != nil {
@@ -238,7 +239,7 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 	if pw, ok := u.User.Password(); ok {
 		c.Auth = []ssh.AuthMethod{ssh.Password(pw)}
 	} else {
-		buf, err := os.ReadFile(q.Get("key"))
+		buf, err := os.ReadFile(q.Get(KeyParam))
 		if err != nil {
 			return nil, err
 		}
@@ -247,3 +248,15 @@ func sshConfig(u *url.URL) (c *ssh.ClientConfig, err error) {
 	}
 	return
 }
+
+func KnownHosts(q url.Values) (string, error) {
+	known := q.Get("known_hosts")
+	if known == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", fmt.Errorf("error reading user home: %w", err)
+		}
+		known = filepath.Join(home, ".ssh", "known_hosts")
+	}
+	return known, nil
+}

pkg/srv/cache.go

@@ -82,7 +82,7 @@ func (cs *CacheServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	h.Del("Last-Modified")
 
-	paths, err := i.GetPaths(p, cs.fs, true)
+	paths, pathAnchor, err := i.GetPaths(p, cs.fs, true)
 	if err == io.EOF {
 		w.WriteHeader(i.Status())
 		return
@@ -94,7 +94,7 @@ func (cs *CacheServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.Set(csp, indexCSP)
 
 	w.WriteHeader(i.Status())
-	err = cache.Execute(w, data{Paths: paths})
+	err = cache.Execute(w, data{Paths: paths, PathAnchor: pathAnchor})
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return

pkg/srv/interceptor.go

@@ -4,14 +4,14 @@ package srv
 
 import (
 	"bytes"
+	"cmp"
 	"encoding/xml"
 	"html/template"
 	"net/http"
+	"slices"
 	"strings"
 
 	"cachefs/pkg/fs"
-
-	"golang.org/x/exp/slices"
 )
 
 type statusInterceptor struct {
@@ -47,26 +47,30 @@ type dirContents struct {
 }
 
 type dir struct {
-	Name template.HTML
+	Name   template.HTML
-	URI  template.HTML
+	URI    template.HTML
+	Anchor string
 }
 
 type dirs []dir
 
-func (dirs) Less(i, j dir) bool { return i.Name < j.Name }
+func (dirs) Comp(a, b dir) int { return cmp.Compare(a.Name, b.Name) }
 
 type file struct {
 	Name    template.HTML
 	URI     template.HTML
 	Anchor  string
+	Size    string
+	Rate    string
 	Status  int
 	Prio    int
+	Errc    int
 	Running bool
 }
 
 type files []file
 
-func (files) Less(i, j file) bool { return i.Name < j.Name }
+func (files) Comp(a, b file) int { return cmp.Compare(a.Name, b.Name) }
 
 type responseInterceptor struct {
 	buf    bytes.Buffer
@@ -93,7 +97,19 @@ func (r *responseInterceptor) Status() int {
 	return r.status
 }
 
-func (r *responseInterceptor) GetPaths(path string, fs *fs.FS, relative bool) (dc dirContents, err error) {
+func xmldecode(s string) string {
+	s = strings.ReplaceAll(s, "&amp;", "&")
+	s = strings.ReplaceAll(s, "&#39;", "'")
+	return s
+}
+
+func urlencode(s string) string {
+	s = strings.ReplaceAll(s, "[", "%5B")
+	s = strings.ReplaceAll(s, "]", "%5D")
+	return s
+}
+
+func (r *responseInterceptor) GetPaths(path string, filesystem *fs.FS, relative bool) (dc dirContents, pa string, err error) {
 	buf := r.buf.Bytes()
 	buf = bytes.ReplaceAll(buf, []byte{'&'}, []byte("&#38;"))
 	err = xml.Unmarshal(buf, &dc)
@@ -102,48 +118,60 @@ func (r *responseInterceptor) GetPaths(path string, fs *fs.FS, relative bool) (d
 	}
 	dc.Base = path
 	path = strings.TrimSuffix(path, "/")
+	if path != "/" && path != "" {
+		pa = "#" + anchor(path)
+	}
 	for _, p := range dc.AllPaths {
-		p = strings.ReplaceAll(p, "&amp;", "&")
+		p = xmldecode(p)
-		p = strings.ReplaceAll(p, "&#39;", "'")
 		name := template.HTML(p)
 		full := path + "/" + p
-		uri := template.HTML(full)
+		uri := template.HTML(urlencode(full))
 		if relative {
-			uri = template.HTML(p)
+			uri = template.HTML(urlencode(p))
 		}
 		if p[len(p)-1] == '/' {
 			dc.Dirs = append(dc.Dirs, dir{
-				Name: name,
+				Name:   name,
-				URI:  uri,
+				URI:    uri,
+				Anchor: pathAnchor(p),
 			})
 		} else {
 			dc.Files = append(dc.Files, file{
 				Name:   name,
 				URI:    uri,
 				Anchor: anchor(p),
-				Status: fs.CacheStatus(full),
+				Size:   filesystem.FileSize(full),
+				Status: filesystem.CacheStatus(full),
 			})
 		}
 	}
-	slices.SortFunc(dc.Dirs, dc.Dirs.Less)
+	slices.SortFunc(dc.Dirs, dc.Dirs.Comp)
-	slices.SortFunc(dc.Files, dc.Files.Less)
+	slices.SortFunc(dc.Files, dc.Files.Comp)
 	return
 }
 
-func getPreloads(path string, fs *fs.FS) (dc dirContents) {
+func getPreloads(path string, filesystem *fs.FS) (dc dirContents, size int64) {
 	dc.Base = path
-	for _, p := range fs.Preloads() {
+	for _, p := range filesystem.Preloads() {
+		size += p.Size
 		dc.Files = append(dc.Files, file{
 			Name:    template.HTML(p.Name),
-			URI:     template.HTML(p.Name),
+			URI:     template.HTML(urlencode(p.Name)),
+			Size:    fs.FileSize(p.Size),
+			Rate:    p.Rate(),
 			Status:  p.Status,
 			Prio:    p.Prio,
+			Errc:    p.Errc,
 			Running: p.Running,
 		})
 	}
 	return
 }
 
+func pathAnchor(s string) string {
+	return anchor(strings.TrimSuffix(s, "/"))
+}
+
 func anchor(s string) string {
 	if i := strings.LastIndex(s, "/"); i > 0 && i+1 < len(s) {
 		s = s[i+1:]

pkg/srv/plain.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+package srv
+
+import (
+	"net/http"
+	"strings"
+
+	"cachefs/pkg/fs"
+
+	"github.com/go-logr/logr"
+)
+
+type PlainServer struct {
+	log logr.Logger
+	fs  *fs.FS
+	h   http.Handler
+}
+
+func NewPlainServer(fs *fs.FS, log logr.Logger) http.Handler {
+	return &PlainServer{log, fs, http.FileServer(fs)}
+}
+
+func skipPlainLog(path string) bool {
+	return strings.HasSuffix(path, "favicon.ico")
+}
+
+func (ps *PlainServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s := &statusInterceptor{w: w}
+	if !skipPlainLog(r.URL.Path) {
+		defer func() {
+			if s.Status() == http.StatusPartialContent {
+				return
+			}
+			ps.log.Info("access",
+				"client", r.RemoteAddr,
+				"method", r.Method,
+				"status", s.Status(),
+				"uri", r.RequestURI,
+			)
+		}()
+	}
+	r.Header.Del("If-Modified-Since")
+	r.Header.Del("Cache-Control")
+	ps.h.ServeHTTP(s, r)
+	w.Header().Del("Last-Modified")
+}

pkg/srv/srv.go

@@ -25,9 +25,11 @@ const (
 )
 
 type data struct {
-	QuotaCur float64
+	QuotaCur   float64
-	QuotaMax float64
+	QuotaEst   float64
-	Paths    dirContents
+	QuotaMax   float64
+	Paths      dirContents
+	PathAnchor string
 }
 
 type FileServer struct {
@@ -57,7 +59,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				"client", r.RemoteAddr,
 				"method", r.Method,
 				"status", s.Status(),
-				"uri", r.URL.Path,
+				"uri", r.RequestURI,
 			)
 		}()
 	}
@@ -117,10 +119,12 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if option == "preloads" {
 		h.Set(csp, indexCSP)
 		cur, max := fs.fs.QuotaUsage()
+		paths, size := getPreloads(p, fs.fs)
 		err = preloads.Execute(w, data{
 			QuotaCur: math.Round(float64(cur)/gib*100) / 100,
+			QuotaEst: math.Round(float64(cur+size)/gib*100) / 100,
 			QuotaMax: float64(max) / gib,
-			Paths:    getPreloads(p, fs.fs),
+			Paths:    paths,
 		})
 		if err != nil {
 			fs.log.Error(err, "error rendering preloads")
@@ -136,7 +140,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	h.Del("Last-Modified")
 
-	paths, err := i.GetPaths(p, fs.fs, false)
+	paths, pathAnchor, err := i.GetPaths(p, fs.fs, false)
 	if err == io.EOF {
 		w.WriteHeader(i.Status())
 		return
@@ -148,7 +152,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.Set(csp, indexCSP)
 
 	w.WriteHeader(i.Status())
-	err = index.Execute(w, data{Paths: paths})
+	err = index.Execute(w, data{Paths: paths, PathAnchor: pathAnchor})
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return

pkg/srv/templates/cache.html

@@ -17,7 +17,7 @@
   <tr class="listitem">
     <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
     <td class="listoptions">
-      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a>
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span>{{$s.Size}}</span><a href="{{$s.URI}}?o=v">[v]</a>
     </td>
   </tr>
   <tr class="spacer"><td colspan="2">

pkg/srv/templates/head.html

@@ -94,6 +94,9 @@ td {
 .green {
   background-color: #4caf50;
 }
+.status-red {
+  color: #e74c3c;
+}
 .status-yellow {
   color: #f39c12;
 }

pkg/srv/templates/index.html

@@ -5,13 +5,13 @@
 <a href="/?o=preloads">[Preloads]</a></pre>
 <table>
   <tr class="listitem">
-    <td class="listpath up"><a href="../">../</a></td>
+    <td class="listpath up"><a href="../{{.PathAnchor}}">../</a></td>
     <td class="listoptions">[dir]</td>
   </tr>
   <tr class="spacer"><td colspan="2"></td></tr>
 {{range $s := .Paths.Dirs -}}
   <tr class="listitem">
-    <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
+    <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
     <td class="listoptions">[dir]</td>
   </tr>
   <tr class="spacer"><td colspan="2"></td></tr>
@@ -20,7 +20,7 @@
   <tr class="listitem">
     <td class="listpath"><a id="{{$s.Anchor}}" href="{{$s.URI}}">{{$s.Name}}</a></td>
     <td class="listoptions">
-      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a>
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span>{{$s.Size}}</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.URI}}?o=n">[n]</a><a href="{{$s.URI}}?o=p">[p]</a>
     </td>
   </tr>
   <tr class="spacer"><td colspan="2">

pkg/srv/templates/preloads.html

@@ -3,7 +3,7 @@
 [+]: increase priority
 [-]: decrease priority
 [s]: stop preloading
-Quota: {{.QuotaCur}} / {{.QuotaMax}} GiB
+Quota: {{.QuotaCur}} ({{.QuotaEst}}) / {{.QuotaMax}} GiB
 </pre>
 <table>
   <tr class="listitem">
@@ -14,7 +14,19 @@ Quota: {{.QuotaCur}} / {{.QuotaMax}} GiB
   <tr class="listitem">
     <td class="listpath"><a href="{{$s.URI}}">{{$s.Name}}</a></td>
     <td class="listoptions">
-      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}<span class="status-{{if $s.Running}}green">[running{{else}}yellow">[queued{{end}}]</span><span>{{$s.Prio}}</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s&r=preloads">[s]</a><a href="{{$s.Name}}?o=i&r=preloads">[+]</a><a href="{{$s.Name}}?o=d&r=preloads">[-]</a>
+      {{if ge $s.Status 0}}<span>{{$s.Status}}%</span>{{end}}
+{{if $s.Running -}}
+      <span class="status-green">[running]</span>
+{{else -}}
+{{if ge $s.Errc 10 -}}
+      <span class="status-red">[error]</span>
+{{else -}}
+      <span class="status-yellow">[queued]</span>
+{{end -}}
+{{end -}}
+      <span>{{$s.Rate}}</span>
+      <span>{{$s.Size}}</span>
+      <span>{{$s.Prio}}</span><a href="{{$s.URI}}?o=v">[v]</a><a href="{{$s.Name}}?o=s&r=preloads">[s]</a><a href="{{$s.Name}}?o=i&r=preloads">[+]</a><a href="{{$s.Name}}?o=d&r=preloads">[-]</a>
     </td>
   </tr>
   <tr class="spacer"><td colspan="2">

pkg/srv/webdav.go

@@ -4,12 +4,12 @@ package srv
 
 import (
 	"net/http"
+	"slices"
 	"strings"
 
 	"cachefs/pkg/fs"
 
 	"github.com/go-logr/logr"
-	"golang.org/x/exp/slices"
 	"golang.org/x/net/webdav"
 )
 

vendor/git.giftfish.de/ston1th/godrop/v2/.gitignore

@@ -0,0 +1,3 @@
+port80
+port80fg
+port80and443

vendor/git.giftfish.de/ston1th/godrop/v2/LICENSE

@@ -0,0 +1,24 @@
+Copyright (C) 2022 Marius Schellenberger
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * The names of the authors and/or contributors may not be used to
+      endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL ston1th BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/git.giftfish.de/ston1th/godrop/v2/README.md

@@ -0,0 +1,5 @@
+# godrop - drop privileges
+
+Godrop is a simple library to drop privileges on Linux and OpenBSD.
+
+See the examples directory on how to use the `Drop` and `MultiDrop` functions.

vendor/git.giftfish.de/ston1th/godrop/v2/godrop.go

@@ -0,0 +1,137 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build go1.11
+// +build go1.11
+
+// Package godrop provides a simple library to drop privileges on Linux and OpenBSD.
+package godrop
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
+
+// Config represents the drop config
+type Config struct {
+	// User is the user to drop privileges to.
+	User string
+	// Group is the group to drop privileges to.
+	Group string
+	// Chroot is the directory to chroot into. Leave this emptry for no chroot.
+	// When compiling without cgo, make sure the chroot directory contains the /etc/passwd and /etc/group files.
+	Chroot string
+	// Set to true, to run the process in the foreground.
+	Foreground bool
+}
+
+// Drop will spawn a new process and hand over the listening socket file descriptor
+func Drop(c Config, f func() (net.Listener, error)) error {
+	return MultiDrop(c, func() ([]net.Listener, error) {
+		l, err := f()
+		return []net.Listener{l}, err
+	})
+}
+
+// MultiDrop will spawn a new process and hand over the all listening sockets
+func MultiDrop(c Config, f func() ([]net.Listener, error)) error {
+	uid, err := userID(c.User)
+	if err != nil {
+		return errors.New("godrop: " + err.Error())
+	}
+	if uid == 0 {
+		return fmt.Errorf("godrop: you can't drop privileges to uid 0 (%s)", c.User)
+	}
+	gid, err := groupID(c.Group)
+	if err != nil {
+		return errors.New("godrop: " + err.Error())
+	}
+	switch os.Getuid() {
+	case 0:
+		cmd := exec.Command(os.Args[0], os.Args[1:]...)
+		ln, err := f()
+		if err != nil {
+			return errors.New("godrop: " + err.Error())
+		}
+		for i, v := range ln {
+			var f *os.File
+			switch l := v.(type) {
+			case *net.TCPListener:
+				f, err = l.File()
+				l.Close()
+			case *net.UnixListener:
+				f, err = l.File()
+				l.Close()
+			default:
+				return fmt.Errorf("godrop: index %d listener is not type of either *net.TCPListener or *net.UnixListener", i)
+			}
+			if err != nil {
+				return fmt.Errorf("godrop: index %d %s", i, err)
+			}
+			cmd.ExtraFiles = append(cmd.ExtraFiles, f)
+		}
+
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Chroot: c.Chroot,
+			Credential: &syscall.Credential{
+				Uid: uint32(uid),
+				Gid: uint32(gid),
+			},
+			Setsid: true,
+		}
+
+		if c.Foreground {
+			cmd.Stdin = os.Stdin
+			cmd.Stdout = os.Stdout
+			cmd.Stderr = os.Stderr
+		}
+
+		if err := cmd.Start(); err != nil {
+			return errors.New("godrop: " + err.Error())
+		}
+
+		if c.Foreground {
+			go func() {
+				term := make(chan os.Signal)
+				signal.Notify(term, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+				sigs := make(chan os.Signal)
+				signal.Notify(sigs, syscall.SIGSTOP, syscall.SIGCONT, syscall.SIGUSR1, syscall.SIGUSR2)
+				for {
+					select {
+					case sig := <-term:
+						cmd.Process.Signal(sig)
+						return
+					case sig := <-sigs:
+						cmd.Process.Signal(sig)
+					}
+				}
+			}()
+			_ = cmd.Wait()
+			os.Exit(int(cmd.ProcessState.Sys().(syscall.WaitStatus)))
+		}
+		cmd.Process.Release()
+		os.Exit(0)
+	case uid:
+		return nil
+	}
+	return errors.New("godrop: dropping priviledges failed")
+}
+
+// GetListener returns the listener socket of file descriptor 3
+func GetListener() (net.Listener, error) {
+	return GetListenerFd(3)
+}
+
+// GetListenerFd returns the listener socket of the given file descriptor
+func GetListenerFd(fd int) (net.Listener, error) {
+	if fd < 3 {
+		return nil, errors.New("godrop: fd is less than 3")
+	}
+	f := os.NewFile(uintptr(fd), "")
+	defer f.Close()
+	return net.FileListener(f)
+}

vendor/git.giftfish.de/ston1th/godrop/v2/lookup.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build go1.11
+// +build go1.11
+
+package godrop
+
+import (
+	"os/user"
+	"strconv"
+)
+
+func atoi(a string) (int, error) {
+	i, err := strconv.Atoi(a)
+	if err != nil {
+		return -1, err
+	}
+	return i, nil
+}
+
+func userID(username string) (int, error) {
+	u, err := user.Lookup(username)
+	if err != nil {
+		return -1, err
+	}
+	return atoi(u.Uid)
+}
+
+func groupID(name string) (int, error) {
+	g, err := user.LookupGroup(name)
+	if err != nil {
+		return -1, err
+	}
+	return atoi(g.Gid)
+}

vendor/git.giftfish.de/ston1th/godrop/v2/pledge.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build !openbsd
+// +build !openbsd
+
+package godrop
+
+// Pledge is currently only supported on OpenBSD.
+func Pledge(promises, execpromises string) error { return nil }
+
+// PledgePromises is currently only supported on OpenBSD.
+func PledgePromises(promises string) error { return nil }
+
+// PledgeExecPromises is currently only supported on OpenBSD.
+func PledgeExecpromises(execpromises string) error { return nil }

vendor/git.giftfish.de/ston1th/godrop/v2/pledge_openbsd.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build openbsd
+// +build openbsd
+
+package godrop
+
+import (
+	"errors"
+	"golang.org/x/sys/unix"
+)
+
+// Pledge is a wrapper for x/sys/unix Pledge.
+//
+// See https://go.googlesource.com/sys/+/master/unix/openbsd_pledge.go for usage.
+func Pledge(promises, execpromises string) (err error) {
+	err = unix.Pledge(promises, execpromises)
+	if err != nil {
+		err = errors.New("pledge: " + err.Error())
+	}
+	return
+}
+
+// PledgePromises is a wrapper for x/sys/unix PledgePromises.
+//
+// See https://go.googlesource.com/sys/+/master/unix/openbsd_pledge.go for usage.
+func PledgePromises(promises string) (err error) {
+	err = unix.PledgePromises(promises)
+	if err != nil {
+		err = errors.New("pledge: " + err.Error())
+	}
+	return
+}
+
+// PledgeExecpromises is a wrapper for x/sys/unix PledgeExecpromises.
+//
+// See https://go.googlesource.com/sys/+/master/unix/openbsd_pledge.go for usage.
+func PledgeExecpromises(execpromises string) (err error) {
+	err = unix.PledgeExecpromises(execpromises)
+	if err != nil {
+		err = errors.New("pledge: " + err.Error())
+	}
+	return
+}

vendor/git.giftfish.de/ston1th/godrop/v2/unveil.go

@@ -0,0 +1,12 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build !openbsd
+// +build !openbsd
+
+package godrop
+
+// Unveil is currently only supported on OpenBSD.
+func Unveil(path, flags string) error { return nil }
+
+// UnveilBlock is currently only supported on OpenBSD.
+func UnveilBlock() error { return nil }

vendor/git.giftfish.de/ston1th/godrop/v2/unveil_openbsd.go

@@ -0,0 +1,33 @@
+// Copyright (C) 2022 Marius Schellenberger
+
+//go:build openbsd
+// +build openbsd
+
+package godrop
+
+import (
+	"errors"
+	"golang.org/x/sys/unix"
+)
+
+// Unveil is a wrapper for x/sys/unix Unveil.
+//
+// See https://go.googlesource.com/sys/+/master/unix/openbsd_unveil.go for usage.
+func Unveil(path, flags string) (err error) {
+	err = unix.Unveil(path, flags)
+	if err != nil {
+		err = errors.New("unveil: " + err.Error())
+	}
+	return
+}
+
+// UnveilBlock is a wrapper for x/sys/unix UnveilBlock.
+//
+// See https://go.googlesource.com/sys/+/master/unix/openbsd_unveil.go for usage.
+func UnveilBlock() (err error) {
+	err = unix.UnveilBlock()
+	if err != nil {
+		err = errors.New("unveil: " + err.Error())
+	}
+	return
+}

vendor/github.com/go-logr/logr/.golangci.yaml

@@ -6,7 +6,6 @@ linters:
   disable-all: true
   enable:
     - asciicheck
-    - deadcode
     - errcheck
     - forcetypeassert
     - gocritic
@@ -18,10 +17,8 @@ linters:
     - misspell
     - revive
     - staticcheck
-    - structcheck
     - typecheck
     - unused
-    - varcheck
 
 issues:
   exclude-use-default: false

vendor/github.com/go-logr/logr/README.md

@@ -105,14 +105,18 @@ with higher verbosity means more (and less important) logs will be generated.
 There are implementations for the following logging libraries:
 
 - **a function** (can bridge to non-structured libraries): [funcr](https://github.com/go-logr/logr/tree/master/funcr)
+- **a testing.T** (for use in Go tests, with JSON-like output): [testr](https://github.com/go-logr/logr/tree/master/testr)
 - **github.com/google/glog**: [glogr](https://github.com/go-logr/glogr)
 - **k8s.io/klog** (for Kubernetes): [klogr](https://git.k8s.io/klog/klogr)
+- **a testing.T** (with klog-like text output): [ktesting](https://git.k8s.io/klog/ktesting)
 - **go.uber.org/zap**: [zapr](https://github.com/go-logr/zapr)
 - **log** (the Go standard library logger): [stdr](https://github.com/go-logr/stdr)
 - **github.com/sirupsen/logrus**: [logrusr](https://github.com/bombsimon/logrusr)
 - **github.com/wojas/genericr**: [genericr](https://github.com/wojas/genericr) (makes it easy to implement your own backend)
 - **logfmt** (Heroku style [logging](https://www.brandur.org/logfmt)): [logfmtr](https://github.com/iand/logfmtr)
 - **github.com/rs/zerolog**: [zerologr](https://github.com/go-logr/zerologr)
+- **github.com/go-kit/log**: [gokitlogr](https://github.com/tonglil/gokitlogr) (also compatible with github.com/go-kit/kit/log since v0.12.0)
+- **bytes.Buffer** (writing to a buffer): [bufrlogr](https://github.com/tonglil/buflogr) (useful for ensuring values were logged, like during testing)
 
 ## FAQ
 

vendor/github.com/go-logr/logr/discard.go

@@ -20,35 +20,5 @@ package logr
 // used whenever the caller is not interested in the logs.  Logger instances
 // produced by this function always compare as equal.
 func Discard() Logger {
-	return Logger{
+	return New(nil)
-		level: 0,
-		sink:  discardLogSink{},
-	}
-}
-
-// discardLogSink is a LogSink that discards all messages.
-type discardLogSink struct{}
-
-// Verify that it actually implements the interface
-var _ LogSink = discardLogSink{}
-
-func (l discardLogSink) Init(RuntimeInfo) {
-}
-
-func (l discardLogSink) Enabled(int) bool {
-	return false
-}
-
-func (l discardLogSink) Info(int, string, ...interface{}) {
-}
-
-func (l discardLogSink) Error(error, string, ...interface{}) {
-}
-
-func (l discardLogSink) WithValues(...interface{}) LogSink {
-	return l
-}
-
-func (l discardLogSink) WithName(string) LogSink {
-	return l
 }

vendor/github.com/go-logr/logr/logr.go

@@ -21,7 +21,7 @@ limitations under the License.
 // to back that API.  Packages in the Go ecosystem can depend on this package,
 // while callers can implement logging with whatever backend is appropriate.
 //
-// Usage
+// # Usage
 //
 // Logging is done using a Logger instance.  Logger is a concrete type with
 // methods, which defers the actual logging to a LogSink interface.  The main
@@ -30,16 +30,20 @@ limitations under the License.
 // "structured logging".
 //
 // With Go's standard log package, we might write:
-//   log.Printf("setting target value %s", targetValue)
+//
+//	log.Printf("setting target value %s", targetValue)
 //
 // With logr's structured logging, we'd write:
-//   logger.Info("setting target", "value", targetValue)
+//
+//	logger.Info("setting target", "value", targetValue)
 //
 // Errors are much the same.  Instead of:
-//   log.Printf("failed to open the pod bay door for user %s: %v", user, err)
+//
+//	log.Printf("failed to open the pod bay door for user %s: %v", user, err)
 //
 // We'd write:
-//   logger.Error(err, "failed to open the pod bay door", "user", user)
+//
+//	logger.Error(err, "failed to open the pod bay door", "user", user)
 //
 // Info() and Error() are very similar, but they are separate methods so that
 // LogSink implementations can choose to do things like attach additional
@@ -47,7 +51,7 @@ limitations under the License.
 // always logged, regardless of the current verbosity.  If there is no error
 // instance available, passing nil is valid.
 //
-// Verbosity
+// # Verbosity
 //
 // Often we want to log information only when the application in "verbose
 // mode".  To write log lines that are more verbose, Logger has a V() method.
@@ -58,20 +62,22 @@ limitations under the License.
 // Error messages do not have a verbosity level and are always logged.
 //
 // Where we might have written:
-//   if flVerbose >= 2 {
+//
-//       log.Printf("an unusual thing happened")
+//	if flVerbose >= 2 {
-//   }
+//	    log.Printf("an unusual thing happened")
+//	}
 //
 // We can write:
-//   logger.V(2).Info("an unusual thing happened")
 //
-// Logger Names
+//	logger.V(2).Info("an unusual thing happened")
+//
+// # Logger Names
 //
 // Logger instances can have name strings so that all messages logged through
 // that instance have additional context.  For example, you might want to add
 // a subsystem name:
 //
-//   logger.WithName("compactor").Info("started", "time", time.Now())
+//	logger.WithName("compactor").Info("started", "time", time.Now())
 //
 // The WithName() method returns a new Logger, which can be passed to
 // constructors or other functions for further use.  Repeated use of WithName()
@@ -82,25 +88,27 @@ limitations under the License.
 // joining operation (e.g. whitespace, commas, periods, slashes, brackets,
 // quotes, etc).
 //
-// Saved Values
+// # Saved Values
 //
 // Logger instances can store any number of key/value pairs, which will be
 // logged alongside all messages logged through that instance.  For example,
 // you might want to create a Logger instance per managed object:
 //
 // With the standard log package, we might write:
-//   log.Printf("decided to set field foo to value %q for object %s/%s",
+//
-//       targetValue, object.Namespace, object.Name)
+//	log.Printf("decided to set field foo to value %q for object %s/%s",
+//	    targetValue, object.Namespace, object.Name)
 //
 // With logr we'd write:
-//   // Elsewhere: set up the logger to log the object name.
-//   obj.logger = mainLogger.WithValues(
-//       "name", obj.name, "namespace", obj.namespace)
 //
-//   // later on...
+//	// Elsewhere: set up the logger to log the object name.
-//   obj.logger.Info("setting foo", "value", targetValue)
+//	obj.logger = mainLogger.WithValues(
+//	    "name", obj.name, "namespace", obj.namespace)
 //
-// Best Practices
+//	// later on...
+//	obj.logger.Info("setting foo", "value", targetValue)
+//
+// # Best Practices
 //
 // Logger has very few hard rules, with the goal that LogSink implementations
 // might have a lot of freedom to differentiate.  There are, however, some
@@ -115,15 +123,24 @@ limitations under the License.
 // may be any Go value, but how the value is formatted is determined by the
 // LogSink implementation.
 //
-// Key Naming Conventions
+// Logger instances are meant to be passed around by value. Code that receives
+// such a value can call its methods without having to check whether the
+// instance is ready for use.
+//
+// Calling methods with the null logger (Logger{}) as instance will crash
+// because it has no LogSink. Therefore this null logger should never be passed
+// around. For cases where passing a logger is optional, a pointer to Logger
+// should be used.
+//
+// # Key Naming Conventions
 //
 // Keys are not strictly required to conform to any specification or regex, but
 // it is recommended that they:
-//   * be human-readable and meaningful (not auto-generated or simple ordinals)
+//   - be human-readable and meaningful (not auto-generated or simple ordinals)
-//   * be constant (not dependent on input data)
+//   - be constant (not dependent on input data)
-//   * contain only printable characters
+//   - contain only printable characters
-//   * not contain whitespace or punctuation
+//   - not contain whitespace or punctuation
-//   * use lower case for simple keys and lowerCamelCase for more complex ones
+//   - use lower case for simple keys and lowerCamelCase for more complex ones
 //
 // These guidelines help ensure that log data is processed properly regardless
 // of the log implementation.  For example, log implementations will try to
@@ -132,51 +149,54 @@ limitations under the License.
 // While users are generally free to use key names of their choice, it's
 // generally best to avoid using the following keys, as they're frequently used
 // by implementations:
-//   * "caller": the calling information (file/line) of a particular log line
+//   - "caller": the calling information (file/line) of a particular log line
-//   * "error": the underlying error value in the `Error` method
+//   - "error": the underlying error value in the `Error` method
-//   * "level": the log level
+//   - "level": the log level
-//   * "logger": the name of the associated logger
+//   - "logger": the name of the associated logger
-//   * "msg": the log message
+//   - "msg": the log message
-//   * "stacktrace": the stack trace associated with a particular log line or
+//   - "stacktrace": the stack trace associated with a particular log line or
-//                   error (often from the `Error` message)
+//     error (often from the `Error` message)
-//   * "ts": the timestamp for a log line
+//   - "ts": the timestamp for a log line
 //
 // Implementations are encouraged to make use of these keys to represent the
 // above concepts, when necessary (for example, in a pure-JSON output form, it
 // would be necessary to represent at least message and timestamp as ordinary
 // named values).
 //
-// Break Glass
+// # Break Glass
 //
 // Implementations may choose to give callers access to the underlying
 // logging implementation.  The recommended pattern for this is:
-//   // Underlier exposes access to the underlying logging implementation.
+//
-//   // Since callers only have a logr.Logger, they have to know which
+//	// Underlier exposes access to the underlying logging implementation.
-//   // implementation is in use, so this interface is less of an abstraction
+//	// Since callers only have a logr.Logger, they have to know which
-//   // and more of way to test type conversion.
+//	// implementation is in use, so this interface is less of an abstraction
-//   type Underlier interface {
+//	// and more of way to test type conversion.
-//       GetUnderlying() <underlying-type>
+//	type Underlier interface {
-//   }
+//	    GetUnderlying() <underlying-type>
+//	}
 //
 // Logger grants access to the sink to enable type assertions like this:
-//   func DoSomethingWithImpl(log logr.Logger) {
+//
-//       if underlier, ok := log.GetSink()(impl.Underlier) {
+//	func DoSomethingWithImpl(log logr.Logger) {
-//          implLogger := underlier.GetUnderlying()
+//	    if underlier, ok := log.GetSink().(impl.Underlier); ok {
-//          ...
+//	       implLogger := underlier.GetUnderlying()
-//       }
+//	       ...
-//   }
+//	    }
+//	}
 //
 // Custom `With*` functions can be implemented by copying the complete
 // Logger struct and replacing the sink in the copy:
-//   // WithFooBar changes the foobar parameter in the log sink and returns a
+//
-//   // new logger with that modified sink.  It does nothing for loggers where
+//	// WithFooBar changes the foobar parameter in the log sink and returns a
-//   // the sink doesn't support that parameter.
+//	// new logger with that modified sink.  It does nothing for loggers where
-//   func WithFoobar(log logr.Logger, foobar int) logr.Logger {
+//	// the sink doesn't support that parameter.
-//      if foobarLogSink, ok := log.GetSink()(FoobarSink); ok {
+//	func WithFoobar(log logr.Logger, foobar int) logr.Logger {
-//         log = log.WithSink(foobarLogSink.WithFooBar(foobar))
+//	   if foobarLogSink, ok := log.GetSink().(FoobarSink); ok {
-//      }
+//	      log = log.WithSink(foobarLogSink.WithFooBar(foobar))
-//      return log
+//	   }
-//   }
+//	   return log
+//	}
 //
 // Don't use New to construct a new Logger with a LogSink retrieved from an
 // existing Logger. Source code attribution might not work correctly and
@@ -192,11 +212,14 @@ import (
 )
 
 // New returns a new Logger instance.  This is primarily used by libraries
-// implementing LogSink, rather than end users.
+// implementing LogSink, rather than end users.  Passing a nil sink will create
+// a Logger which discards all log lines.
 func New(sink LogSink) Logger {
 	logger := Logger{}
 	logger.setSink(sink)
-	sink.Init(runtimeInfo)
+	if sink != nil {
+		sink.Init(runtimeInfo)
+	}
 	return logger
 }
 
@@ -235,7 +258,7 @@ type Logger struct {
 // Enabled tests whether this Logger is enabled.  For example, commandline
 // flags might be used to set the logging verbosity and disable some info logs.
 func (l Logger) Enabled() bool {
-	return l.sink.Enabled(l.level)
+	return l.sink != nil && l.sink.Enabled(l.level)
 }
 
 // Info logs a non-error message with the given key/value pairs as context.
@@ -245,6 +268,9 @@ func (l Logger) Enabled() bool {
 // information.  The key/value pairs must alternate string keys and arbitrary
 // values.
 func (l Logger) Info(msg string, keysAndValues ...interface{}) {
+	if l.sink == nil {
+		return
+	}
 	if l.Enabled() {
 		if withHelper, ok := l.sink.(CallStackHelperLogSink); ok {
 			withHelper.GetCallStackHelper()()
@@ -264,6 +290,9 @@ func (l Logger) Info(msg string, keysAndValues ...interface{}) {
 // triggered this log line, if present. The err parameter is optional
 // and nil may be passed instead of an error instance.
 func (l Logger) Error(err error, msg string, keysAndValues ...interface{}) {
+	if l.sink == nil {
+		return
+	}
 	if withHelper, ok := l.sink.(CallStackHelperLogSink); ok {
 		withHelper.GetCallStackHelper()()
 	}
@@ -275,6 +304,9 @@ func (l Logger) Error(err error, msg string, keysAndValues ...interface{}) {
 // level means a log message is less important.  Negative V-levels are treated
 // as 0.
 func (l Logger) V(level int) Logger {
+	if l.sink == nil {
+		return l
+	}
 	if level < 0 {
 		level = 0
 	}
@@ -285,6 +317,9 @@ func (l Logger) V(level int) Logger {
 // WithValues returns a new Logger instance with additional key/value pairs.
 // See Info for documentation on how key/value pairs work.
 func (l Logger) WithValues(keysAndValues ...interface{}) Logger {
+	if l.sink == nil {
+		return l
+	}
 	l.setSink(l.sink.WithValues(keysAndValues...))
 	return l
 }
@@ -295,6 +330,9 @@ func (l Logger) WithValues(keysAndValues ...interface{}) Logger {
 // contain only letters, digits, and hyphens (see the package documentation for
 // more information).
 func (l Logger) WithName(name string) Logger {
+	if l.sink == nil {
+		return l
+	}
 	l.setSink(l.sink.WithName(name))
 	return l
 }
@@ -315,6 +353,9 @@ func (l Logger) WithName(name string) Logger {
 // WithCallDepth(1) because it works with implementions that support the
 // CallDepthLogSink and/or CallStackHelperLogSink interfaces.
 func (l Logger) WithCallDepth(depth int) Logger {
+	if l.sink == nil {
+		return l
+	}
 	if withCallDepth, ok := l.sink.(CallDepthLogSink); ok {
 		l.setSink(withCallDepth.WithCallDepth(depth))
 	}
@@ -336,6 +377,9 @@ func (l Logger) WithCallDepth(depth int) Logger {
 // implementation does not support either of these, the original Logger will be
 // returned.
 func (l Logger) WithCallStackHelper() (func(), Logger) {
+	if l.sink == nil {
+		return func() {}, l
+	}
 	var helper func()
 	if withCallDepth, ok := l.sink.(CallDepthLogSink); ok {
 		l.setSink(withCallDepth.WithCallDepth(1))
@@ -348,6 +392,11 @@ func (l Logger) WithCallStackHelper() (func(), Logger) {
 	return helper, l
 }
 
+// IsZero returns true if this logger is an uninitialized zero value
+func (l Logger) IsZero() bool {
+	return l.sink == nil
+}
+
 // contextKey is how we find Loggers in a context.Context.
 type contextKey struct{}
 
@@ -433,7 +482,7 @@ type LogSink interface {
 	WithName(name string) LogSink
 }
 
-// CallDepthLogSink represents a Logger that knows how to climb the call stack
+// CallDepthLogSink represents a LogSink that knows how to climb the call stack
 // to identify the original call site and can offset the depth by a specified
 // number of frames.  This is useful for users who have helper functions
 // between the "real" call site and the actual calls to Logger methods.
@@ -458,7 +507,7 @@ type CallDepthLogSink interface {
 	WithCallDepth(depth int) LogSink
 }
 
-// CallStackHelperLogSink represents a Logger that knows how to climb
+// CallStackHelperLogSink represents a LogSink that knows how to climb
 // the call stack to identify the original call site and can skip
 // intermediate helper functions if they mark themselves as
 // helper. Go's testing package uses that approach.

vendor/github.com/pkg/sftp/attrs.go

@@ -1,7 +1,7 @@
 package sftp
 
 // ssh_FXP_ATTRS support
-// see http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-5
+// see https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5
 
 import (
 	"os"
@@ -69,6 +69,20 @@ func fileInfoFromStat(stat *FileStat, name string) os.FileInfo {
 	}
 }
 
+// FileInfoUidGid extends os.FileInfo and adds callbacks for Uid and Gid retrieval,
+// as an alternative to *syscall.Stat_t objects on unix systems.
+type FileInfoUidGid interface {
+	os.FileInfo
+	Uid() uint32
+	Gid() uint32
+}
+
+// FileInfoUidGid extends os.FileInfo and adds a callbacks for extended data retrieval.
+type FileInfoExtendedData interface {
+	os.FileInfo
+	Extended() []StatExtended
+}
+
 func fileStatFromInfo(fi os.FileInfo) (uint32, *FileStat) {
 	mtime := fi.ModTime().Unix()
 	atime := mtime
@@ -86,5 +100,22 @@ func fileStatFromInfo(fi os.FileInfo) (uint32, *FileStat) {
 	// os specific file stat decoding
 	fileStatFromInfoOs(fi, &flags, fileStat)
 
+	// The call above will include the sshFileXferAttrUIDGID in case
+	// the os.FileInfo can be casted to *syscall.Stat_t on unix.
+	// If fi implements FileInfoUidGid, retrieve Uid, Gid from it instead.
+	if fiExt, ok := fi.(FileInfoUidGid); ok {
+		flags |= sshFileXferAttrUIDGID
+		fileStat.UID = fiExt.Uid()
+		fileStat.GID = fiExt.Gid()
+	}
+
+	// if fi implements FileInfoExtendedData, retrieve extended data from it
+	if fiExt, ok := fi.(FileInfoExtendedData); ok {
+		fileStat.Extended = fiExt.Extended()
+		if len(fileStat.Extended) > 0 {
+			flags |= sshFileXferAttrExtended
+		}
+	}
+
 	return flags, fileStat
 }

vendor/github.com/pkg/sftp/attrs_stubs.go

@@ -1,3 +1,4 @@
+//go:build plan9 || windows || android
 // +build plan9 windows android
 
 package sftp

vendor/github.com/pkg/sftp/attrs_unix.go

@@ -1,3 +1,4 @@
+//go:build darwin || dragonfly || freebsd || (!android && linux) || netbsd || openbsd || solaris || aix || js
 // +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris aix js
 
 package sftp

vendor/github.com/pkg/sftp/client.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 	"math"
 	"os"
@@ -226,15 +227,22 @@ func NewClientPipe(rd io.Reader, wr io.WriteCloser, opts ...ClientOption) (*Clie
 
 	if err := sftp.sendInit(); err != nil {
 		wr.Close()
-		return nil, err
+		return nil, fmt.Errorf("error sending init packet to server: %w", err)
 	}
+
 	if err := sftp.recvVersion(); err != nil {
 		wr.Close()
-		return nil, err
+		return nil, fmt.Errorf("error receiving version packet from server: %w", err)
 	}
 
 	sftp.clientConn.wg.Add(1)
-	go sftp.loop()
+	go func() {
+		defer sftp.clientConn.wg.Done()
+
+		if err := sftp.clientConn.recv(); err != nil {
+			sftp.clientConn.broadcastErr(err)
+		}
+	}()
 
 	return sftp, nil
 }
@@ -251,11 +259,11 @@ func (c *Client) Create(path string) (*File, error) {
 	return c.open(path, flags(os.O_RDWR|os.O_CREATE|os.O_TRUNC))
 }
 
-const sftpProtocolVersion = 3 // http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
+const sftpProtocolVersion = 3 // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
 
 func (c *Client) sendInit() error {
 	return c.clientConn.conn.sendPacket(&sshFxInitPacket{
-		Version: sftpProtocolVersion, // http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
+		Version: sftpProtocolVersion, // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
 	})
 }
 
@@ -267,8 +275,13 @@ func (c *Client) nextID() uint32 {
 func (c *Client) recvVersion() error {
 	typ, data, err := c.recvPacket(0)
 	if err != nil {
+		if err == io.EOF {
+			return fmt.Errorf("server unexpectedly closed connection: %w", io.ErrUnexpectedEOF)
+		}
+
 		return err
 	}
+
 	if typ != sshFxpVersion {
 		return &unexpectedPacketErr{sshFxpVersion, typ}
 	}
@@ -277,6 +290,7 @@ func (c *Client) recvVersion() error {
 	if err != nil {
 		return err
 	}
+
 	if version != sftpProtocolVersion {
 		return &unexpectedVersionErr{sftpProtocolVersion, version}
 	}
@@ -910,6 +924,45 @@ func (c *Client) MkdirAll(path string) error {
 	return nil
 }
 
+// RemoveAll delete files recursively in the directory and Recursively delete subdirectories.
+// An error will be returned if no file or directory with the specified path exists
+func (c *Client) RemoveAll(path string) error {
+
+	// Get the file/directory information
+	fi, err := c.Stat(path)
+	if err != nil {
+		return err
+	}
+
+	if fi.IsDir() {
+		// Delete files recursively in the directory
+		files, err := c.ReadDir(path)
+		if err != nil {
+			return err
+		}
+
+		for _, file := range files {
+			if file.IsDir() {
+				// Recursively delete subdirectories
+				err = c.RemoveAll(path + "/" + file.Name())
+				if err != nil {
+					return err
+				}
+			} else {
+				// Delete individual files
+				err = c.Remove(path + "/" + file.Name())
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+	}
+
+	return c.Remove(path)
+
+}
+
 // File represents a remote file.
 type File struct {
 	c      *Client
@@ -999,9 +1052,6 @@ func (f *File) readAtSequential(b []byte, off int64) (read int, err error) {
 			read += n
 		}
 		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return read, nil // return nil explicitly.
-			}
 			return read, err
 		}
 	}
@@ -1179,11 +1229,11 @@ func (f *File) writeToSequential(w io.Writer) (written int64, err error) {
 		if n > 0 {
 			f.offset += int64(n)
 
-			m, err2 := w.Write(b[:n])
+			m, err := w.Write(b[:n])
 			written += int64(m)
 
-			if err == nil {
+			if err != nil {
-				err = err2
+				return written, err
 			}
 		}
 
@@ -1461,11 +1511,20 @@ func (f *File) writeAtConcurrent(b []byte, off int64) (int, error) {
 	cancel := make(chan struct{})
 
 	type work struct {
-		b   []byte
+		id  uint32
+		res chan result
+
 		off int64
 	}
 	workCh := make(chan work)
 
+	concurrency := len(b)/f.c.maxPacket + 1
+	if concurrency > f.c.maxConcurrentRequests || concurrency < 1 {
+		concurrency = f.c.maxConcurrentRequests
+	}
+
+	pool := newResChanPool(concurrency)
+
 	// Slice: cut up the Read into any number of buffers of length <= f.c.maxPacket, and at appropriate offsets.
 	go func() {
 		defer close(workCh)
@@ -1479,8 +1538,20 @@ func (f *File) writeAtConcurrent(b []byte, off int64) (int, error) {
 				wb = wb[:chunkSize]
 			}
 
+			id := f.c.nextID()
+			res := pool.Get()
+			off := off + int64(read)
+
+			f.c.dispatchRequest(res, &sshFxpWritePacket{
+				ID:     id,
+				Handle: f.handle,
+				Offset: uint64(off),
+				Length: uint32(len(wb)),
+				Data:   wb,
+			})
+
 			select {
-			case workCh <- work{wb, off + int64(read)}:
+			case workCh <- work{id, res, off}:
 			case <-cancel:
 				return
 			}
@@ -1495,11 +1566,6 @@ func (f *File) writeAtConcurrent(b []byte, off int64) (int, error) {
 	}
 	errCh := make(chan wErr)
 
-	concurrency := len(b)/f.c.maxPacket + 1
-	if concurrency > f.c.maxConcurrentRequests || concurrency < 1 {
-		concurrency = f.c.maxConcurrentRequests
-	}
-
 	var wg sync.WaitGroup
 	wg.Add(concurrency)
 	for i := 0; i < concurrency; i++ {
@@ -1507,13 +1573,22 @@ func (f *File) writeAtConcurrent(b []byte, off int64) (int, error) {
 		go func() {
 			defer wg.Done()
 
-			ch := make(chan result, 1) // reusable channel per mapper.
+			for work := range workCh {
+				s := <-work.res
+				pool.Put(work.res)
+
+				err := s.err
+				if err == nil {
+					switch s.typ {
+					case sshFxpStatus:
+						err = normaliseError(unmarshalStatus(work.id, s.data))
+					default:
+						err = unimplementedPacketErr(s.typ)
+					}
+				}
 
-			for packet := range workCh {
-				n, err := f.writeChunkAt(ch, packet.b, packet.off)
 				if err != nil {
-					// return the offset as the start + how much we wrote before the error.
+					errCh <- wErr{work.off, err}
-					errCh <- wErr{packet.off + int64(n), err}
 				}
 			}
 		}()
@@ -1598,8 +1673,9 @@ func (f *File) ReadFromWithConcurrency(r io.Reader, concurrency int) (read int64
 	cancel := make(chan struct{})
 
 	type work struct {
-		b   []byte
+		id  uint32
-		n   int
+		res chan result
+
 		off int64
 	}
 	workCh := make(chan work)
@@ -1614,24 +1690,34 @@ func (f *File) ReadFromWithConcurrency(r io.Reader, concurrency int) (read int64
 		concurrency = f.c.maxConcurrentRequests
 	}
 
-	pool := newBufPool(concurrency, f.c.maxPacket)
+	pool := newResChanPool(concurrency)
 
 	// Slice: cut up the Read into any number of buffers of length <= f.c.maxPacket, and at appropriate offsets.
 	go func() {
 		defer close(workCh)
 
+		b := make([]byte, f.c.maxPacket)
 		off := f.offset
 
 		for {
-			b := pool.Get()
-
 			n, err := r.Read(b)
+
 			if n > 0 {
 				read += int64(n)
 
+				id := f.c.nextID()
+				res := pool.Get()
+
+				f.c.dispatchRequest(res, &sshFxpWritePacket{
+					ID:     id,
+					Handle: f.handle,
+					Offset: uint64(off),
+					Length: uint32(n),
+					Data:   b[:n],
+				})
+
 				select {
-				case workCh <- work{b, n, off}:
+				case workCh <- work{id, res, off}:
-					// We need the pool.Put(b) to put the whole slice, not just trunced.
 				case <-cancel:
 					return
 				}
@@ -1655,15 +1741,23 @@ func (f *File) ReadFromWithConcurrency(r io.Reader, concurrency int) (read int64
 		go func() {
 			defer wg.Done()
 
-			ch := make(chan result, 1) // reusable channel per mapper.
+			for work := range workCh {
+				s := <-work.res
+				pool.Put(work.res)
 
-			for packet := range workCh {
+				err := s.err
-				n, err := f.writeChunkAt(ch, packet.b[:packet.n], packet.off)
+				if err == nil {
-				if err != nil {
+					switch s.typ {
-					// return the offset as the start + how much we wrote before the error.
+					case sshFxpStatus:
-					errCh <- rwErr{packet.off + int64(n), err}
+						err = normaliseError(unmarshalStatus(work.id, s.data))
+					default:
+						err = unimplementedPacketErr(s.typ)
+					}
+				}
+
+				if err != nil {
+					errCh <- rwErr{work.off, err}
 				}
-				pool.Put(packet.b)
 			}
 		}()
 	}

vendor/github.com/pkg/sftp/conn.go

@@ -18,7 +18,9 @@ type conn struct {
 }
 
 // the orderID is used in server mode if the allocator is enabled.
-// For the client mode just pass 0
+// For the client mode just pass 0.
+// It returns io.EOF if the connection is closed and
+// there are no more packets to read.
 func (c *conn) recvPacket(orderID uint32) (uint8, []byte, error) {
 	return recvPacket(c, c.alloc, orderID)
 }
@@ -61,14 +63,6 @@ func (c *clientConn) Close() error {
 	return c.conn.Close()
 }
 
-func (c *clientConn) loop() {
-	defer c.wg.Done()
-	err := c.recv()
-	if err != nil {
-		c.broadcastErr(err)
-	}
-}
-
 // recv continuously reads from the server and forwards responses to the
 // appropriate channel.
 func (c *clientConn) recv() error {

vendor/github.com/pkg/sftp/debug.go

@@ -1,3 +1,4 @@
+//go:build debug
 // +build debug
 
 package sftp

vendor/github.com/pkg/sftp/fuzz.go

@@ -1,3 +1,4 @@
+//go:build gofuzz
 // +build gofuzz
 
 package sftp

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/attrs.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // Attributes related flags.
 const (
@@ -12,7 +12,7 @@ const (
 
 // Attributes defines the file attributes type defined in draft-ietf-secsh-filexfer-02
 //
-// Defined in: https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-5
+// Defined in: https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5
 type Attributes struct {
 	Flags uint32
 
@@ -74,7 +74,6 @@ func (a *Attributes) SetPermissions(perms FileMode) {
 // GetACModTime returns the ATime and MTime fields and a bool that is true if and only if the values are valid/defined.
 func (a *Attributes) GetACModTime() (atime, mtime uint32, ok bool) {
 	return a.ATime, a.MTime, a.Flags&AttrACModTime != 0
-	return a.ATime, a.MTime, a.Flags&AttrACModTime != 0
 }
 
 // SetACModTime is a convenience function that sets the ATime and MTime fields,
@@ -117,32 +116,32 @@ func (a *Attributes) Len() int {
 }
 
 // MarshalInto marshals e onto the end of the given Buffer.
-func (a *Attributes) MarshalInto(b *Buffer) {
+func (a *Attributes) MarshalInto(buf *Buffer) {
-	b.AppendUint32(a.Flags)
+	buf.AppendUint32(a.Flags)
 
 	if a.Flags&AttrSize != 0 {
-		b.AppendUint64(a.Size)
+		buf.AppendUint64(a.Size)
 	}
 
 	if a.Flags&AttrUIDGID != 0 {
-		b.AppendUint32(a.UID)
+		buf.AppendUint32(a.UID)
-		b.AppendUint32(a.GID)
+		buf.AppendUint32(a.GID)
 	}
 
 	if a.Flags&AttrPermissions != 0 {
-		b.AppendUint32(uint32(a.Permissions))
+		buf.AppendUint32(uint32(a.Permissions))
 	}
 
 	if a.Flags&AttrACModTime != 0 {
-		b.AppendUint32(a.ATime)
+		buf.AppendUint32(a.ATime)
-		b.AppendUint32(a.MTime)
+		buf.AppendUint32(a.MTime)
 	}
 
 	if a.Flags&AttrExtended != 0 {
-		b.AppendUint32(uint32(len(a.ExtendedAttributes)))
+		buf.AppendUint32(uint32(len(a.ExtendedAttributes)))
 
 		for _, ext := range a.ExtendedAttributes {
-			ext.MarshalInto(b)
+			ext.MarshalInto(buf)
 		}
 	}
 }
@@ -157,74 +156,51 @@ func (a *Attributes) MarshalBinary() ([]byte, error) {
 // UnmarshalFrom unmarshals an Attributes from the given Buffer into e.
 //
 // NOTE: The values of fields not covered in the a.Flags are explicitly undefined.
-func (a *Attributes) UnmarshalFrom(b *Buffer) (err error) {
+func (a *Attributes) UnmarshalFrom(buf *Buffer) (err error) {
-	flags, err := b.ConsumeUint32()
+	flags := buf.ConsumeUint32()
-	if err != nil {
-		return err
-	}
 
-	return a.XXX_UnmarshalByFlags(flags, b)
+	return a.XXX_UnmarshalByFlags(flags, buf)
 }
 
 // XXX_UnmarshalByFlags uses the pre-existing a.Flags field to determine which fields to decode.
 // DO NOT USE THIS: it is an anti-corruption function to implement existing internal usage in pkg/sftp.
 // This function is not a part of any compatibility promise.
-func (a *Attributes) XXX_UnmarshalByFlags(flags uint32, b *Buffer) (err error) {
+func (a *Attributes) XXX_UnmarshalByFlags(flags uint32, buf *Buffer) (err error) {
 	a.Flags = flags
 
 	// Short-circuit dummy attributes.
 	if a.Flags == 0 {
-		return nil
+		return buf.Err
 	}
 
 	if a.Flags&AttrSize != 0 {
-		if a.Size, err = b.ConsumeUint64(); err != nil {
+		a.Size = buf.ConsumeUint64()
-			return err
-		}
 	}
 
 	if a.Flags&AttrUIDGID != 0 {
-		if a.UID, err = b.ConsumeUint32(); err != nil {
+		a.UID = buf.ConsumeUint32()
-			return err
+		a.GID = buf.ConsumeUint32()
-		}
-
-		if a.GID, err = b.ConsumeUint32(); err != nil {
-			return err
-		}
 	}
 
 	if a.Flags&AttrPermissions != 0 {
-		m, err := b.ConsumeUint32()
+		a.Permissions = FileMode(buf.ConsumeUint32())
-		if err != nil {
-			return err
-		}
-
-		a.Permissions = FileMode(m)
 	}
 
 	if a.Flags&AttrACModTime != 0 {
-		if a.ATime, err = b.ConsumeUint32(); err != nil {
+		a.ATime = buf.ConsumeUint32()
-			return err
+		a.MTime = buf.ConsumeUint32()
-		}
-
-		if a.MTime, err = b.ConsumeUint32(); err != nil {
-			return err
-		}
 	}
 
 	if a.Flags&AttrExtended != 0 {
-		count, err := b.ConsumeUint32()
+		count := buf.ConsumeCount()
-		if err != nil {
-			return err
-		}
 
 		a.ExtendedAttributes = make([]ExtendedAttribute, count)
 		for i := range a.ExtendedAttributes {
-			a.ExtendedAttributes[i].UnmarshalFrom(b)
+			a.ExtendedAttributes[i].UnmarshalFrom(buf)
 		}
 	}
 
-	return nil
+	return buf.Err
 }
 
 // UnmarshalBinary decodes the binary encoding of Attributes into e.
@@ -234,7 +210,7 @@ func (a *Attributes) UnmarshalBinary(data []byte) error {
 
 // ExtendedAttribute defines the extended file attribute type defined in draft-ietf-secsh-filexfer-02
 //
-// Defined in: https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-5
+// Defined in: https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5
 type ExtendedAttribute struct {
 	Type string
 	Data string
@@ -246,9 +222,9 @@ func (e *ExtendedAttribute) Len() int {
 }
 
 // MarshalInto marshals e onto the end of the given Buffer.
-func (e *ExtendedAttribute) MarshalInto(b *Buffer) {
+func (e *ExtendedAttribute) MarshalInto(buf *Buffer) {
-	b.AppendString(e.Type)
+	buf.AppendString(e.Type)
-	b.AppendString(e.Data)
+	buf.AppendString(e.Data)
 }
 
 // MarshalBinary returns e as the binary encoding of e.
@@ -259,16 +235,13 @@ func (e *ExtendedAttribute) MarshalBinary() ([]byte, error) {
 }
 
 // UnmarshalFrom unmarshals an ExtendedAattribute from the given Buffer into e.
-func (e *ExtendedAttribute) UnmarshalFrom(b *Buffer) (err error) {
+func (e *ExtendedAttribute) UnmarshalFrom(buf *Buffer) (err error) {
-	if e.Type, err = b.ConsumeString(); err != nil {
+	*e = ExtendedAttribute{
-		return err
+		Type: buf.ConsumeString(),
+		Data: buf.ConsumeString(),
 	}
 
-	if e.Data, err = b.ConsumeString(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	return nil
 }
 
 // UnmarshalBinary decodes the binary encoding of ExtendedAttribute into e.
@@ -291,11 +264,11 @@ func (e *NameEntry) Len() int {
 }
 
 // MarshalInto marshals e onto the end of the given Buffer.
-func (e *NameEntry) MarshalInto(b *Buffer) {
+func (e *NameEntry) MarshalInto(buf *Buffer) {
-	b.AppendString(e.Filename)
+	buf.AppendString(e.Filename)
-	b.AppendString(e.Longname)
+	buf.AppendString(e.Longname)
 
-	e.Attrs.MarshalInto(b)
+	e.Attrs.MarshalInto(buf)
 }
 
 // MarshalBinary returns e as the binary encoding of e.
@@ -308,16 +281,13 @@ func (e *NameEntry) MarshalBinary() ([]byte, error) {
 // UnmarshalFrom unmarshals an NameEntry from the given Buffer into e.
 //
 // NOTE: The values of fields not covered in the a.Flags are explicitly undefined.
-func (e *NameEntry) UnmarshalFrom(b *Buffer) (err error) {
+func (e *NameEntry) UnmarshalFrom(buf *Buffer) (err error) {
-	if e.Filename, err = b.ConsumeString(); err != nil {
+	*e = NameEntry{
-		return err
+		Filename: buf.ConsumeString(),
+		Longname: buf.ConsumeString(),
 	}
 
-	if e.Longname, err = b.ConsumeString(); err != nil {
+	return e.Attrs.UnmarshalFrom(buf)
-		return err
-	}
-
-	return e.Attrs.UnmarshalFrom(b)
 }
 
 // UnmarshalBinary decodes the binary encoding of NameEntry into e.

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/buffer.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 import (
 	"encoding/binary"
@@ -17,6 +17,7 @@ var (
 type Buffer struct {
 	b   []byte
 	off int
+	Err error
 }
 
 // NewBuffer creates and initializes a new buffer using buf as its initial contents.
@@ -51,14 +52,17 @@ func (b *Buffer) Cap() int { return cap(b.b) }
 
 // Reset resets the buffer to be empty, but it retains the underlying storage for use by future Appends.
 func (b *Buffer) Reset() {
-	b.b = b.b[:0]
+	*b = Buffer{
-	b.off = 0
+		b: b.b[:0],
+	}
 }
 
 // StartPacket resets and initializes the buffer to be ready to start marshaling a packet into.
 // It truncates the buffer, reserves space for uint32(length), then appends the given packetType and requestID.
 func (b *Buffer) StartPacket(packetType PacketType, requestID uint32) {
-	b.b, b.off = append(b.b[:0], make([]byte, 4)...), 0
+	*b = Buffer{
+		b: append(b.b[:0], make([]byte, 4)...),
+	}
 
 	b.AppendUint8(uint8(packetType))
 	b.AppendUint32(requestID)
@@ -81,15 +85,21 @@ func (b *Buffer) Packet(payload []byte) (header, payloadPassThru []byte, err err
 }
 
 // ConsumeUint8 consumes a single byte from the buffer.
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeUint8() (uint8, error) {
+func (b *Buffer) ConsumeUint8() uint8 {
+	if b.Err != nil {
+		return 0
+	}
+
 	if b.Len() < 1 {
-		return 0, ErrShortPacket
+		b.off = len(b.b)
+		b.Err = ErrShortPacket
+		return 0
 	}
 
 	var v uint8
 	v, b.off = b.b[b.off], b.off+1
-	return v, nil
+	return v
 }
 
 // AppendUint8 appends a single byte into the buffer.
@@ -98,14 +108,9 @@ func (b *Buffer) AppendUint8(v uint8) {
 }
 
 // ConsumeBool consumes a single byte from the buffer, and returns true if that byte is non-zero.
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeBool() (bool, error) {
+func (b *Buffer) ConsumeBool() bool {
-	v, err := b.ConsumeUint8()
+	return b.ConsumeUint8() != 0
-	if err != nil {
-		return false, err
-	}
-
-	return v != 0, nil
 }
 
 // AppendBool appends a single bool into the buffer.
@@ -119,15 +124,21 @@ func (b *Buffer) AppendBool(v bool) {
 }
 
 // ConsumeUint16 consumes a single uint16 from the buffer, in network byte order (big-endian).
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeUint16() (uint16, error) {
+func (b *Buffer) ConsumeUint16() uint16 {
+	if b.Err != nil {
+		return 0
+	}
+
 	if b.Len() < 2 {
-		return 0, ErrShortPacket
+		b.off = len(b.b)
+		b.Err = ErrShortPacket
+		return 0
 	}
 
 	v := binary.BigEndian.Uint16(b.b[b.off:])
 	b.off += 2
-	return v, nil
+	return v
 }
 
 // AppendUint16 appends single uint16 into the buffer, in network byte order (big-endian).
@@ -146,15 +157,21 @@ func unmarshalUint32(b []byte) uint32 {
 }
 
 // ConsumeUint32 consumes a single uint32 from the buffer, in network byte order (big-endian).
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeUint32() (uint32, error) {
+func (b *Buffer) ConsumeUint32() uint32 {
+	if b.Err != nil {
+		return 0
+	}
+
 	if b.Len() < 4 {
-		return 0, ErrShortPacket
+		b.off = len(b.b)
+		b.Err = ErrShortPacket
+		return 0
 	}
 
 	v := binary.BigEndian.Uint32(b.b[b.off:])
 	b.off += 4
-	return v, nil
+	return v
 }
 
 // AppendUint32 appends a single uint32 into the buffer, in network byte order (big-endian).
@@ -167,16 +184,33 @@ func (b *Buffer) AppendUint32(v uint32) {
 	)
 }
 
+// ConsumeCount consumes a single uint32 count from the buffer, in network byte order (big-endian) as an int.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
+func (b *Buffer) ConsumeCount() int {
+	return int(b.ConsumeUint32())
+}
+
+// AppendCount appends a single int length as a uint32 into the buffer, in network byte order (big-endian).
+func (b *Buffer) AppendCount(v int) {
+	b.AppendUint32(uint32(v))
+}
+
 // ConsumeUint64 consumes a single uint64 from the buffer, in network byte order (big-endian).
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeUint64() (uint64, error) {
+func (b *Buffer) ConsumeUint64() uint64 {
+	if b.Err != nil {
+		return 0
+	}
+
 	if b.Len() < 8 {
-		return 0, ErrShortPacket
+		b.off = len(b.b)
+		b.Err = ErrShortPacket
+		return 0
 	}
 
 	v := binary.BigEndian.Uint64(b.b[b.off:])
 	b.off += 8
-	return v, nil
+	return v
 }
 
 // AppendUint64 appends a single uint64 into the buffer, in network byte order (big-endian).
@@ -194,14 +228,9 @@ func (b *Buffer) AppendUint64(v uint64) {
 }
 
 // ConsumeInt64 consumes a single int64 from the buffer, in network byte order (big-endian) with two’s complement.
-// If the buffer does not have enough data, it will return ErrShortPacket.
+// If the buffer does not have enough data, it will set Err to ErrShortPacket.
-func (b *Buffer) ConsumeInt64() (int64, error) {
+func (b *Buffer) ConsumeInt64() int64 {
-	u, err := b.ConsumeUint64()
+	return int64(b.ConsumeUint64())
-	if err != nil {
-		return 0, err
-	}
-
-	return int64(u), err
 }
 
 // AppendInt64 appends a single int64 into the buffer, in network byte order (big-endian) with two’s complement.
@@ -211,29 +240,52 @@ func (b *Buffer) AppendInt64(v int64) {
 
 // ConsumeByteSlice consumes a single string of raw binary data from the buffer.
 // A string is a uint32 length, followed by that number of raw bytes.
-// If the buffer does not have enough data, or defines a length larger than available, it will return ErrShortPacket.
+// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket.
 //
 // The returned slice aliases the buffer contents, and is valid only as long as the buffer is not reused
 // (that is, only until the next call to Reset, PutLength, StartPacket, or UnmarshalBinary).
 //
 // In no case will any Consume calls return overlapping slice aliases,
 // and Append calls are guaranteed to not disturb this slice alias.
-func (b *Buffer) ConsumeByteSlice() ([]byte, error) {
+func (b *Buffer) ConsumeByteSlice() []byte {
-	length, err := b.ConsumeUint32()
+	length := int(b.ConsumeUint32())
-	if err != nil {
+	if b.Err != nil {
-		return nil, err
+		return nil
 	}
 
-	if b.Len() < int(length) {
+	if b.Len() < length || length < 0 {
-		return nil, ErrShortPacket
+		b.off = len(b.b)
+		b.Err = ErrShortPacket
+		return nil
 	}
 
 	v := b.b[b.off:]
-	if len(v) > int(length) {
+	if len(v) > length || cap(v) > length {
 		v = v[:length:length]
 	}
 	b.off += int(length)
-	return v, nil
+	return v
+}
+
+// ConsumeByteSliceCopy consumes a single string of raw binary data as a copy from the buffer.
+// A string is a uint32 length, followed by that number of raw bytes.
+// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket.
+//
+// The returned slice does not alias any buffer contents,
+// and will therefore be valid even if the buffer is later reused.
+//
+// If hint has sufficient capacity to hold the data, it will be reused and overwritten,
+// otherwise a new backing slice will be allocated and returned.
+func (b *Buffer) ConsumeByteSliceCopy(hint []byte) []byte {
+	data := b.ConsumeByteSlice()
+
+	if grow := len(data) - len(hint); grow > 0 {
+		hint = append(hint, make([]byte, grow)...)
+	}
+
+	n := copy(hint, data)
+	hint = hint[:n]
+	return hint
 }
 
 // AppendByteSlice appends a single string of raw binary data into the buffer.
@@ -245,17 +297,12 @@ func (b *Buffer) AppendByteSlice(v []byte) {
 
 // ConsumeString consumes a single string of binary data from the buffer.
 // A string is a uint32 length, followed by that number of raw bytes.
-// If the buffer does not have enough data, or defines a length larger than available, it will return ErrShortPacket.
+// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket.
 //
 // NOTE: Go implicitly assumes that strings contain UTF-8 encoded data.
 // All caveats on using arbitrary binary data in Go strings applies.
-func (b *Buffer) ConsumeString() (string, error) {
+func (b *Buffer) ConsumeString() string {
-	v, err := b.ConsumeByteSlice()
+	return string(b.ConsumeByteSlice())
-	if err != nil {
-		return "", err
-	}
-
-	return string(v), nil
 }
 
 // AppendString appends a single string of binary data into the buffer.

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extended_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 import (
 	"encoding"
@@ -86,8 +86,9 @@ func (p *ExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload
 // If the extension has not been registered, then a new Buffer will be allocated.
 // Then the request-specific-data will be unmarshaled from the rest of the buffer.
 func (p *ExtendedPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.ExtendedRequest, err = buf.ConsumeString(); err != nil {
+	p.ExtendedRequest = buf.ConsumeString()
-		return err
+	if buf.Err != nil {
+		return buf.Err
 	}
 
 	if p.Data == nil {

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extensions.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // ExtensionPair defines the extension-pair type defined in draft-ietf-secsh-filexfer-13.
 // This type is backwards-compatible with how draft-ietf-secsh-filexfer-02 defines extensions.
@@ -29,15 +29,12 @@ func (e *ExtensionPair) MarshalBinary() ([]byte, error) {
 
 // UnmarshalFrom unmarshals an ExtensionPair from the given Buffer into e.
 func (e *ExtensionPair) UnmarshalFrom(buf *Buffer) (err error) {
-	if e.Name, err = buf.ConsumeString(); err != nil {
+	*e = ExtensionPair{
-		return err
+		Name: buf.ConsumeString(),
+		Data: buf.ConsumeString(),
 	}
 
-	if e.Data, err = buf.ConsumeString(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	return nil
 }
 
 // UnmarshalBinary decodes the binary encoding of ExtensionPair into e.

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/filexfer.go

@@ -1,5 +1,5 @@
-// Package filexfer implements the wire encoding for secsh-filexfer as described in https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
+// Package sshfx implements the wire encoding for secsh-filexfer as described in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
-package filexfer
+package sshfx
 
 // PacketMarshaller narrowly defines packets that will only be transmitted.
 //

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fx.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 import (
 	"fmt"
@@ -10,7 +10,7 @@ type Status uint32
 // Defines the various SSH_FX_* values.
 const (
 	// see draft-ietf-secsh-filexfer-02
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-7
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-7
 	StatusOK = Status(iota)
 	StatusEOF
 	StatusNoSuchFile
@@ -21,28 +21,28 @@ const (
 	StatusConnectionLost
 	StatusOPUnsupported
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-03#section-7
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-03.txt#section-7
 	StatusV4InvalidHandle
 	StatusV4NoSuchPath
 	StatusV4FileAlreadyExists
 	StatusV4WriteProtect
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-04#section-7
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-04.txt#section-7
 	StatusV4NoMedia
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-05#section-7
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-05.txt#section-7
 	StatusV5NoSpaceOnFilesystem
 	StatusV5QuotaExceeded
 	StatusV5UnknownPrincipal
 	StatusV5LockConflict
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-06#section-8
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-06.txt#section-8
 	StatusV6DirNotEmpty
 	StatusV6NotADirectory
 	StatusV6InvalidFilename
 	StatusV6LinkLoop
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-07#section-8
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-07.txt#section-8
 	StatusV6CannotDelete
 	StatusV6InvalidParameter
 	StatusV6FileIsADirectory
@@ -50,10 +50,10 @@ const (
 	StatusV6ByteRangeLockRefused
 	StatusV6DeletePending
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-08#section-8.1
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-08.txt#section-8.1
 	StatusV6FileCorrupt
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-10#section-9.1
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-10.txt#section-9.1
 	StatusV6OwnerInvalid
 	StatusV6GroupInvalid
 

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fxp.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 import (
 	"fmt"
@@ -9,7 +9,7 @@ type PacketType uint8
 
 // Request packet types.
 const (
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-3
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3
 	PacketTypeInit = PacketType(iota + 1)
 	PacketTypeVersion
 	PacketTypeOpen
@@ -31,17 +31,17 @@ const (
 	PacketTypeReadLink
 	PacketTypeSymlink
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-07#section-3.3
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-07.txt#section-3.3
 	PacketTypeV6Link
 
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-08#section-3.3
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-08.txt#section-3.3
 	PacketTypeV6Block
 	PacketTypeV6Unblock
 )
 
 // Response packet types.
 const (
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-3
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3
 	PacketTypeStatus = PacketType(iota + 101)
 	PacketTypeHandle
 	PacketTypeData
@@ -51,7 +51,7 @@ const (
 
 // Extended packet types.
 const (
-	// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-3
+	// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3
 	PacketTypeExtended = PacketType(iota + 200)
 	PacketTypeExtendedReply
 )
@@ -122,3 +122,48 @@ func (f PacketType) String() string {
 		return fmt.Sprintf("SSH_FXP_UNKNOWN(%d)", f)
 	}
 }
+
+func newPacketFromType(typ PacketType) (Packet, error) {
+	switch typ {
+	case PacketTypeOpen:
+		return new(OpenPacket), nil
+	case PacketTypeClose:
+		return new(ClosePacket), nil
+	case PacketTypeRead:
+		return new(ReadPacket), nil
+	case PacketTypeWrite:
+		return new(WritePacket), nil
+	case PacketTypeLStat:
+		return new(LStatPacket), nil
+	case PacketTypeFStat:
+		return new(FStatPacket), nil
+	case PacketTypeSetstat:
+		return new(SetstatPacket), nil
+	case PacketTypeFSetstat:
+		return new(FSetstatPacket), nil
+	case PacketTypeOpenDir:
+		return new(OpenDirPacket), nil
+	case PacketTypeReadDir:
+		return new(ReadDirPacket), nil
+	case PacketTypeRemove:
+		return new(RemovePacket), nil
+	case PacketTypeMkdir:
+		return new(MkdirPacket), nil
+	case PacketTypeRmdir:
+		return new(RmdirPacket), nil
+	case PacketTypeRealPath:
+		return new(RealPathPacket), nil
+	case PacketTypeStat:
+		return new(StatPacket), nil
+	case PacketTypeRename:
+		return new(RenamePacket), nil
+	case PacketTypeReadLink:
+		return new(ReadLinkPacket), nil
+	case PacketTypeSymlink:
+		return new(SymlinkPacket), nil
+	case PacketTypeExtended:
+		return new(ExtendedPacket), nil
+	default:
+		return nil, fmt.Errorf("unexpected request packet type: %v", typ)
+	}
+}

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/handle_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // ClosePacket defines the SSH_FXP_CLOSE packet.
 type ClosePacket struct {
@@ -27,18 +27,18 @@ func (p *ClosePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *ClosePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = ClosePacket{
-		return err
+		Handle: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // ReadPacket defines the SSH_FXP_READ packet.
 type ReadPacket struct {
 	Handle string
 	Offset uint64
-	Len    uint32
+	Length uint32
 }
 
 // Type returns the SSH_FXP_xy value associated with this packet type.
@@ -58,7 +58,7 @@ func (p *ReadPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 	buf.StartPacket(PacketTypeRead, reqid)
 	buf.AppendString(p.Handle)
 	buf.AppendUint64(p.Offset)
-	buf.AppendUint32(p.Len)
+	buf.AppendUint32(p.Length)
 
 	return buf.Packet(payload)
 }
@@ -66,19 +66,13 @@ func (p *ReadPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *ReadPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = ReadPacket{
-		return err
+		Handle: buf.ConsumeString(),
+		Offset: buf.ConsumeUint64(),
+		Length: buf.ConsumeUint32(),
 	}
 
-	if p.Offset, err = buf.ConsumeUint64(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	if p.Len, err = buf.ConsumeUint32(); err != nil {
-		return err
-	}
-
-	return nil
 }
 
 // WritePacket defines the SSH_FXP_WRITE packet.
@@ -121,26 +115,13 @@ func (p *WritePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 //
 // This means this _does not_ alias any of the data buffer that is passed in.
 func (p *WritePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = WritePacket{
-		return err
+		Handle: buf.ConsumeString(),
+		Offset: buf.ConsumeUint64(),
+		Data:   buf.ConsumeByteSliceCopy(p.Data),
 	}
 
-	if p.Offset, err = buf.ConsumeUint64(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	data, err := buf.ConsumeByteSlice()
-	if err != nil {
-		return err
-	}
-
-	if len(p.Data) < len(data) {
-		p.Data = make([]byte, len(data))
-	}
-
-	n := copy(p.Data, data)
-	p.Data = p.Data[:n]
-	return nil
 }
 
 // FStatPacket defines the SSH_FXP_FSTAT packet.
@@ -170,11 +151,11 @@ func (p *FStatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *FStatPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = FStatPacket{
-		return err
+		Handle: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // FSetstatPacket defines the SSH_FXP_FSETSTAT packet.
@@ -207,8 +188,8 @@ func (p *FSetstatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *FSetstatPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = FSetstatPacket{
-		return err
+		Handle: buf.ConsumeString(),
 	}
 
 	return p.Attrs.UnmarshalFrom(buf)
@@ -241,9 +222,9 @@ func (p *ReadDirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload [
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *ReadDirPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = ReadDirPacket{
-		return err
+		Handle: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/init_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // InitPacket defines the SSH_FXP_INIT packet.
 type InitPacket struct {
@@ -33,8 +33,8 @@ func (p *InitPacket) MarshalBinary() ([]byte, error) {
 func (p *InitPacket) UnmarshalBinary(data []byte) (err error) {
 	buf := NewBuffer(data)
 
-	if p.Version, err = buf.ConsumeUint32(); err != nil {
+	*p = InitPacket{
-		return err
+		Version: buf.ConsumeUint32(),
 	}
 
 	for buf.Len() > 0 {
@@ -46,7 +46,7 @@ func (p *InitPacket) UnmarshalBinary(data []byte) (err error) {
 		p.Extensions = append(p.Extensions, &ext)
 	}
 
-	return nil
+	return buf.Err
 }
 
 // VersionPacket defines the SSH_FXP_VERSION packet.
@@ -82,8 +82,8 @@ func (p *VersionPacket) MarshalBinary() ([]byte, error) {
 func (p *VersionPacket) UnmarshalBinary(data []byte) (err error) {
 	buf := NewBuffer(data)
 
-	if p.Version, err = buf.ConsumeUint32(); err != nil {
+	*p = VersionPacket{
-		return err
+		Version: buf.ConsumeUint32(),
 	}
 
 	for buf.Len() > 0 {

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/open_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // SSH_FXF_* flags.
 const (
@@ -43,12 +43,9 @@ func (p *OpenPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *OpenPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Filename, err = buf.ConsumeString(); err != nil {
+	*p = OpenPacket{
-		return err
+		Filename: buf.ConsumeString(),
-	}
+		PFlags:   buf.ConsumeUint32(),
-
-	if p.PFlags, err = buf.ConsumeUint32(); err != nil {
-		return err
 	}
 
 	return p.Attrs.UnmarshalFrom(buf)
@@ -81,9 +78,9 @@ func (p *OpenDirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload [
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *OpenDirPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = OpenDirPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/packets.go

@@ -1,59 +1,13 @@
-package filexfer
+package sshfx
 
 import (
 	"errors"
-	"fmt"
 	"io"
 )
 
 // smallBufferSize is an initial allocation minimal capacity.
 const smallBufferSize = 64
 
-func newPacketFromType(typ PacketType) (Packet, error) {
-	switch typ {
-	case PacketTypeOpen:
-		return new(OpenPacket), nil
-	case PacketTypeClose:
-		return new(ClosePacket), nil
-	case PacketTypeRead:
-		return new(ReadPacket), nil
-	case PacketTypeWrite:
-		return new(WritePacket), nil
-	case PacketTypeLStat:
-		return new(LStatPacket), nil
-	case PacketTypeFStat:
-		return new(FStatPacket), nil
-	case PacketTypeSetstat:
-		return new(SetstatPacket), nil
-	case PacketTypeFSetstat:
-		return new(FSetstatPacket), nil
-	case PacketTypeOpenDir:
-		return new(OpenDirPacket), nil
-	case PacketTypeReadDir:
-		return new(ReadDirPacket), nil
-	case PacketTypeRemove:
-		return new(RemovePacket), nil
-	case PacketTypeMkdir:
-		return new(MkdirPacket), nil
-	case PacketTypeRmdir:
-		return new(RmdirPacket), nil
-	case PacketTypeRealPath:
-		return new(RealPathPacket), nil
-	case PacketTypeStat:
-		return new(StatPacket), nil
-	case PacketTypeRename:
-		return new(RenamePacket), nil
-	case PacketTypeReadLink:
-		return new(ReadLinkPacket), nil
-	case PacketTypeSymlink:
-		return new(SymlinkPacket), nil
-	case PacketTypeExtended:
-		return new(ExtendedPacket), nil
-	default:
-		return nil, fmt.Errorf("unexpected request packet type: %v", typ)
-	}
-}
-
 // RawPacket implements the general packet format from draft-ietf-secsh-filexfer-02
 //
 // RawPacket is intended for use in clients receiving responses,
@@ -63,7 +17,7 @@ func newPacketFromType(typ PacketType) (Packet, error) {
 // For servers expecting to receive arbitrary request packet types,
 // use RequestPacket.
 //
-// Defined in https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-3
+// Defined in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3
 type RawPacket struct {
 	PacketType PacketType
 	RequestID  uint32
@@ -110,19 +64,14 @@ func (p *RawPacket) MarshalBinary() ([]byte, error) {
 // The Data field will alias the passed in Buffer,
 // so the buffer passed in should not be reused before RawPacket.Reset().
 func (p *RawPacket) UnmarshalFrom(buf *Buffer) error {
-	typ, err := buf.ConsumeUint8()
+	*p = RawPacket{
-	if err != nil {
+		PacketType: PacketType(buf.ConsumeUint8()),
-		return err
+		RequestID:  buf.ConsumeUint32(),
-	}
-
-	p.PacketType = PacketType(typ)
-
-	if p.RequestID, err = buf.ConsumeUint32(); err != nil {
-		return err
 	}
 
 	p.Data = *buf
-	return nil
+
+	return buf.Err
 }
 
 // UnmarshalBinary decodes a full raw packet out of the given data.
@@ -225,7 +174,7 @@ func (p *RawPacket) ReadFrom(r io.Reader, b []byte, maxPacketLength uint32) erro
 // where automatic unmarshaling of the packet body does not make sense,
 // use RawPacket.
 //
-// Defined in https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-3
+// Defined in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3
 type RequestPacket struct {
 	RequestID uint32
 
@@ -268,18 +217,19 @@ func (p *RequestPacket) MarshalBinary() ([]byte, error) {
 // The Request field may alias the passed in Buffer, (e.g. SSH_FXP_WRITE),
 // so the buffer passed in should not be reused before RequestPacket.Reset().
 func (p *RequestPacket) UnmarshalFrom(buf *Buffer) error {
-	typ, err := buf.ConsumeUint8()
+	typ := PacketType(buf.ConsumeUint8())
+	if buf.Err != nil {
+		return buf.Err
+	}
+
+	req, err := newPacketFromType(typ)
 	if err != nil {
 		return err
 	}
 
-	p.Request, err = newPacketFromType(PacketType(typ))
+	*p = RequestPacket{
-	if err != nil {
+		RequestID: buf.ConsumeUint32(),
-		return err
+		Request:   req,
-	}
-
-	if p.RequestID, err = buf.ConsumeUint32(); err != nil {
-		return err
 	}
 
 	return p.Request.UnmarshalPacketBody(buf)

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/path_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // LStatPacket defines the SSH_FXP_LSTAT packet.
 type LStatPacket struct {
@@ -27,11 +27,11 @@ func (p *LStatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *LStatPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = LStatPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // SetstatPacket defines the SSH_FXP_SETSTAT packet.
@@ -64,8 +64,8 @@ func (p *SetstatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload [
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *SetstatPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = SetstatPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
 	return p.Attrs.UnmarshalFrom(buf)
@@ -98,11 +98,11 @@ func (p *RemovePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *RemovePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = RemovePacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // MkdirPacket defines the SSH_FXP_MKDIR packet.
@@ -135,8 +135,8 @@ func (p *MkdirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *MkdirPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = MkdirPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
 	return p.Attrs.UnmarshalFrom(buf)
@@ -169,11 +169,11 @@ func (p *RmdirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []b
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *RmdirPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = RmdirPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // RealPathPacket defines the SSH_FXP_REALPATH packet.
@@ -203,11 +203,11 @@ func (p *RealPathPacket) MarshalPacket(reqid uint32, b []byte) (header, payload
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *RealPathPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = RealPathPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // StatPacket defines the SSH_FXP_STAT packet.
@@ -237,11 +237,11 @@ func (p *StatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *StatPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = StatPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // RenamePacket defines the SSH_FXP_RENAME packet.
@@ -274,15 +274,12 @@ func (p *RenamePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *RenamePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.OldPath, err = buf.ConsumeString(); err != nil {
+	*p = RenamePacket{
-		return err
+		OldPath: buf.ConsumeString(),
+		NewPath: buf.ConsumeString(),
 	}
 
-	if p.NewPath, err = buf.ConsumeString(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	return nil
 }
 
 // ReadLinkPacket defines the SSH_FXP_READLINK packet.
@@ -312,18 +309,18 @@ func (p *ReadLinkPacket) MarshalPacket(reqid uint32, b []byte) (header, payload
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *ReadLinkPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Path, err = buf.ConsumeString(); err != nil {
+	*p = ReadLinkPacket{
-		return err
+		Path: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // SymlinkPacket defines the SSH_FXP_SYMLINK packet.
 //
 // The order of the arguments to the SSH_FXP_SYMLINK method was inadvertently reversed.
 // Unfortunately, the reversal was not noticed until the server was widely deployed.
-// Covered in Section 3.1 of https://github.com/openssh/openssh-portable/blob/master/PROTOCOL
+// Covered in Section 4.1 of https://github.com/openssh/openssh-portable/blob/master/PROTOCOL
 type SymlinkPacket struct {
 	LinkPath   string
 	TargetPath string
@@ -355,14 +352,11 @@ func (p *SymlinkPacket) MarshalPacket(reqid uint32, b []byte) (header, payload [
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *SymlinkPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	// Arguments were inadvertently reversed.
+	*p = SymlinkPacket{
-	if p.TargetPath, err = buf.ConsumeString(); err != nil {
+		// Arguments were inadvertently reversed.
-		return err
+		TargetPath: buf.ConsumeString(),
+		LinkPath:   buf.ConsumeString(),
 	}
 
-	if p.LinkPath, err = buf.ConsumeString(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	return nil
 }

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/permissions.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 // FileMode represents a file’s mode and permission bits.
 // The bits are defined according to POSIX standards,

vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/response_packets.go

@@ -1,4 +1,4 @@
-package filexfer
+package sshfx
 
 import (
 	"fmt"
@@ -6,7 +6,7 @@ import (
 
 // StatusPacket defines the SSH_FXP_STATUS packet.
 //
-// Specified in https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-7
+// Specified in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-7
 type StatusPacket struct {
 	StatusCode   Status
 	ErrorMessage string
@@ -19,7 +19,7 @@ func (p *StatusPacket) Error() string {
 		return "sftp: " + p.StatusCode.String()
 	}
 
-	return fmt.Sprintf("sftp: %q (%s)", p.ErrorMessage, p.StatusCode)
+	return fmt.Sprintf("sftp: %s: %q", p.StatusCode, p.ErrorMessage)
 }
 
 // Is returns true if target is a StatusPacket with the same StatusCode,
@@ -57,21 +57,13 @@ func (p *StatusPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *StatusPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	statusCode, err := buf.ConsumeUint32()
+	*p = StatusPacket{
-	if err != nil {
+		StatusCode:   Status(buf.ConsumeUint32()),
-		return err
+		ErrorMessage: buf.ConsumeString(),
-	}
+		LanguageTag:  buf.ConsumeString(),
-	p.StatusCode = Status(statusCode)
-
-	if p.ErrorMessage, err = buf.ConsumeString(); err != nil {
-		return err
 	}
 
-	if p.LanguageTag, err = buf.ConsumeString(); err != nil {
+	return buf.Err
-		return err
-	}
-
-	return nil
 }
 
 // HandlePacket defines the SSH_FXP_HANDLE packet.
@@ -101,11 +93,11 @@ func (p *HandlePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *HandlePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	if p.Handle, err = buf.ConsumeString(); err != nil {
+	*p = HandlePacket{
-		return err
+		Handle: buf.ConsumeString(),
 	}
 
-	return nil
+	return buf.Err
 }
 
 // DataPacket defines the SSH_FXP_DATA packet.
@@ -143,18 +135,11 @@ func (p *DataPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 //
 // This means this _does not_ alias any of the data buffer that is passed in.
 func (p *DataPacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	data, err := buf.ConsumeByteSlice()
+	*p = DataPacket{
-	if err != nil {
+		Data: buf.ConsumeByteSliceCopy(p.Data),
-		return err
 	}
 
-	if len(p.Data) < len(data) {
+	return buf.Err
-		p.Data = make([]byte, len(data))
-	}
-
-	n := copy(p.Data, data)
-	p.Data = p.Data[:n]
-	return nil
 }
 
 // NamePacket defines the SSH_FXP_NAME packet.
@@ -193,14 +178,16 @@ func (p *NamePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []by
 // UnmarshalPacketBody unmarshals the packet body from the given Buffer.
 // It is assumed that the uint32(request-id) has already been consumed.
 func (p *NamePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
-	count, err := buf.ConsumeUint32()
+	count := buf.ConsumeCount()
-	if err != nil {
+	if buf.Err != nil {
-		return err
+		return buf.Err
 	}
 
-	p.Entries = make([]*NameEntry, 0, count)
+	*p = NamePacket{
+		Entries: make([]*NameEntry, 0, count),
+	}
 
-	for i := uint32(0); i < count; i++ {
+	for i := 0; i < count; i++ {
 		var e NameEntry
 		if err := e.UnmarshalFrom(buf); err != nil {
 			return err
@@ -209,7 +196,7 @@ func (p *NamePacket) UnmarshalPacketBody(buf *Buffer) (err error) {
 		p.Entries = append(p.Entries, &e)
 	}
 
-	return nil
+	return buf.Err
 }
 
 // AttrsPacket defines the SSH_FXP_ATTRS packet.

vendor/github.com/pkg/sftp/ls_formatting.go

@@ -60,6 +60,13 @@ func runLs(idLookup NameLookupFileLister, dirent os.FileInfo) string {
 		uid = lsFormatID(sys.UID)
 		gid = lsFormatID(sys.GID)
 	default:
+		if fiExt, ok := dirent.(FileInfoUidGid); ok {
+			uid = lsFormatID(fiExt.Uid())
+			gid = lsFormatID(fiExt.Gid())
+
+			break
+		}
+
 		numLinks, uid, gid = lsLinksUIDGID(dirent)
 	}
 

vendor/github.com/pkg/sftp/ls_plan9.go

@@ -1,3 +1,4 @@
+//go:build plan9
 // +build plan9
 
 package sftp

vendor/github.com/pkg/sftp/ls_stub.go

@@ -1,3 +1,4 @@
+//go:build windows || android
 // +build windows android
 
 package sftp

vendor/github.com/pkg/sftp/ls_unix.go

@@ -1,3 +1,4 @@
+//go:build aix || darwin || dragonfly || freebsd || (!android && linux) || netbsd || openbsd || solaris || js
 // +build aix darwin dragonfly freebsd !android,linux netbsd openbsd solaris js
 
 package sftp

vendor/github.com/pkg/sftp/packet-manager.go

@@ -40,7 +40,7 @@ func newPktMgr(sender packetSender) *packetManager {
 	return s
 }
 
-//// packet ordering
+// // packet ordering
 func (s *packetManager) newOrderID() uint32 {
 	s.packetCount++
 	return s.packetCount
@@ -89,7 +89,7 @@ func (o orderedPackets) Sort() {
 	})
 }
 
-//// packet registry
+// // packet registry
 // register incoming packets to be handled
 func (s *packetManager) incomingPacket(pkt orderedRequest) {
 	s.working.Add(1)

vendor/github.com/pkg/sftp/packet-typing.go

@@ -31,7 +31,7 @@ type notReadOnly interface {
 	notReadOnly()
 }
 
-//// define types by adding methods
+// // define types by adding methods
 // hasPath
 func (p *sshFxpLstatPacket) getPath() string    { return p.Path }
 func (p *sshFxpStatPacket) getPath() string     { return p.Path }

vendor/github.com/pkg/sftp/packet.go

@@ -71,6 +71,15 @@ func marshalFileInfo(b []byte, fi os.FileInfo) []byte {
 		b = marshalUint32(b, fileStat.Mtime)
 	}
 
+	if flags&sshFileXferAttrExtended != 0 {
+		b = marshalUint32(b, uint32(len(fileStat.Extended)))
+
+		for _, attr := range fileStat.Extended {
+			b = marshalString(b, attr.ExtType)
+			b = marshalString(b, attr.ExtData)
+		}
+	}
+
 	return b
 }
 
@@ -281,6 +290,11 @@ func recvPacket(r io.Reader, alloc *allocator, orderID uint32) (uint8, []byte, e
 		b = make([]byte, length)
 	}
 	if _, err := io.ReadFull(r, b[:length]); err != nil {
+		// ReadFull only returns EOF if it has read no bytes.
+		// In this case, that means a partial packet, and thus unexpected.
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
 		debug("recv packet %d bytes: err %v", length, err)
 		return 0, nil, err
 	}
@@ -522,7 +536,12 @@ func (p *sshFxpRmdirPacket) UnmarshalBinary(b []byte) error {
 }
 
 type sshFxpSymlinkPacket struct {
-	ID         uint32
+	ID uint32
+
+	// The order of the arguments to the SSH_FXP_SYMLINK method was inadvertently reversed.
+	// Unfortunately, the reversal was not noticed until the server was widely deployed.
+	// Covered in Section 4.1 of https://github.com/openssh/openssh-portable/blob/master/PROTOCOL
+
 	Targetpath string
 	Linkpath   string
 }
@@ -1242,7 +1261,7 @@ func (p *sshFxpExtendedPacketPosixRename) UnmarshalBinary(b []byte) error {
 }
 
 func (p *sshFxpExtendedPacketPosixRename) respond(s *Server) responsePacket {
-	err := os.Rename(p.Oldpath, p.Newpath)
+	err := os.Rename(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath))
 	return statusFromError(p.ID, err)
 }
 
@@ -1271,6 +1290,6 @@ func (p *sshFxpExtendedPacketHardlink) UnmarshalBinary(b []byte) error {
 }
 
 func (p *sshFxpExtendedPacketHardlink) respond(s *Server) responsePacket {
-	err := os.Link(p.Oldpath, p.Newpath)
+	err := os.Link(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath))
 	return statusFromError(p.ID, err)
 }

vendor/github.com/pkg/sftp/release.go

@@ -1,3 +1,4 @@
+//go:build !debug
 // +build !debug
 
 package sftp

vendor/github.com/pkg/sftp/request-example.go

@@ -391,21 +391,6 @@ func (fs *root) Filelist(r *Request) (ListerAt, error) {
 			return nil, err
 		}
 		return listerat{file}, nil
-
-	case "Readlink":
-		symlink, err := fs.readlink(r.Filepath)
-		if err != nil {
-			return nil, err
-		}
-
-		// SFTP-v2: The server will respond with a SSH_FXP_NAME packet containing only
-		// one name and a dummy attributes value.
-		return listerat{
-			&memFile{
-				name: symlink,
-				err:  os.ErrNotExist, // prevent accidental use as a reader/writer.
-			},
-		}, nil
 	}
 
 	return nil, errors.New("unsupported")
@@ -434,7 +419,7 @@ func (fs *root) readdir(pathname string) ([]os.FileInfo, error) {
 	return files, nil
 }
 
-func (fs *root) readlink(pathname string) (string, error) {
+func (fs *root) Readlink(pathname string) (string, error) {
 	file, err := fs.lfetch(pathname)
 	if err != nil {
 		return "", err
@@ -464,19 +449,10 @@ func (fs *root) Lstat(r *Request) (ListerAt, error) {
 	return listerat{file}, nil
 }
 
-// implements RealpathFileLister interface
-func (fs *root) Realpath(p string) string {
-	if fs.startDirectory == "" || fs.startDirectory == "/" {
-		return cleanPath(p)
-	}
-	return cleanPathWithBase(fs.startDirectory, p)
-}
-
 // In memory file-system-y thing that the Hanlders live on
 type root struct {
-	rootFile       *memFile
+	rootFile *memFile
-	mockErr        error
+	mockErr  error
-	startDirectory string
 
 	mu    sync.Mutex
 	files map[string]*memFile
@@ -534,8 +510,8 @@ func (fs *root) exists(path string) bool {
 	return err != os.ErrNotExist
 }
 
-func (fs *root) fetch(path string) (*memFile, error) {
+func (fs *root) fetch(pathname string) (*memFile, error) {
-	file, err := fs.lfetch(path)
+	file, err := fs.lfetch(pathname)
 	if err != nil {
 		return nil, err
 	}
@@ -546,7 +522,12 @@ func (fs *root) fetch(path string) (*memFile, error) {
 			return nil, errTooManySymlinks
 		}
 
-		file, err = fs.lfetch(file.symlink)
+		linkTarget := file.symlink
+		if !path.IsAbs(linkTarget) {
+			linkTarget = path.Join(path.Dir(file.name), linkTarget)
+		}
+
+		file, err = fs.lfetch(linkTarget)
 		if err != nil {
 			return nil, err
 		}

vendor/github.com/pkg/sftp/request-interfaces.go

@@ -74,6 +74,11 @@ type StatVFSFileCmder interface {
 // FileLister should return an object that fulfils the ListerAt interface
 // Note in cases of an error, the error text will be sent to the client.
 // Called for Methods: List, Stat, Readlink
+//
+// Since Filelist returns an os.FileInfo, this can make it non-ideal for implementing Readlink.
+// This is because the Name receiver method defined by that interface defines that it should only return the base name.
+// However, Readlink is required to be capable of returning essentially any arbitrary valid path relative or absolute.
+// In order to implement this more expressive requirement, implement [ReadlinkFileLister] which will then be used instead.
 type FileLister interface {
 	Filelist(*Request) (ListerAt, error)
 }
@@ -87,10 +92,33 @@ type LstatFileLister interface {
 }
 
 // RealPathFileLister is a FileLister that implements the Realpath method.
-// We use "/" as start directory for relative paths, implementing this
+// The built-in RealPath implementation does not resolve symbolic links.
-// interface you can customize the start directory.
+// By implementing this interface you can customize the returned path
+// and, for example, resolve symbolinc links if needed for your use case.
 // You have to return an absolute POSIX path.
+//
+// Up to v1.13.5 the signature for the RealPath method was:
+//
+// # RealPath(string) string
+//
+// we have added a legacyRealPathFileLister that implements the old method
+// to ensure that your code does not break.
+// You should use the new method signature to avoid future issues
 type RealPathFileLister interface {
+	FileLister
+	RealPath(string) (string, error)
+}
+
+// ReadlinkFileLister is a FileLister that implements the Readlink method.
+// By implementing the Readlink method, it is possible to return any arbitrary valid path relative or absolute.
+// This allows giving a better response than via the default FileLister (which is limited to os.FileInfo, whose Name method should only return the base name of a file)
+type ReadlinkFileLister interface {
+	FileLister
+	Readlink(string) (string, error)
+}
+
+// This interface is here for backward compatibility only
+type legacyRealPathFileLister interface {
 	FileLister
 	RealPath(string) string
 }
@@ -103,11 +131,19 @@ type NameLookupFileLister interface {
 	LookupGroupName(string) string
 }
 
-// ListerAt does for file lists what io.ReaderAt does for files.
+// ListerAt does for file lists what io.ReaderAt does for files, i.e. a []os.FileInfo buffer is passed to the ListAt function
-// ListAt should return the number of entries copied and an io.EOF
+// and the entries that are populated in the buffer will be passed to the client.
-// error if at end of list. This is testable by comparing how many you
+//
-// copied to how many could be copied (eg. n < len(ls) below).
+// ListAt should return the number of entries copied and an io.EOF error if at end of list.
+// This is testable by comparing how many you copied to how many could be copied (eg. n < len(ls) below).
 // The copy() builtin is best for the copying.
+//
+// Uid and gid information will on unix systems be retrieved from [os.FileInfo.Sys]
+// if this function returns a [syscall.Stat_t] when called on a populated entry.
+// Alternatively, if the entry implements [FileInfoUidGid], it will be used for uid and gid information.
+//
+// If a populated entry implements [FileInfoExtendedData], extended attributes will also be returned to the client.
+//
 // Note in cases of an error, the error text will be sent to the client.
 type ListerAt interface {
 	ListAt([]os.FileInfo, int64) (int, error)

vendor/github.com/pkg/sftp/request-plan9.go

@@ -1,10 +1,9 @@
+//go:build plan9
 // +build plan9
 
 package sftp
 
 import (
-	"path"
-	"path/filepath"
 	"syscall"
 )
 
@@ -15,20 +14,3 @@ func fakeFileInfoSys() interface{} {
 func testOsSys(sys interface{}) error {
 	return nil
 }
-
-func toLocalPath(p string) string {
-	lp := filepath.FromSlash(p)
-
-	if path.IsAbs(p) {
-		tmp := lp[1:]
-
-		if filepath.IsAbs(tmp) {
-			// If the FromSlash without any starting slashes is absolute,
-			// then we have a filepath encoded with a prefix '/'.
-			// e.g. "/#s/boot" to "#s/boot"
-			return tmp
-		}
-	}
-
-	return lp
-}

vendor/github.com/pkg/sftp/request-readme.md

@@ -28,7 +28,7 @@ then sends to the client.
 Handler for "Put" method and returns an io.Writer for the file which the server
 then writes the uploaded file to. The file opening "pflags" are currently
 preserved in the Request.Flags field as a 32bit bitmask value. See the [SFTP
-spec](https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-6.3) for
+spec](https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-6.3) for
 details.
 
 ###    Filecmd(*Request) error

vendor/github.com/pkg/sftp/request-server.go

@@ -27,6 +27,8 @@ type RequestServer struct {
 	*serverConn
 	pktMgr *packetManager
 
+	startDirectory string
+
 	mu           sync.RWMutex
 	handleCount  int
 	openRequests map[string]*Request
@@ -47,6 +49,14 @@ func WithRSAllocator() RequestServerOption {
 	}
 }
 
+// WithStartDirectory sets a start directory to use as base for relative paths.
+// If unset the default is "/"
+func WithStartDirectory(startDirectory string) RequestServerOption {
+	return func(rs *RequestServer) {
+		rs.startDirectory = cleanPath(startDirectory)
+	}
+}
+
 // NewRequestServer creates/allocates/returns new RequestServer.
 // Normally there will be one server per user-session.
 func NewRequestServer(rwc io.ReadWriteCloser, h Handlers, options ...RequestServerOption) *RequestServer {
@@ -62,6 +72,8 @@ func NewRequestServer(rwc io.ReadWriteCloser, h Handlers, options ...RequestServ
 		serverConn: svrConn,
 		pktMgr:     newPktMgr(svrConn),
 
+		startDirectory: "/",
+
 		openRequests: make(map[string]*Request),
 	}
 
@@ -207,14 +219,23 @@ func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedR
 			rpkt = statusFromError(pkt.ID, rs.closeRequest(handle))
 		case *sshFxpRealpathPacket:
 			var realPath string
-			if realPather, ok := rs.Handlers.FileList.(RealPathFileLister); ok {
+			var err error
-				realPath = realPather.RealPath(pkt.getPath())
+
-			} else {
+			switch pather := rs.Handlers.FileList.(type) {
-				realPath = cleanPath(pkt.getPath())
+			case RealPathFileLister:
+				realPath, err = pather.RealPath(pkt.getPath())
+			case legacyRealPathFileLister:
+				realPath = pather.RealPath(pkt.getPath())
+			default:
+				realPath = cleanPathWithBase(rs.startDirectory, pkt.getPath())
+			}
+			if err != nil {
+				rpkt = statusFromError(pkt.ID, err)
+			} else {
+				rpkt = cleanPacketPath(pkt, realPath)
 			}
-			rpkt = cleanPacketPath(pkt, realPath)
 		case *sshFxpOpendirPacket:
-			request := requestFromPacket(ctx, pkt)
+			request := requestFromPacket(ctx, pkt, rs.startDirectory)
 			handle := rs.nextRequest(request)
 			rpkt = request.opendir(rs.Handlers, pkt)
 			if _, ok := rpkt.(*sshFxpHandlePacket); !ok {
@@ -222,7 +243,7 @@ func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedR
 				rs.closeRequest(handle)
 			}
 		case *sshFxpOpenPacket:
-			request := requestFromPacket(ctx, pkt)
+			request := requestFromPacket(ctx, pkt, rs.startDirectory)
 			handle := rs.nextRequest(request)
 			rpkt = request.open(rs.Handlers, pkt)
 			if _, ok := rpkt.(*sshFxpHandlePacket); !ok {
@@ -235,7 +256,10 @@ func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedR
 			if !ok {
 				rpkt = statusFromError(pkt.ID, EBADF)
 			} else {
-				request = NewRequest("Stat", request.Filepath)
+				request = &Request{
+					Method:   "Stat",
+					Filepath: cleanPathWithBase(rs.startDirectory, request.Filepath),
+				}
 				rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 			}
 		case *sshFxpFsetstatPacket:
@@ -244,15 +268,24 @@ func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedR
 			if !ok {
 				rpkt = statusFromError(pkt.ID, EBADF)
 			} else {
-				request = NewRequest("Setstat", request.Filepath)
+				request = &Request{
+					Method:   "Setstat",
+					Filepath: cleanPathWithBase(rs.startDirectory, request.Filepath),
+				}
 				rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 			}
 		case *sshFxpExtendedPacketPosixRename:
-			request := NewRequest("PosixRename", pkt.Oldpath)
+			request := &Request{
-			request.Target = pkt.Newpath
+				Method:   "PosixRename",
+				Filepath: cleanPathWithBase(rs.startDirectory, pkt.Oldpath),
+				Target:   cleanPathWithBase(rs.startDirectory, pkt.Newpath),
+			}
 			rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 		case *sshFxpExtendedPacketStatVFS:
-			request := NewRequest("StatVFS", pkt.Path)
+			request := &Request{
+				Method:   "StatVFS",
+				Filepath: cleanPathWithBase(rs.startDirectory, pkt.Path),
+			}
 			rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 		case hasHandle:
 			handle := pkt.getHandle()
@@ -263,7 +296,7 @@ func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedR
 				rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 			}
 		case hasPath:
-			request := requestFromPacket(ctx, pkt)
+			request := requestFromPacket(ctx, pkt, rs.startDirectory)
 			rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID)
 			request.close()
 		default:

vendor/github.com/pkg/sftp/request-unix.go

@@ -1,3 +1,4 @@
+//go:build !windows && !plan9
 // +build !windows,!plan9
 
 package sftp
@@ -21,7 +22,3 @@ func testOsSys(sys interface{}) error {
 	}
 	return nil
 }
-
-func toLocalPath(p string) string {
-	return p
-}

vendor/github.com/pkg/sftp/request.go

@@ -168,9 +168,11 @@ func (r *Request) copy() *Request {
 }
 
 // New Request initialized based on packet data
-func requestFromPacket(ctx context.Context, pkt hasPath) *Request {
+func requestFromPacket(ctx context.Context, pkt hasPath, baseDir string) *Request {
-	method := requestMethod(pkt)
+	request := &Request{
-	request := NewRequest(method, pkt.getPath())
+		Method:   requestMethod(pkt),
+		Filepath: cleanPathWithBase(baseDir, pkt.getPath()),
+	}
 	request.ctx, request.cancelCtx = context.WithCancel(ctx)
 
 	switch p := pkt.(type) {
@@ -180,13 +182,14 @@ func requestFromPacket(ctx context.Context, pkt hasPath) *Request {
 		request.Flags = p.Flags
 		request.Attrs = p.Attrs.([]byte)
 	case *sshFxpRenamePacket:
-		request.Target = cleanPath(p.Newpath)
+		request.Target = cleanPathWithBase(baseDir, p.Newpath)
 	case *sshFxpSymlinkPacket:
 		// NOTE: given a POSIX compliant signature: symlink(target, linkpath string)
 		// this makes Request.Target the linkpath, and Request.Filepath the target.
-		request.Target = cleanPath(p.Linkpath)
+		request.Target = cleanPathWithBase(baseDir, p.Linkpath)
+		request.Filepath = p.Targetpath
 	case *sshFxpExtendedPacketHardlink:
-		request.Target = cleanPath(p.Newpath)
+		request.Target = cleanPathWithBase(baseDir, p.Newpath)
 	}
 	return request
 }
@@ -292,7 +295,12 @@ func (r *Request) call(handlers Handlers, pkt requestPacket, alloc *allocator, o
 		return filecmd(handlers.FileCmd, r, pkt)
 	case "List":
 		return filelist(handlers.FileList, r, pkt)
-	case "Stat", "Lstat", "Readlink":
+	case "Stat", "Lstat":
+		return filestat(handlers.FileList, r, pkt)
+	case "Readlink":
+		if readlinkFileLister, ok := handlers.FileList.(ReadlinkFileLister); ok {
+			return readlink(readlinkFileLister, r, pkt)
+		}
 		return filestat(handlers.FileList, r, pkt)
 	default:
 		return statusFromError(pkt.id(), fmt.Errorf("unexpected method: %s", r.Method))
@@ -596,6 +604,23 @@ func filestat(h FileLister, r *Request, pkt requestPacket) responsePacket {
 	}
 }
 
+func readlink(readlinkFileLister ReadlinkFileLister, r *Request, pkt requestPacket) responsePacket {
+	resolved, err := readlinkFileLister.Readlink(r.Filepath)
+	if err != nil {
+		return statusFromError(pkt.id(), err)
+	}
+	return &sshFxpNamePacket{
+		ID: pkt.id(),
+		NameAttrs: []*sshFxpNameAttr{
+			{
+				Name:     resolved,
+				LongName: resolved,
+				Attrs:    emptyFileStat,
+			},
+		},
+	}
+}
+
 // init attributes of request object from packet data
 func requestMethod(p requestPacket) (method string) {
 	switch p.(type) {

vendor/github.com/pkg/sftp/request_windows.go

@@ -1,8 +1,6 @@
 package sftp
 
 import (
-	"path"
-	"path/filepath"
 	"syscall"
 )
 
@@ -13,32 +11,3 @@ func fakeFileInfoSys() interface{} {
 func testOsSys(sys interface{}) error {
 	return nil
 }
-
-func toLocalPath(p string) string {
-	lp := filepath.FromSlash(p)
-
-	if path.IsAbs(p) {
-		tmp := lp
-		for len(tmp) > 0 && tmp[0] == '\\' {
-			tmp = tmp[1:]
-		}
-
-		if filepath.IsAbs(tmp) {
-			// If the FromSlash without any starting slashes is absolute,
-			// then we have a filepath encoded with a prefix '/'.
-			// e.g. "/C:/Windows" to "C:\\Windows"
-			return tmp
-		}
-
-		tmp += "\\"
-
-		if filepath.IsAbs(tmp) {
-			// If the FromSlash without any starting slashes but with extra end slash is absolute,
-			// then we have a filepath encoded with a prefix '/' and a dropped '/' at the end.
-			// e.g. "/C:" to "C:\\"
-			return tmp
-		}
-	}
-
-	return lp
-}

vendor/github.com/pkg/sftp/server.go

@@ -24,7 +24,7 @@ const (
 // Server is an SSH File Transfer Protocol (sftp) server.
 // This is intended to provide the sftp subsystem to an ssh server daemon.
 // This implementation currently supports most of sftp server protocol version 3,
-// as specified at http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
+// as specified at https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
 type Server struct {
 	*serverConn
 	debugStream   io.Writer
@@ -33,6 +33,7 @@ type Server struct {
 	openFiles     map[string]*os.File
 	openFilesLock sync.RWMutex
 	handleCount   int
+	workDir       string
 }
 
 func (svr *Server) nextHandle(f *os.File) string {
@@ -128,6 +129,16 @@ func WithAllocator() ServerOption {
 	}
 }
 
+// WithServerWorkingDirectory sets a working directory to use as base
+// for relative paths.
+// If unset the default is current working directory (os.Getwd).
+func WithServerWorkingDirectory(workDir string) ServerOption {
+	return func(s *Server) error {
+		s.workDir = cleanPath(workDir)
+		return nil
+	}
+}
+
 type rxPacket struct {
 	pktType  fxp
 	pktBytes []byte
@@ -174,7 +185,7 @@ func handlePacket(s *Server, p orderedRequest) error {
 		}
 	case *sshFxpStatPacket:
 		// stat the requested file
-		info, err := os.Stat(toLocalPath(p.Path))
+		info, err := os.Stat(s.toLocalPath(p.Path))
 		rpkt = &sshFxpStatResponse{
 			ID:   p.ID,
 			info: info,
@@ -184,7 +195,7 @@ func handlePacket(s *Server, p orderedRequest) error {
 		}
 	case *sshFxpLstatPacket:
 		// stat the requested file
-		info, err := os.Lstat(toLocalPath(p.Path))
+		info, err := os.Lstat(s.toLocalPath(p.Path))
 		rpkt = &sshFxpStatResponse{
 			ID:   p.ID,
 			info: info,
@@ -208,24 +219,24 @@ func handlePacket(s *Server, p orderedRequest) error {
 		}
 	case *sshFxpMkdirPacket:
 		// TODO FIXME: ignore flags field
-		err := os.Mkdir(toLocalPath(p.Path), 0755)
+		err := os.Mkdir(s.toLocalPath(p.Path), 0o755)
 		rpkt = statusFromError(p.ID, err)
 	case *sshFxpRmdirPacket:
-		err := os.Remove(toLocalPath(p.Path))
+		err := os.Remove(s.toLocalPath(p.Path))
 		rpkt = statusFromError(p.ID, err)
 	case *sshFxpRemovePacket:
-		err := os.Remove(toLocalPath(p.Filename))
+		err := os.Remove(s.toLocalPath(p.Filename))
 		rpkt = statusFromError(p.ID, err)
 	case *sshFxpRenamePacket:
-		err := os.Rename(toLocalPath(p.Oldpath), toLocalPath(p.Newpath))
+		err := os.Rename(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath))
 		rpkt = statusFromError(p.ID, err)
 	case *sshFxpSymlinkPacket:
-		err := os.Symlink(toLocalPath(p.Targetpath), toLocalPath(p.Linkpath))
+		err := os.Symlink(s.toLocalPath(p.Targetpath), s.toLocalPath(p.Linkpath))
 		rpkt = statusFromError(p.ID, err)
 	case *sshFxpClosePacket:
 		rpkt = statusFromError(p.ID, s.closeHandle(p.Handle))
 	case *sshFxpReadlinkPacket:
-		f, err := os.Readlink(toLocalPath(p.Path))
+		f, err := os.Readlink(s.toLocalPath(p.Path))
 		rpkt = &sshFxpNamePacket{
 			ID: p.ID,
 			NameAttrs: []*sshFxpNameAttr{
@@ -240,7 +251,7 @@ func handlePacket(s *Server, p orderedRequest) error {
 			rpkt = statusFromError(p.ID, err)
 		}
 	case *sshFxpRealpathPacket:
-		f, err := filepath.Abs(toLocalPath(p.Path))
+		f, err := filepath.Abs(s.toLocalPath(p.Path))
 		f = cleanPath(f)
 		rpkt = &sshFxpNamePacket{
 			ID: p.ID,
@@ -256,13 +267,14 @@ func handlePacket(s *Server, p orderedRequest) error {
 			rpkt = statusFromError(p.ID, err)
 		}
 	case *sshFxpOpendirPacket:
-		p.Path = toLocalPath(p.Path)
+		lp := s.toLocalPath(p.Path)
 
-		if stat, err := os.Stat(p.Path); err != nil {
+		if stat, err := os.Stat(lp); err != nil {
 			rpkt = statusFromError(p.ID, err)
 		} else if !stat.IsDir() {
 			rpkt = statusFromError(p.ID, &os.PathError{
-				Path: p.Path, Err: syscall.ENOTDIR})
+				Path: lp, Err: syscall.ENOTDIR,
+			})
 		} else {
 			rpkt = (&sshFxpOpenPacket{
 				ID:     p.ID,
@@ -315,7 +327,7 @@ func handlePacket(s *Server, p orderedRequest) error {
 }
 
 // Serve serves SFTP connections until the streams stop or the SFTP subsystem
-// is stopped.
+// is stopped. It returns nil if the server exits cleanly.
 func (svr *Server) Serve() error {
 	defer func() {
 		if svr.pktMgr.alloc != nil {
@@ -341,6 +353,10 @@ func (svr *Server) Serve() error {
 	for {
 		pktType, pktBytes, err = svr.serverConn.recvPacket(svr.pktMgr.getNextOrderID())
 		if err != nil {
+			// Check whether the connection terminated cleanly in-between packets.
+			if err == io.EOF {
+				err = nil
+			}
 			// we don't care about releasing allocated pages here, the server will quit and the allocator freed
 			break
 		}
@@ -446,7 +462,7 @@ func (p *sshFxpOpenPacket) respond(svr *Server) responsePacket {
 		osFlags |= os.O_EXCL
 	}
 
-	f, err := os.OpenFile(toLocalPath(p.Path), osFlags, 0644)
+	f, err := os.OpenFile(svr.toLocalPath(p.Path), osFlags, 0o644)
 	if err != nil {
 		return statusFromError(p.ID, err)
 	}
@@ -484,7 +500,7 @@ func (p *sshFxpSetstatPacket) respond(svr *Server) responsePacket {
 	b := p.Attrs.([]byte)
 	var err error
 
-	p.Path = toLocalPath(p.Path)
+	p.Path = svr.toLocalPath(p.Path)
 
 	debug("setstat name \"%s\"", p.Path)
 	if (p.Flags & sshFileXferAttrSize) != 0 {
@@ -603,13 +619,15 @@ func statusFromError(id uint32, err error) *sshFxpStatusPacket {
 		return ret
 	}
 
-	switch e := err.(type) {
+	if errors.Is(err, io.EOF) {
-	case fxerr:
+		ret.StatusError.Code = sshFxEOF
+		return ret
+	}
+
+	var e fxerr
+	if errors.As(err, &e) {
 		ret.StatusError.Code = uint32(e)
-	default:
+		return ret
-		if e == io.EOF {
-			ret.StatusError.Code = sshFxEOF
-		}
 	}
 
 	return ret

vendor/github.com/pkg/sftp/server_plan9.go

@@ -0,0 +1,27 @@
+package sftp
+
+import (
+	"path"
+	"path/filepath"
+)
+
+func (s *Server) toLocalPath(p string) string {
+	if s.workDir != "" && !path.IsAbs(p) {
+		p = path.Join(s.workDir, p)
+	}
+
+	lp := filepath.FromSlash(p)
+
+	if path.IsAbs(p) {
+		tmp := lp[1:]
+
+		if filepath.IsAbs(tmp) {
+			// If the FromSlash without any starting slashes is absolute,
+			// then we have a filepath encoded with a prefix '/'.
+			// e.g. "/#s/boot" to "#s/boot"
+			return tmp
+		}
+	}
+
+	return lp
+}

vendor/github.com/pkg/sftp/server_statvfs_impl.go

@@ -1,3 +1,4 @@
+//go:build darwin || linux
 // +build darwin linux
 
 // fill in statvfs structure with OS specific values

vendor/github.com/pkg/sftp/server_statvfs_linux.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package sftp

vendor/github.com/pkg/sftp/server_statvfs_stubs.go

@@ -1,3 +1,4 @@
+//go:build !darwin && !linux && !plan9
 // +build !darwin,!linux,!plan9
 
 package sftp

vendor/github.com/pkg/sftp/server_unix.go

@@ -0,0 +1,16 @@
+//go:build !windows && !plan9
+// +build !windows,!plan9
+
+package sftp
+
+import (
+	"path"
+)
+
+func (s *Server) toLocalPath(p string) string {
+	if s.workDir != "" && !path.IsAbs(p) {
+		p = path.Join(s.workDir, p)
+	}
+
+	return p
+}

vendor/github.com/pkg/sftp/server_windows.go

@@ -0,0 +1,39 @@
+package sftp
+
+import (
+	"path"
+	"path/filepath"
+)
+
+func (s *Server) toLocalPath(p string) string {
+	if s.workDir != "" && !path.IsAbs(p) {
+		p = path.Join(s.workDir, p)
+	}
+
+	lp := filepath.FromSlash(p)
+
+	if path.IsAbs(p) {
+		tmp := lp
+		for len(tmp) > 0 && tmp[0] == '\\' {
+			tmp = tmp[1:]
+		}
+
+		if filepath.IsAbs(tmp) {
+			// If the FromSlash without any starting slashes is absolute,
+			// then we have a filepath encoded with a prefix '/'.
+			// e.g. "/C:/Windows" to "C:\\Windows"
+			return tmp
+		}
+
+		tmp += "\\"
+
+		if filepath.IsAbs(tmp) {
+			// If the FromSlash without any starting slashes but with extra end slash is absolute,
+			// then we have a filepath encoded with a prefix '/' and a dropped '/' at the end.
+			// e.g. "/C:" to "C:\\"
+			return tmp
+		}
+	}
+
+	return lp
+}

vendor/github.com/pkg/sftp/sftp.go

@@ -1,5 +1,5 @@
 // Package sftp implements the SSH File Transfer Protocol as described in
-// https://tools.ietf.org/html/draft-ietf-secsh-filexfer-02
+// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
 package sftp
 
 import (

vendor/github.com/pkg/sftp/stat_posix.go

@@ -1,3 +1,4 @@
+//go:build !plan9
 // +build !plan9
 
 package sftp
@@ -23,7 +24,7 @@ func translateErrno(errno syscall.Errno) uint32 {
 		return sshFxOk
 	case syscall.ENOENT:
 		return sshFxNoSuchFile
-	case syscall.EPERM:
+	case syscall.EACCES, syscall.EPERM:
 		return sshFxPermissionDenied
 	}
 

vendor/github.com/pkg/sftp/syscall_fixed.go

@@ -1,3 +1,4 @@
+//go:build plan9 || windows || (js && wasm)
 // +build plan9 windows js,wasm
 
 // Go defines S_IFMT on windows, plan9 and js/wasm as 0x1f000 instead of

vendor/github.com/pkg/sftp/syscall_good.go

@@ -1,4 +1,6 @@
-// +build !plan9,!windows
+//go:build !plan9 && !windows && (!js || !wasm)
+// +build !plan9
+// +build !windows
 // +build !js !wasm
 
 package sftp

vendor/golang.org/x/crypto/AUTHORS

@@ -1,3 +0,0 @@
-# This source code refers to The Go Authors for copyright purposes.
-# The master list of authors is in the main Go distribution,
-# visible at https://tip.golang.org/AUTHORS.