ston1th/cachefs
0
0
Fork 0
Code Issues Releases Activity

added webdav

Browse source
This commit is contained in:
ston1th 2022-03-14 22:58:36 +01:00
commit 78bd754298
25 changed files with 7783 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

179
pkg/fs/fs.go
View file

@ -61,40 +61,75 @@ func NewFS(src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
if err == io.EOF {
err = nil
}
for k := range fs.mm {
_, err = stat(fs.dst, k)
if errors.Is(err, os.ErrNotExist) {
delete(fs.mm, k)
}
}
fs.initMetadata()
ctx, cancel := context.WithCancel(context.Background())
fs.cancel = cancel
go fs.flusher(ctx)
return fs, err
}
func (fs *FS) initMetadata() {
log := fs.log
for k, v := range fs.mm {
_, err := fs.statDst(k)
if errors.Is(err, os.ErrNotExist) {
log.V(2).Info("removing not existing file from metadata", "file", k)
delete(fs.mm, k)
continue
}
if len(v.Chunks) == 0 {
log.V(2).Info("removing empty file", "file", k)
err := fs.removeDst(k)
if err != nil {
log.Error(err, "error removing empty file", "file", k)
continue
}
delete(fs.mm, k)
continue
}
v.fs = fs
v.name = k
}
}
func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
fi, _, err = fs.StatWithOffline(name)
return
}
func (fs *FS) StatWithOffline(name string) (fi stdfs.FileInfo, offline bool, err error) {
f, err := fs.open(name, true)
if err != nil {
return
sp, dp := fs.paths(name)
fi, err = os.Stat(sp)
if errors.Is(err, os.ErrNotExist) {
fi, err = os.Stat(dp)
if err != nil {
return
}
offline = true
}
fi, err = f.Stat()
f.Close()
offline = f.offline
return
}
func (fs *FS) removeDst(name string) error {
_, dp := fs.paths(name)
return os.Remove(dp)
}
func (fs *FS) statDst(name string) (fi stdfs.FileInfo, err error) {
_, dp := fs.paths(name)
return os.Stat(dp)
}
func (fs *FS) Open(name string) (http.File, error) {
return fs.open(name, false)
return fs.open(name)
}
func (fs *FS) OpenFile(name string) (*File, error) {
return fs.open(name)
}
func (fs *FS) Preload(name string) {
f, err := fs.open(name, false)
f, err := fs.open(name)
if err != nil {
return
}
@ -110,11 +145,16 @@ func skipLog(name string) bool {
return strings.HasSuffix(name, "index.html") || strings.HasSuffix(name, "favicon.ico")
}
func (fs *FS) open(name string, stat bool) (f *File, err error) {
log := fs.log.WithValues("file", name)
func (fs *FS) paths(name string) (sp string, dp string) {
p := filepath.FromSlash(path.Clean("/" + name))
sp := filepath.Join(fs.src, p)
dp := filepath.Join(fs.dst, p)
sp = filepath.Join(fs.src, p)
dp = filepath.Join(fs.dst, p)
return
}
func (fs *FS) open(name string) (f *File, err error) {
log := fs.log.WithValues("file", name)
sp, dp := fs.paths(name)
offline := false
sf, err := os.Open(sp)
if err != nil {
@ -128,6 +168,9 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
}
offline = true
log.V(2).Info("file offline mode", "path", dp)
} else {
log.Error(err, "error opening source file")
return
}
}
sfi, err := sf.Stat()
@ -136,8 +179,8 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
log.Error(err, "error stat source file")
return
}
if sfi.IsDir() || stat {
if empty, err := dirEmpty(sf.Name()); empty && err == nil {
if sfi.IsDir() {
if empty, e := dirEmpty(sf.Name()); empty && e == nil {
sf.Close()
sf, err = os.Open(dp)
offline = true
@ -145,41 +188,7 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
}
return &File{f: sf, offline: offline}, err
}
var df *os.File
i := strings.LastIndex(dp, "/")
if i > 0 {
dir := dp[:i]
_, err = os.Stat(dir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(dir, 0o755)
if err != nil {
log.Error(err, "error creating cache dir", "dir", dp)
return nil, err
}
}
}
_, err = os.Stat(dp)
truncate := errors.Is(err, os.ErrNotExist)
if !fs.isOpen(name) {
df, err = os.OpenFile(dp, os.O_RDWR|os.O_CREATE, 0o644)
if err != nil {
log.Error(err, "error opening cache file")
return nil, err
}
if truncate {
err = df.Truncate(sfi.Size())
if err != nil {
log.Error(err, "error truncating cache file")
return nil, err
}
err = df.Sync()
if err != nil {
log.Error(err, "error syncing cache file")
return nil, err
}
}
}
md := fs.metadata(name, sfi.Size(), df)
md := fs.metadata(name, sfi.Size())
return &File{
log: log,
f: sf,
@ -188,23 +197,49 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
}, nil
}
func (fs *FS) isOpen(name string) bool {
fs.mu.RLock()
defer fs.mu.RUnlock()
md, ok := fs.mm[name]
return ok && md.f != nil
func (fs *FS) openCacheFile(name string, size int64) (df *os.File, err error) {
_, dp := fs.paths(name)
log := fs.log
if i := strings.LastIndex(dp, "/"); i > 0 {
dir := dp[:i]
_, err = os.Stat(dir)
if errors.Is(err, os.ErrNotExist) {
err = os.MkdirAll(dir, 0o755)
if err != nil {
log.Error(err, "error creating cache dir", "dir", dp)
return
}
}
}
_, err = os.Stat(dp)
truncate := errors.Is(err, os.ErrNotExist)
df, err = os.OpenFile(dp, os.O_RDWR|os.O_CREATE, 0o644)
if err != nil {
log.Error(err, "error opening cache file")
return
}
if truncate {
err = df.Truncate(size)
if err != nil {
log.Error(err, "error truncating cache file")
return
}
err = df.Sync()
if err != nil {
log.Error(err, "error syncing cache file")
return
}
}
return
}
func (fs *FS) metadata(name string, size int64, f *os.File) (md *Metadata) {
func (fs *FS) metadata(name string, size int64) (md *Metadata) {
fs.mu.Lock()
defer fs.mu.Unlock()
if md, ok := fs.mm[name]; ok {
if md.f == nil {
md.f = f
}
return md
}
md = &Metadata{Size: size, f: f}
md = &Metadata{Size: size, fs: fs, name: name}
fs.mm[name] = md
return
}
@ -224,6 +259,12 @@ func (fs *FS) flushMetadata() {
log := fs.log
fs.mu.Lock()
defer fs.mu.Unlock()
mm := make(map[string]*Metadata)
for k, v := range fs.mm {
if v.f != nil || len(v.Chunks) > 0 {
mm[k] = v
}
}
_, err := fs.mdf.Seek(0, io.SeekStart)
if err != nil {
log.Error(err, "failure seeking metadata file")
@ -234,7 +275,7 @@ func (fs *FS) flushMetadata() {
log.Error(err, "failure truncating metadata file")
return
}
err = fs.jenc.Encode(fs.mm)
err = fs.jenc.Encode(mm)
if err != nil {
log.Error(err, "failure flushing metadata file")
}
@ -261,9 +302,3 @@ func dirEmpty(name string) (bool, error) {
}
return false, err
}
func stat(dst, name string) (fi stdfs.FileInfo, err error) {
p := filepath.FromSlash(path.Clean("/" + name))
dp := filepath.Join(dst, p)
return os.Stat(dp)
}

25
pkg/fs/metadata.go
View file

@ -9,7 +9,9 @@ import (
type Metadata struct {
mu sync.RWMutex `json:"-"`
fs *FS `json:"-"`
f *os.File `json:"-"`
name string `json:"-"`
Size int64 `json:"s"`
Chunks Chunks `json:"c"`
preload bool `json:"-"`
@ -28,13 +30,36 @@ func (md *Metadata) AddChunk(off int64, n int) {
}
func (md *Metadata) ReadAt(p []byte, pos int64) (int, error) {
if md.f == nil {
err := md.openCacheFile()
if err != nil {
return 0, err
}
}
return md.f.ReadAt(p, pos)
}
func (md *Metadata) WriteAt(data []byte, pos int64) (int, error) {
if md.f == nil {
err := md.openCacheFile()
if err != nil {
return 0, err
}
}
return md.f.WriteAt(data, pos)
}
func (md *Metadata) openCacheFile() error {
md.mu.Lock()
defer md.mu.Unlock()
if md.f != nil {
return nil
}
f, err := md.fs.openCacheFile(md.name, md.Size)
md.f = f
return err
}
func (md *Metadata) Preload() bool {
md.mu.Lock()
defer md.mu.Unlock()

53
pkg/fs/webdav.go Normal file
View file

@ -0,0 +1,53 @@
package fs
import (
"context"
"os"
"golang.org/x/net/webdav"
)
type WebDavFile struct {
*File
}
func (*WebDavFile) Write(_ []byte) (int, error) {
return 0, os.ErrPermission
}
type WebDavFS struct {
fs *FS
}
func NewWebDavFS(fs *FS) *WebDavFS {
return &WebDavFS{fs}
}
func (*WebDavFS) Mkdir(_ context.Context, _ string, _ os.FileMode) error {
return os.ErrPermission
}
func (*WebDavFS) RemoveAll(_ context.Context, _ string) error {
return os.ErrPermission
}
func (*WebDavFS) Rename(_ context.Context, _, _ string) error {
return os.ErrPermission
}
func (w *WebDavFS) OpenFile(_ context.Context, name string, flags int, _ os.FileMode) (webdav.File, error) {
//O_WRONLY int = syscall.O_WRONLY // open the file write-only.
//O_RDWR int = syscall.O_RDWR // open the file read-write.
//O_APPEND int = syscall.O_APPEND // append data to the file when writing.
//O_CREATE int = syscall.O_CREAT // create a new file if none exists.
//O_EXCL int = syscall.O_EXCL // used with O_CREATE, file must not exist.
if flags&os.O_RDWR == os.O_RDWR ||
flags&os.O_CREATE == os.O_CREATE ||
flags&os.O_TRUNC == os.O_TRUNC {
return nil, os.ErrPermission
}
f, err := w.fs.OpenFile(name)
return &WebDavFile{f}, err
}
func (w *WebDavFS) Stat(_ context.Context, name string) (os.FileInfo, error) {
return w.fs.Stat(name)
}

29
pkg/srv/srv.go
View file

@ -32,24 +32,26 @@ func skipLog(path string) bool {
func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s := &statusInterceptor{w: w}
w = s
defer func() {
if skipLog(r.RequestURI) {
return
}
fs.log.Info("access",
"client", r.RemoteAddr,
"method", r.Method,
"status", s.Status(),
"uri", r.RequestURI,
)
}()
if !skipLog(r.RequestURI) {
defer func() {
if s.Status() == http.StatusPartialContent {
return
}
fs.log.Info("access",
"client", r.RemoteAddr,
"method", r.Method,
"status", s.Status(),
"uri", r.RequestURI,
)
}()
}
upath := r.URL.Path
if !strings.HasPrefix(upath, "/") {
upath = "/" + upath
r.URL.Path = upath
}
p := path.Clean(upath)
d, offline, err := fs.fs.StatWithOffline(p)
d, _, err := fs.fs.StatWithOffline(p)
if err != nil {
msg, code := toHTTPError(err)
http.Error(w, msg, code)
@ -83,7 +85,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
i := &responseInterceptor{w: w}
// TODO implement directory listing cache
offline = true
offline := true
if offline {
r.Header.Del("If-Modified-Since")
r.Header.Del("Cache-Control")
@ -117,6 +119,5 @@ func toHTTPError(err error) (string, int) {
if errors.Is(err, stdfs.ErrPermission) {
return "403 Forbidden", http.StatusForbidden
}
// Default:
return "500 Internal Server Error", http.StatusInternalServerError
}

74
pkg/srv/webdav.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright (C) 2022 Marius Schellenberger
package srv
import (
"net/http"
"strings"
"cachefs/pkg/fs"
"github.com/go-logr/logr"
"golang.org/x/net/webdav"
)
type WebDavServer struct {
log logr.Logger
h http.Handler
}
func NewWebDavServer(fs *fs.WebDavFS, log logr.Logger) http.Handler {
return &WebDavServer{
log,
&webdav.Handler{
FileSystem: fs,
LockSystem: webdav.NewMemLS(),
},
}
}
func (dav *WebDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s := &statusInterceptor{w: w}
if r.Method == http.MethodGet && !skipDavLog(r.RequestURI) {
defer func() {
if s.Status() == http.StatusPartialContent {
return
}
dav.log.Info("access",
"client", r.RemoteAddr,
"method", r.Method,
"status", s.Status(),
"uri", r.RequestURI,
)
}()
}
dav.h.ServeHTTP(s, r)
}
var skipDavFiles = []string{
"favicon.ico",
"AlbumArtSmall.jpg",
"AlbumArt.jpg",
"Album.jpg",
"cover.jpg",
"cover.png",
"cover.gif",
"front.jpg",
"front.png",
"front.gif",
"front.bmp",
"thumb.jpg",
}
func skipDavLog(path string) bool {
i := strings.LastIndex(path, "/")
if i > 0 {
path = path[i+1:]
}
for _, v := range skipDavFiles {
if path == v {
return true
}
}
return false
}