added webdav
This commit is contained in:
parent
2e78593526
commit
78bd754298
25 changed files with 7783 additions and 90 deletions
179
pkg/fs/fs.go
179
pkg/fs/fs.go
|
|
@ -61,40 +61,75 @@ func NewFS(src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
|
|||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
for k := range fs.mm {
|
||||
_, err = stat(fs.dst, k)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
delete(fs.mm, k)
|
||||
}
|
||||
}
|
||||
fs.initMetadata()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
fs.cancel = cancel
|
||||
go fs.flusher(ctx)
|
||||
return fs, err
|
||||
}
|
||||
|
||||
func (fs *FS) initMetadata() {
|
||||
log := fs.log
|
||||
for k, v := range fs.mm {
|
||||
_, err := fs.statDst(k)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.V(2).Info("removing not existing file from metadata", "file", k)
|
||||
delete(fs.mm, k)
|
||||
continue
|
||||
}
|
||||
if len(v.Chunks) == 0 {
|
||||
log.V(2).Info("removing empty file", "file", k)
|
||||
err := fs.removeDst(k)
|
||||
if err != nil {
|
||||
log.Error(err, "error removing empty file", "file", k)
|
||||
continue
|
||||
}
|
||||
delete(fs.mm, k)
|
||||
continue
|
||||
}
|
||||
v.fs = fs
|
||||
v.name = k
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FS) Stat(name string) (fi stdfs.FileInfo, err error) {
|
||||
fi, _, err = fs.StatWithOffline(name)
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FS) StatWithOffline(name string) (fi stdfs.FileInfo, offline bool, err error) {
|
||||
f, err := fs.open(name, true)
|
||||
if err != nil {
|
||||
return
|
||||
sp, dp := fs.paths(name)
|
||||
fi, err = os.Stat(sp)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
fi, err = os.Stat(dp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
offline = true
|
||||
}
|
||||
fi, err = f.Stat()
|
||||
f.Close()
|
||||
offline = f.offline
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FS) removeDst(name string) error {
|
||||
_, dp := fs.paths(name)
|
||||
return os.Remove(dp)
|
||||
}
|
||||
|
||||
func (fs *FS) statDst(name string) (fi stdfs.FileInfo, err error) {
|
||||
_, dp := fs.paths(name)
|
||||
return os.Stat(dp)
|
||||
}
|
||||
|
||||
func (fs *FS) Open(name string) (http.File, error) {
|
||||
return fs.open(name, false)
|
||||
return fs.open(name)
|
||||
}
|
||||
|
||||
func (fs *FS) OpenFile(name string) (*File, error) {
|
||||
return fs.open(name)
|
||||
}
|
||||
|
||||
func (fs *FS) Preload(name string) {
|
||||
f, err := fs.open(name, false)
|
||||
f, err := fs.open(name)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
@ -110,11 +145,16 @@ func skipLog(name string) bool {
|
|||
return strings.HasSuffix(name, "index.html") || strings.HasSuffix(name, "favicon.ico")
|
||||
}
|
||||
|
||||
func (fs *FS) open(name string, stat bool) (f *File, err error) {
|
||||
log := fs.log.WithValues("file", name)
|
||||
func (fs *FS) paths(name string) (sp string, dp string) {
|
||||
p := filepath.FromSlash(path.Clean("/" + name))
|
||||
sp := filepath.Join(fs.src, p)
|
||||
dp := filepath.Join(fs.dst, p)
|
||||
sp = filepath.Join(fs.src, p)
|
||||
dp = filepath.Join(fs.dst, p)
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FS) open(name string) (f *File, err error) {
|
||||
log := fs.log.WithValues("file", name)
|
||||
sp, dp := fs.paths(name)
|
||||
offline := false
|
||||
sf, err := os.Open(sp)
|
||||
if err != nil {
|
||||
|
|
@ -128,6 +168,9 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
|
|||
}
|
||||
offline = true
|
||||
log.V(2).Info("file offline mode", "path", dp)
|
||||
} else {
|
||||
log.Error(err, "error opening source file")
|
||||
return
|
||||
}
|
||||
}
|
||||
sfi, err := sf.Stat()
|
||||
|
|
@ -136,8 +179,8 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
|
|||
log.Error(err, "error stat source file")
|
||||
return
|
||||
}
|
||||
if sfi.IsDir() || stat {
|
||||
if empty, err := dirEmpty(sf.Name()); empty && err == nil {
|
||||
if sfi.IsDir() {
|
||||
if empty, e := dirEmpty(sf.Name()); empty && e == nil {
|
||||
sf.Close()
|
||||
sf, err = os.Open(dp)
|
||||
offline = true
|
||||
|
|
@ -145,41 +188,7 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
|
|||
}
|
||||
return &File{f: sf, offline: offline}, err
|
||||
}
|
||||
var df *os.File
|
||||
i := strings.LastIndex(dp, "/")
|
||||
if i > 0 {
|
||||
dir := dp[:i]
|
||||
_, err = os.Stat(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.MkdirAll(dir, 0o755)
|
||||
if err != nil {
|
||||
log.Error(err, "error creating cache dir", "dir", dp)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = os.Stat(dp)
|
||||
truncate := errors.Is(err, os.ErrNotExist)
|
||||
if !fs.isOpen(name) {
|
||||
df, err = os.OpenFile(dp, os.O_RDWR|os.O_CREATE, 0o644)
|
||||
if err != nil {
|
||||
log.Error(err, "error opening cache file")
|
||||
return nil, err
|
||||
}
|
||||
if truncate {
|
||||
err = df.Truncate(sfi.Size())
|
||||
if err != nil {
|
||||
log.Error(err, "error truncating cache file")
|
||||
return nil, err
|
||||
}
|
||||
err = df.Sync()
|
||||
if err != nil {
|
||||
log.Error(err, "error syncing cache file")
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
md := fs.metadata(name, sfi.Size(), df)
|
||||
md := fs.metadata(name, sfi.Size())
|
||||
return &File{
|
||||
log: log,
|
||||
f: sf,
|
||||
|
|
@ -188,23 +197,49 @@ func (fs *FS) open(name string, stat bool) (f *File, err error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (fs *FS) isOpen(name string) bool {
|
||||
fs.mu.RLock()
|
||||
defer fs.mu.RUnlock()
|
||||
md, ok := fs.mm[name]
|
||||
return ok && md.f != nil
|
||||
func (fs *FS) openCacheFile(name string, size int64) (df *os.File, err error) {
|
||||
_, dp := fs.paths(name)
|
||||
log := fs.log
|
||||
if i := strings.LastIndex(dp, "/"); i > 0 {
|
||||
dir := dp[:i]
|
||||
_, err = os.Stat(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.MkdirAll(dir, 0o755)
|
||||
if err != nil {
|
||||
log.Error(err, "error creating cache dir", "dir", dp)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = os.Stat(dp)
|
||||
truncate := errors.Is(err, os.ErrNotExist)
|
||||
df, err = os.OpenFile(dp, os.O_RDWR|os.O_CREATE, 0o644)
|
||||
if err != nil {
|
||||
log.Error(err, "error opening cache file")
|
||||
return
|
||||
}
|
||||
if truncate {
|
||||
err = df.Truncate(size)
|
||||
if err != nil {
|
||||
log.Error(err, "error truncating cache file")
|
||||
return
|
||||
}
|
||||
err = df.Sync()
|
||||
if err != nil {
|
||||
log.Error(err, "error syncing cache file")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (fs *FS) metadata(name string, size int64, f *os.File) (md *Metadata) {
|
||||
func (fs *FS) metadata(name string, size int64) (md *Metadata) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
if md, ok := fs.mm[name]; ok {
|
||||
if md.f == nil {
|
||||
md.f = f
|
||||
}
|
||||
return md
|
||||
}
|
||||
md = &Metadata{Size: size, f: f}
|
||||
md = &Metadata{Size: size, fs: fs, name: name}
|
||||
fs.mm[name] = md
|
||||
return
|
||||
}
|
||||
|
|
@ -224,6 +259,12 @@ func (fs *FS) flushMetadata() {
|
|||
log := fs.log
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
mm := make(map[string]*Metadata)
|
||||
for k, v := range fs.mm {
|
||||
if v.f != nil || len(v.Chunks) > 0 {
|
||||
mm[k] = v
|
||||
}
|
||||
}
|
||||
_, err := fs.mdf.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
log.Error(err, "failure seeking metadata file")
|
||||
|
|
@ -234,7 +275,7 @@ func (fs *FS) flushMetadata() {
|
|||
log.Error(err, "failure truncating metadata file")
|
||||
return
|
||||
}
|
||||
err = fs.jenc.Encode(fs.mm)
|
||||
err = fs.jenc.Encode(mm)
|
||||
if err != nil {
|
||||
log.Error(err, "failure flushing metadata file")
|
||||
}
|
||||
|
|
@ -261,9 +302,3 @@ func dirEmpty(name string) (bool, error) {
|
|||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func stat(dst, name string) (fi stdfs.FileInfo, err error) {
|
||||
p := filepath.FromSlash(path.Clean("/" + name))
|
||||
dp := filepath.Join(dst, p)
|
||||
return os.Stat(dp)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue