ston1th/cachefs
0
0
Fork 0
Code Issues Releases Activity

added preload status and cancel

Browse source
This commit is contained in:
ston1th 2022-03-16 01:58:10 +01:00
commit a069baf48d
10 changed files with 153 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

2
TODO.txt
View file

@ -1,3 +1,5 @@
* add disk quota
* preload: dont load file if quota exceeded
* add LRU cache
* implement directory listing cache
* clear cache if file sizes differ and > 0

7
pkg/fs/chunk.go
View file

@ -30,6 +30,13 @@ func (cs *Chunks) Add(off int64, n int) {
cs.merge()
}
func (cs Chunks) Size() (sum int64) {
for _, c := range cs {
sum += (c[1] - c[0])
}
return
}
func (cs *Chunks) merge() {
c := *cs
if len(c) < 2 {

32
pkg/fs/dir.go Normal file
View file

@ -0,0 +1,32 @@
// Copyright (C) 2022 Marius Schellenberger
package fs
import (
"io"
"os"
)
type Dir struct {
f *os.File
}
func (f *Dir) Read(_ []byte) (int, error) {
return 0, io.EOF
}
func (f *Dir) Seek(_ int64, _ int) (int64, error) {
return 0, io.EOF
}
func (f *Dir) Readdir(count int) ([]os.FileInfo, error) {
return f.f.Readdir(count)
}
func (f *Dir) Stat() (os.FileInfo, error) {
return f.f.Stat()
}
func (f *Dir) Close() error {
return f.f.Close()
}

28
pkg/fs/file.go
View file

@ -3,6 +3,7 @@
package fs
import (
"context"
"errors"
"io"
"os"
@ -22,9 +23,15 @@ type preload struct {
f *File
skipped int64
written int64
ctx context.Context
}
func (p *preload) Read(data []byte) (n int, err error) {
select {
case <-p.ctx.Done():
return 0, p.ctx.Err()
default:
}
if p.f.offset >= p.f.size() {
return 0, io.EOF
}
@ -39,7 +46,7 @@ func (p *preload) Read(data []byte) (n int, err error) {
return
}
func (f *File) Preload(unlock func()) {
func (f *File) Preload(ctx context.Context, unlock func()) {
log := f.log
if f.offline {
log.V(2).Error(errors.New("no preload in offline mode"), "error preloading file")
@ -47,8 +54,13 @@ func (f *File) Preload(unlock func()) {
return
}
log.V(2).Info("preload started")
p := &preload{f: f}
p := &preload{f: f, ctx: ctx}
_, err := io.Copy(io.Discard, p)
if err == context.Canceled {
log.V(2).Info("preload canceled", "skipped", p.skipped, "written", p.written)
unlock()
return
}
if err != nil && err != io.EOF {
log.Error(err, "error preloading file")
}
@ -103,11 +115,13 @@ func (f *File) readWithoutCache(p []byte) (n int, err error) {
}
func (f *File) Seek(offset int64, whence int) (n int64, err error) {
if !f.offline {
n, err = f.f.Seek(offset, whence)
if err != nil {
f.log.Error(err, "error seeking source file")
return
}
}
switch whence {
case io.SeekStart:
f.offset = offset
@ -117,14 +131,20 @@ func (f *File) Seek(offset int64, whence int) (n int64, err error) {
return
}
func (f *File) Readdir(count int) ([]os.FileInfo, error) {
return f.f.Readdir(count)
func (f *File) Readdir(_ int) ([]os.FileInfo, error) {
return nil, nil
}
func (f *File) Stat() (os.FileInfo, error) {
if f.offline {
return f.md.Stat()
}
return f.f.Stat()
}
func (f *File) Close() error {
if f.offline {
return nil
}
return f.f.Close()
}

44
pkg/fs/fs.go
View file

@ -8,6 +8,7 @@ import (
"errors"
"io"
stdfs "io/fs"
"math"
"net/http"
"os"
"path"
@ -28,6 +29,7 @@ type FS struct {
mdf *os.File
jenc *json.Encoder
mm map[string]*Metadata
prem map[string]func()
cancel func()
}
@ -55,6 +57,7 @@ func NewFS(src, dst, metadata string, log logr.Logger) (fs *FS, err error) {
dst: dst,
mdf: mdf,
mm: make(map[string]*Metadata),
prem: make(map[string]func()),
jenc: json.NewEncoder(mdf),
}
err = json.NewDecoder(mdf).Decode(&fs.mm)
@ -79,7 +82,7 @@ func (fs *FS) initMetadata() {
}
if len(v.Chunks) == 0 {
log.V(2).Info("removing empty file", "file", k)
err := fs.removeDst(k)
err := fs.RemoveDst(k)
if err != nil {
log.Error(err, "error removing empty file", "file", k)
continue
@ -110,7 +113,7 @@ func (fs *FS) StatWithOffline(name string) (fi stdfs.FileInfo, offline bool, err
return
}
func (fs *FS) removeDst(name string) error {
func (fs *FS) RemoveDst(name string) error {
_, dp := fs.paths(name)
return os.Remove(dp)
}
@ -120,25 +123,33 @@ func (fs *FS) statDst(name string) (fi stdfs.FileInfo, err error) {
return os.Stat(dp)
}
func (fs *FS) Open(name string) (http.File, error) {
return fs.open(name)
func (fs *FS) CancelPreload(name string) {
if cancel, ok := fs.prem[name]; ok {
cancel()
}
func (fs *FS) OpenFile(name string) (*File, error) {
return fs.open(name)
}
func (fs *FS) Preload(name string) {
f, err := fs.open(name)
file, err := fs.Open(name)
if err != nil {
return
}
f, ok := file.(*File)
if !ok {
return
}
if f.md.Preload() {
fs.log.WithValues("file", name).Error(errors.New("preload is running"), "error starting preload")
return
}
unlock := func() { f.md.UnlockPreload() }
go f.Preload(unlock)
ctx, cancel := context.WithCancel(context.Background())
fs.prem[name] = cancel
unlock := func() {
delete(fs.prem, name)
f.md.UnlockPreload()
}
go f.Preload(ctx, unlock)
}
func skipLog(name string) bool {
@ -152,7 +163,7 @@ func (fs *FS) paths(name string) (sp string, dp string) {
return
}
func (fs *FS) open(name string) (f *File, err error) {
func (fs *FS) Open(name string) (f http.File, err error) {
log := fs.log.WithValues("file", name)
sp, dp := fs.paths(name)
offline := false
@ -186,7 +197,7 @@ func (fs *FS) open(name string) (f *File, err error) {
offline = true
log.V(2).Info("dir offline mode", "path", dp)
}
return &File{f: sf, offline: offline}, err
return &Dir{f: sf}, err
}
md := fs.metadata(name, sfi.Size())
return &File{
@ -244,6 +255,15 @@ func (fs *FS) metadata(name string, size int64) (md *Metadata) {
return
}
func (fs *FS) CacheStatus(name string) int {
fs.mu.RLock()
defer fs.mu.RUnlock()
if md, ok := fs.mm[name]; ok {
return int(math.Round(float64(md.Chunks.Size()) / float64(md.Size) * 100))
}
return -1
}
func (fs *FS) flusher(ctx context.Context) {
for {
select {

19
pkg/fs/metadata.go
View file

@ -17,6 +17,15 @@ type Metadata struct {
preload bool `json:"-"`
}
func (md *Metadata) Delete() error {
md.mu.Lock()
defer md.mu.Unlock()
md.Chunks = Chunks{}
md.f.Close()
md.f = nil
return md.fs.RemoveDst(md.name)
}
func (md *Metadata) HasChunk(off int64, n int) bool {
md.mu.RLock()
defer md.mu.RUnlock()
@ -75,3 +84,13 @@ func (md *Metadata) UnlockPreload() {
defer md.mu.Unlock()
md.preload = false
}
func (md *Metadata) Stat() (os.FileInfo, error) {
if md.f == nil {
err := md.openCacheFile()
if err != nil {
return nil, err
}
}
return md.f.Stat()
}

8
pkg/fs/webdav.go
View file

@ -4,13 +4,14 @@ package fs
import (
"context"
"net/http"
"os"
"golang.org/x/net/webdav"
)
type WebDavFile struct {
*File
http.File
}
func (*WebDavFile) Write(_ []byte) (int, error) {
@ -41,7 +42,10 @@ func (w *WebDavFS) OpenFile(_ context.Context, name string, flags int, _ os.File
flags&os.O_TRUNC == os.O_TRUNC {
return nil, os.ErrPermission
}
f, err := w.fs.OpenFile(name)
f, err := w.fs.Open(name)
if err != nil {
return nil, err
}
return &WebDavFile{f}, err
}

12
pkg/srv/interceptor.go
View file

@ -8,6 +8,8 @@ import (
"net/http"
"sort"
"strings"
"cachefs/pkg/fs"
)
type statusInterceptor struct {
@ -44,6 +46,7 @@ type dirContents struct {
type file struct {
Name string
Anchor string
Status int
}
type files []file
@ -77,16 +80,21 @@ func (r *responseInterceptor) Status() int {
return r.status
}
func (r *responseInterceptor) GetPaths() (dir dirContents, err error) {
func (r *responseInterceptor) GetPaths(path string, fs *fs.FS) (dir dirContents, err error) {
err = xml.Unmarshal(r.buf.Bytes(), &dir)
if err != nil {
return
}
path = strings.TrimSuffix(path, "/")
for _, p := range dir.AllPaths {
if p[len(p)-1] == '/' {
dir.Dirs = append(dir.Dirs, p)
} else {
dir.Files = append(dir.Files, file{Name: p, Anchor: anchor(p)})
dir.Files = append(dir.Files, file{
Name: p,
Anchor: anchor(p),
Status: fs.CacheStatus(path + "/" + p),
})
}
}
sort.Strings(dir.Dirs)

13
pkg/srv/srv.go
View file

@ -57,7 +57,8 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, msg, code)
return
}
if r.FormValue("v") == "t" {
option := r.FormValue("o")
if option == "v" {
err = video.Execute(w, r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -66,8 +67,12 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if !d.IsDir() {
if r.FormValue("p") == "t" {
if option == "p" || option == "sp" {
if option == "p" {
fs.fs.Preload(p)
} else {
fs.fs.CancelPreload(p)
}
rdir := "/"
if i := strings.LastIndex(p, "/"); i > 0 {
rdir = p[:i]
@ -75,7 +80,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, rdir+"#"+anchor(p), http.StatusFound)
return
}
if r.FormValue("n") == "t" {
if option == "n" {
fs.fs.NoCache.ServeHTTP(w, r)
return
}
@ -94,7 +99,7 @@ func (fs *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if offline {
w.Header().Del("Last-Modified")
}
paths, err := i.GetPaths()
paths, err := i.GetPaths(p, fs.fs)
if err == io.EOF {
w.WriteHeader(i.Status())
return

11
pkg/srv/templates.go
View file

@ -9,9 +9,9 @@ var (
<html>
<head>
<style>
body { text-align: center; padding: 0 150px 0 150px; font: 20px Helvetica, sans-serif; color: #333; }
article { display: block; text-align: left; width: 650px; margin: 0 auto; }
table { text-align: left; }
body { padding: 0 150px 0 150px; font: 20px Helvetica, sans-serif; color: #333; }
article { display: block; text-align: left; margin: 0 auto; }
table { text-align: left; width: 100%; }
</style>
</head>
<body>
@ -20,16 +20,17 @@ var (
[v]: show video
[n]: skip file caching
[p]: preload file
[s]: stop preloading
</pre>
<table>
<tr><th>Path</th><th>Options</th></tr>
<tr><th style="width:100%;">Path</th><th>Options</th></tr>
<tr><td><a href="../">../</a></td><td></td></tr>
{{range $s := .Dirs -}}
<tr><td><a href="{{$s}}">{{$s}}</a></td><td></td></tr>
{{end -}}
{{range $s := .Files -}}
<tr><td><a id="{{$s.Anchor}}" href="{{$s.Name}}">{{$s.Name}}</a></td>
<td><a href="{{$s.Name}}?v=t">[v]</a>&nbsp;<a href="{{$s.Name}}?n=t">[n]</a>&nbsp;<a href="{{$s.Name}}?p=t">[p]</a></td></tr>
<td><a href="{{$s.Name}}?o=v">[v]</a>&nbsp;<a href="{{$s.Name}}?o=n">[n]</a>&nbsp;<a href="{{$s.Name}}?o=p">[p]</a>&nbsp;<a href="{{$s.Name}}?o=sp">[sp]</a>&nbsp;{{if ge $s.Status 0}}{{$s.Status}}%{{end}}</td></tr>
{{end -}}
</table>
</article>