diff --git a/Makefile b/Makefile index 6eb03ed..3e78265 100644 --- a/Makefile +++ b/Makefile @@ -4,23 +4,29 @@ VERSION=$(shell cat VERSION) GCFLAGS=-gcflags '-e' LDFLAGS=-ldflags '-X main.version=$(VERSION) -s -w' PROGRAM=docstore -ENV=CGO_ENABLED=0 +ENV=CGO_ENABLED=0 GO111MODULE=on all: $(PROGRAM) setup: $(CC) get github.com/client9/misspell -vendor: clean codeqa +release: clean generate gofmt codeqa + $(ENV) $(CC) $(BUILD) $(LDFLAGS) + +release-vendor: clean generate gofmt codeqa $(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS) -$(PROGRAM): clean codeqa +vendor: clean generate gofmt + $(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS) + +$(PROGRAM): clean generate gofmt $(ENV) $(CC) $(BUILD) $(LDFLAGS) clean: $(CC) clean -x -codeqa: generate gofmt misspell golint test +codeqa: misspell golint test generate: $(CC) generate @@ -45,4 +51,4 @@ else $(info skipping tests of other platforms) endif -.PHONY: setup vendor build clean codeqa generate gofmt golint misspell test +.PHONY: setup release release-vendor vendor build clean codeqa generate gofmt golint misspell test diff --git a/README.md b/README.md index a15d39d..c4926d5 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,31 @@ GOOS=openbsd make vendor ## Install See the `scripts/` directory on how to install docstore on OpenBSD. + +## Config + +``` +{ + "run_dir": "/var/lib/docstore", // running directory. data will be stored here + "user": "docstore", // drop privileges to user + "group": "docstore", // drop privileges to group + "listen_addr": "127.0.0.1:8080", // listening
: + "log_file": "docstore.log", // log file (use - for stdout, this only works in combination with 'foreground: true') + "secret": "", // static hmac secret (at least 64 chars, used for JWT signing) + "langs": ["deu", "eng"], // languages for tesseract ocr + "cmd_timeout": 600, // timeout in seconds for executed teseract and pdftotext commands + "tesseract_threads": 1, // tesseract thread limit + "tesseract_oem": 1, // tesseract oem value + "foreground": false, // do not fork into the background + "secure_cookie": false, // enable secure cookie flag + "debug": false // enable debugging +} +``` + +## Usage + +DocStore manages all files and directories under the `$run_dir/data` directory. + +Do not delete or move files around in the `$run_dir/data` directory using the filesystem as this messes with the index mapping. + +You can however add more files and directories using the underlying file system. diff --git a/main.go b/main.go index ee17219..65029ed 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,8 @@ //go:generate ./templates.sh +// +build go1.12 + package main import "git.giftfish.de/ston1th/docstore/pkg/cmd" diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index a5dd7a1..15802b5 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -29,8 +29,10 @@ const ( defRunUser = "docstore" defRunGroup = "docstore" - defTimeout = 600 + defTimeout = 600 + tesseractThreads = 1 + tesseractOEM = 1 ) var ( @@ -76,7 +78,7 @@ func initServer(cfg core.Config) (err error) { debugCfg.Secret = "***hidden***" log.Debugf("docstore config: %+v", debugCfg) - scanner, err := scan.New(cfg.DataDir, cfg.Langs, cfg.TesseractThreads, time.Second*time.Duration(cfg.CMDTimeout)) + scanner, err := scan.New(cfg) if err != nil { return errors.New("scanner: " + err.Error()) } @@ -265,6 +267,12 @@ func serverFlags() []cli.Flag { Usage: "tesseract thread limit (1-4)", Destination: &config.TesseractThreads, }, + cli.IntFlag{ + Name: "oem", + Value: tesseractOEM, + Usage: "tesseract oem value (1-4)", + Destination: &config.TesseractOEM, + }, cli.BoolFlag{ Name: "foreground, f", Usage: "do not fork into the background", @@ -272,7 +280,7 @@ func serverFlags() []cli.Flag { }, cli.BoolFlag{ Name: "secure, s", - Usage: "enable secure cookie", + Usage: "enable secure cookie flag", Destination: &config.SecureCookie, }, cli.BoolFlag{ diff --git a/pkg/core/types.go b/pkg/core/types.go index dda9512..488d740 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -45,6 +45,7 @@ type Config struct { Langs []string `json:"langs,omitempty"` CMDTimeout int `json:"cmd_timeout,omitempty"` TesseractThreads int `json:"tesseract_threads,omitempty"` + TesseractOEM int `json:"tesseract_oem,omitempty"` Foreground bool `json:"foreground,omitempty"` SecureCookie bool `json:"secure_cookie,omitempty"` Debug bool `json:"debug,omitempty"` diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 8c9c9fd..7cf5874 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "syscall" ) type Mode int @@ -34,10 +35,11 @@ func (fs *Filesystem) GetFile(path string) (f *os.File, fi os.FileInfo, err erro return } -func (fs *Filesystem) Mode(path string) (mode Mode, err error) { +func (fs *Filesystem) Mode(path string) (mode Mode, enoent bool, err error) { p := filepath.Join(fs.Base, Clean(path)) fi, err := os.Stat(p) if err != nil { + enoent = err.(*os.PathError).Err == syscall.ENOENT return } m := fi.Mode() diff --git a/pkg/scan/args.go b/pkg/scan/args.go deleted file mode 100644 index ad1ab73..0000000 --- a/pkg/scan/args.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (C) 2019 Marius Schellenberger - -// +build !openbsd - -package scan - -const tesseract_oem = "2" diff --git a/pkg/scan/args_openbsd.go b/pkg/scan/args_openbsd.go deleted file mode 100644 index 4f866f4..0000000 --- a/pkg/scan/args_openbsd.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (C) 2019 Marius Schellenberger - -// +build openbsd - -package scan - -const tesseract_oem = "1" diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 6507682..7f14b09 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -50,7 +50,7 @@ type Scanner struct { p PDF } -func New(base string, langs []string, threads int, timeout time.Duration) (s *Scanner, err error) { +func New(c core.Config) (s *Scanner, err error) { var ( tcmd string pcmd string @@ -72,24 +72,27 @@ func New(base string, langs []string, threads int, timeout time.Duration) (s *Sc if err != nil { return } - if langs == nil { + llen := len(c.Langs) + langs := c.Langs + if langs == nil || llen == 0 { langs = defaultLangs } for i, l := range langs { lang += l - if i < len(langs)-1 { + if i < llen-1 { lang += "+" } } + threads := c.TesseractThreads if threads > maxThreads { threads = maxThreads } else if threads < minThreads { threads = minThreads } - args := []string{"-l", lang, "--oem", tesseract_oem} + args := []string{"-l", lang, "--oem", fmt.Sprintf("%d", c.TesseractOEM)} s = &Scanner{ - base, - timeout, + c.DataDir, + time.Second * time.Duration(c.CMDTimeout), Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", threads)}}, PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}}, } diff --git a/pkg/server/context.go b/pkg/server/context.go index 6ae2bda..8040156 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -66,7 +66,7 @@ type webData struct { Login bool - Key string + NotFound string User string Token string diff --git a/pkg/server/handler.go b/pkg/server/handler.go index 06fd7cb..1b34280 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -280,7 +280,7 @@ func newDirHandler(ctx *Context) { func moveHandler(ctx *Context) { path := strings.TrimPrefix(ctx.Path(), core.MovePrefix) - mode, err := ctx.Srv.FS.Mode(path) + mode, _, err := ctx.Srv.FS.Mode(path) if err != nil { ctx.Error(err) return @@ -323,11 +323,12 @@ func moveHandler(ctx *Context) { return } for _, m := range moved { - log.Debugf("move: %s to %s", m.Old, m.New) err = ctx.Srv.DB.MoveFile(m.Old, m.New) if err != nil { log.Printf("move: %s: %s", m, err) + continue } + log.Debugf("move: %s to %s", m.Old, m.New) } ctx.Redirect(newfile, http.StatusFound) } @@ -335,8 +336,8 @@ func moveHandler(ctx *Context) { func deleteHandler(ctx *Context) { path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix) - mode, err := ctx.Srv.FS.Mode(path) - if err != nil { + mode, enoent, err := ctx.Srv.FS.Mode(path) + if !enoent && err != nil { ctx.Error(err) return } @@ -352,12 +353,14 @@ func deleteHandler(ctx *Context) { BodyTitle: "Delete Directory", } default: - ctx.Data = webData{ - Title: "Delete", - BodyTitle: "Delete", + if ctx.Method() == "GET" { + ctx.Data = webData{ + Title: "Delete", + BodyTitle: "Delete", + } + ctx.Error(noSuchFile) + return } - ctx.Error(noSuchFile) - return } ctx.Data.Path = path ctx.Data.Data = mode @@ -369,6 +372,27 @@ func deleteHandler(ctx *Context) { if !ctx.CheckXsrf() { return } + p, _ := filepath.Split(path) + p = filepath.Dir(p) + clean := ctx.Form("clean") + if clean == "true" { + f := fs.Clean(path) + id, err := ctx.Srv.DB.DeleteFile(f) + if err != nil { + log.Printf("db: delete: %s %s", f, err) + } + if id == "" { + ctx.Redirect(p, http.StatusFound) + return + } + log.Debugf("delete %s id: %s from index", f, id) + err = ctx.Srv.DB.Index.Delete(id) + if err != nil { + log.Printf("index: delete: %s %s", f, err) + } + ctx.Redirect(p, http.StatusFound) + return + } files, err := ctx.Srv.FS.Delete(path) if err != nil { ctx.Error(err) @@ -379,17 +403,15 @@ func deleteHandler(ctx *Context) { if err != nil { log.Printf("db: delete: %s %s", f, err) } - log.Debugf("delete %s id: %s from index", f, id) if id == "" { continue } + log.Debugf("delete %s id: %s from index", f, id) err = ctx.Srv.DB.Index.Delete(id) if err != nil { log.Printf("index: delete: %s %s", f, err) } } - p, _ := filepath.Split(path) - p = filepath.Dir(p) ctx.Redirect(p, http.StatusFound) } } @@ -399,8 +421,8 @@ func dirHandler(ctx *Context) { path := ctx.Path() switch ctx.Method() { case "GET": - mode, err := ctx.Srv.FS.Mode(path) - if err != nil { + mode, enoent, err := ctx.Srv.FS.Mode(path) + if !enoent && err != nil { ctx.Error(err) return } @@ -429,10 +451,17 @@ func dirHandler(ctx *Context) { ctx.Data.Paths = fs.Paths(path) ctx.Exec() default: - ctx.Data = webData{ - Title: "Viewer", + p := fs.Clean(path) + if enoent && ctx.Srv.DB.IsIndexed(p) { + ctx.Template("enoentHandler") + ctx.Data = webData{ + Title: "Stalled File", + } + ctx.Data.Path = p + ctx.Exec() + return } - ctx.Error(noSuchFile) + ctx.Error(err) } } } diff --git a/pkg/server/templates.go b/pkg/server/templates.go index f490e68..69047d9 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -96,6 +96,18 @@ const ( +{{end}}` + enoent = `{{define "body"}} +
+

File missing!

+

This file is indexed but has been removed from the file system.

+

Do you want to delete this entry from the index?

+
+
+ + + +
{{end}}` file = `{{define "body"}}
@@ -427,7 +439,7 @@ const (
- +
@@ -721,6 +733,7 @@ func (s *HTTPServer) loadTemplates() { s.templ["uploadHandler"] = parse(index, menu, upload) s.templ["newDirHandler"] = parse(index, menu, newDir) s.templ["moveHandler"] = parse(index, menu, move) + s.templ["enoentHandler"] = parse(index, menu, enoent) s.templ["deleteHandler"] = parse(index, menu, delete) s.templ["notFoundHandler"] = parse(index, menu, notFound) s.templ["forbiddenHandler"] = parse(index, menu, forbidden) diff --git a/templates.sh b/templates.sh index 1fc36e6..f5b32ba 100755 --- a/templates.sh +++ b/templates.sh @@ -85,6 +85,7 @@ func (s *HTTPServer) loadTemplates() { s.templ["uploadHandler"] = parse(index, menu, upload) s.templ["newDirHandler"] = parse(index, menu, newDir) s.templ["moveHandler"] = parse(index, menu, move) + s.templ["enoentHandler"] = parse(index, menu, enoent) s.templ["deleteHandler"] = parse(index, menu, delete) s.templ["notFoundHandler"] = parse(index, menu, notFound) s.templ["forbiddenHandler"] = parse(index, menu, forbidden) diff --git a/templates/enoent.html b/templates/enoent.html new file mode 100644 index 0000000..5b9da19 --- /dev/null +++ b/templates/enoent.html @@ -0,0 +1,12 @@ +{{define "body"}} +
+

File missing!

+

This file is indexed but has been removed from the file system.

+

Do you want to delete this entry from the index?

+
+
+ + + +
+{{end}} diff --git a/templates/upload.html b/templates/upload.html index b5cb159..3e4cf1e 100644 --- a/templates/upload.html +++ b/templates/upload.html @@ -10,7 +10,7 @@
- +