ready for v1.0rc1

Makefile

@@ -4,23 +4,29 @@ VERSION=$(shell cat VERSION)
 GCFLAGS=-gcflags '-e'
 LDFLAGS=-ldflags '-X main.version=$(VERSION) -s -w'
 PROGRAM=docstore
-ENV=CGO_ENABLED=0
+ENV=CGO_ENABLED=0 GO111MODULE=on
 
 all: $(PROGRAM)
 
 setup:
 	$(CC) get github.com/client9/misspell
 
-vendor: clean codeqa
+release: clean generate gofmt codeqa
+	$(ENV) $(CC) $(BUILD) $(LDFLAGS)
+
+release-vendor: clean generate gofmt codeqa
 	$(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS)
 
-$(PROGRAM): clean codeqa
+vendor: clean generate gofmt
+	$(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS)
+
+$(PROGRAM): clean generate gofmt
 	$(ENV) $(CC) $(BUILD) $(LDFLAGS)
 
 clean:
 	$(CC) clean -x
 
-codeqa: generate gofmt misspell golint test
+codeqa: misspell golint test
 
 generate:
 	$(CC) generate
@@ -45,4 +51,4 @@ else
 	$(info skipping tests of other platforms)
 endif
 
-.PHONY: setup vendor build clean codeqa generate gofmt golint misspell test
+.PHONY: setup release release-vendor vendor build clean codeqa generate gofmt golint misspell test

README.md

@@ -25,3 +25,31 @@ GOOS=openbsd make vendor
 ## Install
 
 See the `scripts/` directory on how to install docstore on OpenBSD.
+
+## Config
+
+```
+{
+	"run_dir": "/var/lib/docstore",    // running directory. data will be stored here
+	"user": "docstore",                // drop privileges to user
+	"group": "docstore",               // drop privileges to group
+	"listen_addr": "127.0.0.1:8080",   // listening <address>:<port>
+	"log_file": "docstore.log",        // log file (use - for stdout, this only works in combination with 'foreground: true')
+	"secret": "",                      // static hmac secret (at least 64 chars, used for JWT signing)
+	"langs": ["deu", "eng"],           // languages for tesseract ocr
+	"cmd_timeout": 600,                // timeout in seconds for executed teseract and pdftotext commands
+	"tesseract_threads": 1,            // tesseract thread limit
+	"tesseract_oem": 1,                // tesseract oem value
+	"foreground": false,               // do not fork into the background
+	"secure_cookie": false,            // enable secure cookie flag
+	"debug": false                     // enable debugging
+}
+```
+
+## Usage
+
+DocStore manages all files and directories under the `$run_dir/data` directory.
+
+Do not delete or move files around in the `$run_dir/data` directory using the filesystem as this messes with the index mapping.
+
+You can however add more files and directories using the underlying file system.

main.go

@@ -2,6 +2,8 @@
 
 //go:generate ./templates.sh
 
+// +build go1.12
+
 package main
 
 import "git.giftfish.de/ston1th/docstore/pkg/cmd"

pkg/cmd/cmd.go

@@ -30,7 +30,9 @@ const (
 	defRunGroup = "docstore"
 
 	defTimeout = 600
+
 	tesseractThreads = 1
+	tesseractOEM     = 1
 )
 
 var (
@@ -76,7 +78,7 @@ func initServer(cfg core.Config) (err error) {
 	debugCfg.Secret = "***hidden***"
 	log.Debugf("docstore config: %+v", debugCfg)
 
-	scanner, err := scan.New(cfg.DataDir, cfg.Langs, cfg.TesseractThreads, time.Second*time.Duration(cfg.CMDTimeout))
+	scanner, err := scan.New(cfg)
 	if err != nil {
 		return errors.New("scanner: " + err.Error())
 	}
@@ -265,6 +267,12 @@ func serverFlags() []cli.Flag {
 			Usage:       "tesseract thread limit (1-4)",
 			Destination: &config.TesseractThreads,
 		},
+		cli.IntFlag{
+			Name:        "oem",
+			Value:       tesseractOEM,
+			Usage:       "tesseract oem value (1-4)",
+			Destination: &config.TesseractOEM,
+		},
 		cli.BoolFlag{
 			Name:        "foreground, f",
 			Usage:       "do not fork into the background",
@@ -272,7 +280,7 @@ func serverFlags() []cli.Flag {
 		},
 		cli.BoolFlag{
 			Name:        "secure, s",
-			Usage:       "enable secure cookie",
+			Usage:       "enable secure cookie flag",
 			Destination: &config.SecureCookie,
 		},
 		cli.BoolFlag{

pkg/core/types.go

@@ -45,6 +45,7 @@ type Config struct {
 	Langs            []string `json:"langs,omitempty"`
 	CMDTimeout       int      `json:"cmd_timeout,omitempty"`
 	TesseractThreads int      `json:"tesseract_threads,omitempty"`
+	TesseractOEM     int      `json:"tesseract_oem,omitempty"`
 	Foreground       bool     `json:"foreground,omitempty"`
 	SecureCookie     bool     `json:"secure_cookie,omitempty"`
 	Debug            bool     `json:"debug,omitempty"`

pkg/fs/file.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 )
 
 type Mode int
@@ -34,10 +35,11 @@ func (fs *Filesystem) GetFile(path string) (f *os.File, fi os.FileInfo, err erro
 	return
 }
 
-func (fs *Filesystem) Mode(path string) (mode Mode, err error) {
+func (fs *Filesystem) Mode(path string) (mode Mode, enoent bool, err error) {
 	p := filepath.Join(fs.Base, Clean(path))
 	fi, err := os.Stat(p)
 	if err != nil {
+		enoent = err.(*os.PathError).Err == syscall.ENOENT
 		return
 	}
 	m := fi.Mode()

pkg/scan/args.go

@@ -1,7 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-// +build !openbsd
-
-package scan
-
-const tesseract_oem = "2"

pkg/scan/args_openbsd.go

@@ -1,7 +0,0 @@
-// Copyright (C) 2019 Marius Schellenberger
-
-// +build openbsd
-
-package scan
-
-const tesseract_oem = "1"

pkg/scan/scanner.go

@@ -50,7 +50,7 @@ type Scanner struct {
 	p       PDF
 }
 
-func New(base string, langs []string, threads int, timeout time.Duration) (s *Scanner, err error) {
+func New(c core.Config) (s *Scanner, err error) {
 	var (
 		tcmd string
 		pcmd string
@@ -72,24 +72,27 @@ func New(base string, langs []string, threads int, timeout time.Duration) (s *Sc
 	if err != nil {
 		return
 	}
-	if langs == nil {
+	llen := len(c.Langs)
+	langs := c.Langs
+	if langs == nil || llen == 0 {
 		langs = defaultLangs
 	}
 	for i, l := range langs {
 		lang += l
-		if i < len(langs)-1 {
+		if i < llen-1 {
 			lang += "+"
 		}
 	}
+	threads := c.TesseractThreads
 	if threads > maxThreads {
 		threads = maxThreads
 	} else if threads < minThreads {
 		threads = minThreads
 	}
-	args := []string{"-l", lang, "--oem", tesseract_oem}
+	args := []string{"-l", lang, "--oem", fmt.Sprintf("%d", c.TesseractOEM)}
 	s = &Scanner{
-		base,
+		c.DataDir,
-		timeout,
+		time.Second * time.Duration(c.CMDTimeout),
 		Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", threads)}},
 		PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}},
 	}

pkg/server/context.go

@@ -66,7 +66,7 @@ type webData struct {
 
 	Login bool
 
-	Key string
+	NotFound string
 
 	User   string
 	Token  string

pkg/server/handler.go

@@ -280,7 +280,7 @@ func newDirHandler(ctx *Context) {
 
 func moveHandler(ctx *Context) {
 	path := strings.TrimPrefix(ctx.Path(), core.MovePrefix)
-	mode, err := ctx.Srv.FS.Mode(path)
+	mode, _, err := ctx.Srv.FS.Mode(path)
 	if err != nil {
 		ctx.Error(err)
 		return
@@ -323,11 +323,12 @@ func moveHandler(ctx *Context) {
 			return
 		}
 		for _, m := range moved {
-			log.Debugf("move: %s to %s", m.Old, m.New)
 			err = ctx.Srv.DB.MoveFile(m.Old, m.New)
 			if err != nil {
 				log.Printf("move: %s: %s", m, err)
+				continue
 			}
+			log.Debugf("move: %s to %s", m.Old, m.New)
 		}
 		ctx.Redirect(newfile, http.StatusFound)
 	}
@@ -335,8 +336,8 @@ func moveHandler(ctx *Context) {
 
 func deleteHandler(ctx *Context) {
 	path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix)
-	mode, err := ctx.Srv.FS.Mode(path)
+	mode, enoent, err := ctx.Srv.FS.Mode(path)
-	if err != nil {
+	if !enoent && err != nil {
 		ctx.Error(err)
 		return
 	}
@@ -352,6 +353,7 @@ func deleteHandler(ctx *Context) {
 			BodyTitle: "Delete Directory",
 		}
 	default:
+		if ctx.Method() == "GET" {
 			ctx.Data = webData{
 				Title:     "Delete",
 				BodyTitle: "Delete",
@@ -359,6 +361,7 @@ func deleteHandler(ctx *Context) {
 			ctx.Error(noSuchFile)
 			return
 		}
+	}
 	ctx.Data.Path = path
 	ctx.Data.Data = mode
 	ctx.Template("deleteHandler")
@@ -369,6 +372,27 @@ func deleteHandler(ctx *Context) {
 		if !ctx.CheckXsrf() {
 			return
 		}
+		p, _ := filepath.Split(path)
+		p = filepath.Dir(p)
+		clean := ctx.Form("clean")
+		if clean == "true" {
+			f := fs.Clean(path)
+			id, err := ctx.Srv.DB.DeleteFile(f)
+			if err != nil {
+				log.Printf("db: delete: %s %s", f, err)
+			}
+			if id == "" {
+				ctx.Redirect(p, http.StatusFound)
+				return
+			}
+			log.Debugf("delete %s id: %s from index", f, id)
+			err = ctx.Srv.DB.Index.Delete(id)
+			if err != nil {
+				log.Printf("index: delete: %s %s", f, err)
+			}
+			ctx.Redirect(p, http.StatusFound)
+			return
+		}
 		files, err := ctx.Srv.FS.Delete(path)
 		if err != nil {
 			ctx.Error(err)
@@ -379,17 +403,15 @@ func deleteHandler(ctx *Context) {
 			if err != nil {
 				log.Printf("db: delete: %s %s", f, err)
 			}
-			log.Debugf("delete %s id: %s from index", f, id)
 			if id == "" {
 				continue
 			}
+			log.Debugf("delete %s id: %s from index", f, id)
 			err = ctx.Srv.DB.Index.Delete(id)
 			if err != nil {
 				log.Printf("index: delete: %s %s", f, err)
 			}
 		}
-		p, _ := filepath.Split(path)
-		p = filepath.Dir(p)
 		ctx.Redirect(p, http.StatusFound)
 	}
 }
@@ -399,8 +421,8 @@ func dirHandler(ctx *Context) {
 	path := ctx.Path()
 	switch ctx.Method() {
 	case "GET":
-		mode, err := ctx.Srv.FS.Mode(path)
+		mode, enoent, err := ctx.Srv.FS.Mode(path)
-		if err != nil {
+		if !enoent && err != nil {
 			ctx.Error(err)
 			return
 		}
@@ -429,10 +451,17 @@ func dirHandler(ctx *Context) {
 			ctx.Data.Paths = fs.Paths(path)
 			ctx.Exec()
 		default:
+			p := fs.Clean(path)
+			if enoent && ctx.Srv.DB.IsIndexed(p) {
+				ctx.Template("enoentHandler")
 				ctx.Data = webData{
-				Title: "Viewer",
+					Title: "Stalled File",
 				}
-			ctx.Error(noSuchFile)
+				ctx.Data.Path = p
+				ctx.Exec()
+				return
+			}
+			ctx.Error(err)
 		}
 	}
 }

pkg/server/templates.go

@@ -96,6 +96,18 @@ const (
     </tbody>
     </table>
 </div>
+{{end}}`
+	enoent = `{{define "body"}}
+<div class="alert alert-danger msg">
+    <h4>File missing!</h4>
+    <p>This file is indexed but has been removed from the file system.</p>
+    <p>Do you want to delete this entry from the index?</p>
+</div>
+<form class="form-horizontal" action="/delete{{.Path}}" method="post">
+    <input type="hidden" name="token" value="{{.Token}}">
+    <input type="hidden" name="clean" value="true">
+    <button class="btn btn-sm btn-danger" type="submit">Delete</button>
+</form>
 {{end}}`
 	file = `{{define "body"}}
 <div class="row">
@@ -427,7 +439,7 @@ const (
                 <input type="hidden" name="token" value="{{.Token}}">
                 <div class="form-group">
                     <label class="col-form-label" for="name">Name</label>
-                    <input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}">
+                    <input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" placeholder="optional">
                 </div>
                 <div class="form-group">
                     <div class="input-group mb-3">
@@ -721,6 +733,7 @@ func (s *HTTPServer) loadTemplates() {
 	s.templ["uploadHandler"] = parse(index, menu, upload)
 	s.templ["newDirHandler"] = parse(index, menu, newDir)
 	s.templ["moveHandler"] = parse(index, menu, move)
+	s.templ["enoentHandler"] = parse(index, menu, enoent)
 	s.templ["deleteHandler"] = parse(index, menu, delete)
 	s.templ["notFoundHandler"] = parse(index, menu, notFound)
 	s.templ["forbiddenHandler"] = parse(index, menu, forbidden)

templates.sh

@@ -85,6 +85,7 @@ func (s *HTTPServer) loadTemplates() {
 	s.templ["uploadHandler"] = parse(index, menu, upload)
 	s.templ["newDirHandler"] = parse(index, menu, newDir)
 	s.templ["moveHandler"] = parse(index, menu, move)
+	s.templ["enoentHandler"] = parse(index, menu, enoent)
 	s.templ["deleteHandler"] = parse(index, menu, delete)
 	s.templ["notFoundHandler"] = parse(index, menu, notFound)
 	s.templ["forbiddenHandler"] = parse(index, menu, forbidden)

templates/enoent.html

@@ -0,0 +1,12 @@
+{{define "body"}}
+<div class="alert alert-danger msg">
+    <h4>File missing!</h4>
+    <p>This file is indexed but has been removed from the file system.</p>
+    <p>Do you want to delete this entry from the index?</p>
+</div>
+<form class="form-horizontal" action="/delete{{.Path}}" method="post">
+    <input type="hidden" name="token" value="{{.Token}}">
+    <input type="hidden" name="clean" value="true">
+    <button class="btn btn-sm btn-danger" type="submit">Delete</button>
+</form>
+{{end}}

templates/upload.html

@@ -10,7 +10,7 @@
                 <input type="hidden" name="token" value="{{.Token}}">
                 <div class="form-group">
                     <label class="col-form-label" for="name">Name</label>
-                    <input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}">
+                    <input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" placeholder="optional">
                 </div>
                 <div class="form-group">
                     <div class="input-group mb-3">