ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

ready for v1.0rc1

Browse source
This commit is contained in:
ston1th 2019-05-06 21:14:24 +02:00
commit ce5764840f
15 changed files with 140 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Makefile
View file

@ -4,23 +4,29 @@ VERSION=$(shell cat VERSION)
GCFLAGS=-gcflags '-e'
LDFLAGS=-ldflags '-X main.version=$(VERSION) -s -w'
PROGRAM=docstore
ENV=CGO_ENABLED=0
ENV=CGO_ENABLED=0 GO111MODULE=on
all: $(PROGRAM)
setup:
$(CC) get github.com/client9/misspell
vendor: clean codeqa
release: clean generate gofmt codeqa
$(ENV) $(CC) $(BUILD) $(LDFLAGS)
release-vendor: clean generate gofmt codeqa
$(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS)
$(PROGRAM): clean codeqa
vendor: clean generate gofmt
$(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS)
$(PROGRAM): clean generate gofmt
$(ENV) $(CC) $(BUILD) $(LDFLAGS)
clean:
$(CC) clean -x
codeqa: generate gofmt misspell golint test
codeqa: misspell golint test
generate:
$(CC) generate
@ -45,4 +51,4 @@ else
$(info skipping tests of other platforms)
endif
.PHONY: setup vendor build clean codeqa generate gofmt golint misspell test
.PHONY: setup release release-vendor vendor build clean codeqa generate gofmt golint misspell test

28
README.md
View file

@ -25,3 +25,31 @@ GOOS=openbsd make vendor
## Install
See the `scripts/` directory on how to install docstore on OpenBSD.
## Config
```
{
"run_dir": "/var/lib/docstore", // running directory. data will be stored here
"user": "docstore", // drop privileges to user
"group": "docstore", // drop privileges to group
"listen_addr": "127.0.0.1:8080", // listening <address>:<port>
"log_file": "docstore.log", // log file (use - for stdout, this only works in combination with 'foreground: true')
"secret": "", // static hmac secret (at least 64 chars, used for JWT signing)
"langs": ["deu", "eng"], // languages for tesseract ocr
"cmd_timeout": 600, // timeout in seconds for executed teseract and pdftotext commands
"tesseract_threads": 1, // tesseract thread limit
"tesseract_oem": 1, // tesseract oem value
"foreground": false, // do not fork into the background
"secure_cookie": false, // enable secure cookie flag
"debug": false // enable debugging
}
```
## Usage
DocStore manages all files and directories under the `$run_dir/data` directory.
Do not delete or move files around in the `$run_dir/data` directory using the filesystem as this messes with the index mapping.
You can however add more files and directories using the underlying file system.

2
main.go
View file

@ -2,6 +2,8 @@
//go:generate ./templates.sh
// +build go1.12
package main
import "git.giftfish.de/ston1th/docstore/pkg/cmd"

14
pkg/cmd/cmd.go
View file

@ -29,8 +29,10 @@ const (
defRunUser = "docstore"
defRunGroup = "docstore"
defTimeout = 600
defTimeout = 600
tesseractThreads = 1
tesseractOEM = 1
)
var (
@ -76,7 +78,7 @@ func initServer(cfg core.Config) (err error) {
debugCfg.Secret = "***hidden***"
log.Debugf("docstore config: %+v", debugCfg)
scanner, err := scan.New(cfg.DataDir, cfg.Langs, cfg.TesseractThreads, time.Second*time.Duration(cfg.CMDTimeout))
scanner, err := scan.New(cfg)
if err != nil {
return errors.New("scanner: " + err.Error())
}
@ -265,6 +267,12 @@ func serverFlags() []cli.Flag {
Usage: "tesseract thread limit (1-4)",
Destination: &config.TesseractThreads,
},
cli.IntFlag{
Name: "oem",
Value: tesseractOEM,
Usage: "tesseract oem value (1-4)",
Destination: &config.TesseractOEM,
},
cli.BoolFlag{
Name: "foreground, f",
Usage: "do not fork into the background",
@ -272,7 +280,7 @@ func serverFlags() []cli.Flag {
},
cli.BoolFlag{
Name: "secure, s",
Usage: "enable secure cookie",
Usage: "enable secure cookie flag",
Destination: &config.SecureCookie,
},
cli.BoolFlag{

1
pkg/core/types.go
View file

@ -45,6 +45,7 @@ type Config struct {
Langs []string `json:"langs,omitempty"`
CMDTimeout int `json:"cmd_timeout,omitempty"`
TesseractThreads int `json:"tesseract_threads,omitempty"`
TesseractOEM int `json:"tesseract_oem,omitempty"`
Foreground bool `json:"foreground,omitempty"`
SecureCookie bool `json:"secure_cookie,omitempty"`
Debug bool `json:"debug,omitempty"`

4
pkg/fs/file.go
View file

@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
)
type Mode int
@ -34,10 +35,11 @@ func (fs *Filesystem) GetFile(path string) (f *os.File, fi os.FileInfo, err erro
return
}
func (fs *Filesystem) Mode(path string) (mode Mode, err error) {
func (fs *Filesystem) Mode(path string) (mode Mode, enoent bool, err error) {
p := filepath.Join(fs.Base, Clean(path))
fi, err := os.Stat(p)
if err != nil {
enoent = err.(*os.PathError).Err == syscall.ENOENT
return
}
m := fi.Mode()

7
pkg/scan/args.go
View file

@ -1,7 +0,0 @@
// Copyright (C) 2019 Marius Schellenberger
// +build !openbsd
package scan
const tesseract_oem = "2"

7
pkg/scan/args_openbsd.go
View file

@ -1,7 +0,0 @@
// Copyright (C) 2019 Marius Schellenberger
// +build openbsd
package scan
const tesseract_oem = "1"

15
pkg/scan/scanner.go
View file

@ -50,7 +50,7 @@ type Scanner struct {
p PDF
}
func New(base string, langs []string, threads int, timeout time.Duration) (s *Scanner, err error) {
func New(c core.Config) (s *Scanner, err error) {
var (
tcmd string
pcmd string
@ -72,24 +72,27 @@ func New(base string, langs []string, threads int, timeout time.Duration) (s *Sc
if err != nil {
return
}
if langs == nil {
llen := len(c.Langs)
langs := c.Langs
if langs == nil || llen == 0 {
langs = defaultLangs
}
for i, l := range langs {
lang += l
if i < len(langs)-1 {
if i < llen-1 {
lang += "+"
}
}
threads := c.TesseractThreads
if threads > maxThreads {
threads = maxThreads
} else if threads < minThreads {
threads = minThreads
}
args := []string{"-l", lang, "--oem", tesseract_oem}
args := []string{"-l", lang, "--oem", fmt.Sprintf("%d", c.TesseractOEM)}
s = &Scanner{
base,
timeout,
c.DataDir,
time.Second * time.Duration(c.CMDTimeout),
Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", threads)}},
PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}},
}

2
pkg/server/context.go
View file

@ -66,7 +66,7 @@ type webData struct {
Login bool
Key string
NotFound string
User string
Token string

63
pkg/server/handler.go
View file

@ -280,7 +280,7 @@ func newDirHandler(ctx *Context) {
func moveHandler(ctx *Context) {
path := strings.TrimPrefix(ctx.Path(), core.MovePrefix)
mode, err := ctx.Srv.FS.Mode(path)
mode, _, err := ctx.Srv.FS.Mode(path)
if err != nil {
ctx.Error(err)
return
@ -323,11 +323,12 @@ func moveHandler(ctx *Context) {
return
}
for _, m := range moved {
log.Debugf("move: %s to %s", m.Old, m.New)
err = ctx.Srv.DB.MoveFile(m.Old, m.New)
if err != nil {
log.Printf("move: %s: %s", m, err)
continue
}
log.Debugf("move: %s to %s", m.Old, m.New)
}
ctx.Redirect(newfile, http.StatusFound)
}
@ -335,8 +336,8 @@ func moveHandler(ctx *Context) {
func deleteHandler(ctx *Context) {
path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix)
mode, err := ctx.Srv.FS.Mode(path)
if err != nil {
mode, enoent, err := ctx.Srv.FS.Mode(path)
if !enoent && err != nil {
ctx.Error(err)
return
}
@ -352,12 +353,14 @@ func deleteHandler(ctx *Context) {
BodyTitle: "Delete Directory",
}
default:
ctx.Data = webData{
Title: "Delete",
BodyTitle: "Delete",
if ctx.Method() == "GET" {
ctx.Data = webData{
Title: "Delete",
BodyTitle: "Delete",
}
ctx.Error(noSuchFile)
return
}
ctx.Error(noSuchFile)
return
}
ctx.Data.Path = path
ctx.Data.Data = mode
@ -369,6 +372,27 @@ func deleteHandler(ctx *Context) {
if !ctx.CheckXsrf() {
return
}
p, _ := filepath.Split(path)
p = filepath.Dir(p)
clean := ctx.Form("clean")
if clean == "true" {
f := fs.Clean(path)
id, err := ctx.Srv.DB.DeleteFile(f)
if err != nil {
log.Printf("db: delete: %s %s", f, err)
}
if id == "" {
ctx.Redirect(p, http.StatusFound)
return
}
log.Debugf("delete %s id: %s from index", f, id)
err = ctx.Srv.DB.Index.Delete(id)
if err != nil {
log.Printf("index: delete: %s %s", f, err)
}
ctx.Redirect(p, http.StatusFound)
return
}
files, err := ctx.Srv.FS.Delete(path)
if err != nil {
ctx.Error(err)
@ -379,17 +403,15 @@ func deleteHandler(ctx *Context) {
if err != nil {
log.Printf("db: delete: %s %s", f, err)
}
log.Debugf("delete %s id: %s from index", f, id)
if id == "" {
continue
}
log.Debugf("delete %s id: %s from index", f, id)
err = ctx.Srv.DB.Index.Delete(id)
if err != nil {
log.Printf("index: delete: %s %s", f, err)
}
}
p, _ := filepath.Split(path)
p = filepath.Dir(p)
ctx.Redirect(p, http.StatusFound)
}
}
@ -399,8 +421,8 @@ func dirHandler(ctx *Context) {
path := ctx.Path()
switch ctx.Method() {
case "GET":
mode, err := ctx.Srv.FS.Mode(path)
if err != nil {
mode, enoent, err := ctx.Srv.FS.Mode(path)
if !enoent && err != nil {
ctx.Error(err)
return
}
@ -429,10 +451,17 @@ func dirHandler(ctx *Context) {
ctx.Data.Paths = fs.Paths(path)
ctx.Exec()
default:
ctx.Data = webData{
Title: "Viewer",
p := fs.Clean(path)
if enoent && ctx.Srv.DB.IsIndexed(p) {
ctx.Template("enoentHandler")
ctx.Data = webData{
Title: "Stalled File",
}
ctx.Data.Path = p
ctx.Exec()
return
}
ctx.Error(noSuchFile)
ctx.Error(err)
}
}
}

15
pkg/server/templates.go
View file

@ -96,6 +96,18 @@ const (
</tbody>
</table>
</div>
{{end}}`
enoent = `{{define "body"}}
<div class="alert alert-danger msg">
<h4>File missing!</h4>
<p>This file is indexed but has been removed from the file system.</p>
<p>Do you want to delete this entry from the index?</p>
</div>
<form class="form-horizontal" action="/delete{{.Path}}" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<input type="hidden" name="clean" value="true">
<button class="btn btn-sm btn-danger" type="submit">Delete</button>
</form>
{{end}}`
file = `{{define "body"}}
<div class="row">
@ -427,7 +439,7 @@ const (
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}">
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" placeholder="optional">
</div>
<div class="form-group">
<div class="input-group mb-3">
@ -721,6 +733,7 @@ func (s *HTTPServer) loadTemplates() {
s.templ["uploadHandler"] = parse(index, menu, upload)
s.templ["newDirHandler"] = parse(index, menu, newDir)
s.templ["moveHandler"] = parse(index, menu, move)
s.templ["enoentHandler"] = parse(index, menu, enoent)
s.templ["deleteHandler"] = parse(index, menu, delete)
s.templ["notFoundHandler"] = parse(index, menu, notFound)
s.templ["forbiddenHandler"] = parse(index, menu, forbidden)

1
templates.sh
View file

@ -85,6 +85,7 @@ func (s *HTTPServer) loadTemplates() {
s.templ["uploadHandler"] = parse(index, menu, upload)
s.templ["newDirHandler"] = parse(index, menu, newDir)
s.templ["moveHandler"] = parse(index, menu, move)
s.templ["enoentHandler"] = parse(index, menu, enoent)
s.templ["deleteHandler"] = parse(index, menu, delete)
s.templ["notFoundHandler"] = parse(index, menu, notFound)
s.templ["forbiddenHandler"] = parse(index, menu, forbidden)

12
templates/enoent.html Normal file
View file

@ -0,0 +1,12 @@
{{define "body"}}
<div class="alert alert-danger msg">
<h4>File missing!</h4>
<p>This file is indexed but has been removed from the file system.</p>
<p>Do you want to delete this entry from the index?</p>
</div>
<form class="form-horizontal" action="/delete{{.Path}}" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<input type="hidden" name="clean" value="true">
<button class="btn btn-sm btn-danger" type="submit">Delete</button>
</form>
{{end}}

2
templates/upload.html
View file

@ -10,7 +10,7 @@
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<label class="col-form-label" for="name">Name</label>
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}">
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" placeholder="optional">
</div>
<div class="form-group">
<div class="input-group mb-3">