ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

some bug fixes

Browse source
This commit is contained in:
ston1th 2019-05-05 19:06:51 +02:00
commit 0e914c629e
16 changed files with 179 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

30
pkg/cmd/cmd.go
View file

@ -4,9 +4,11 @@ package cmd
import (
"encoding/json"
"errors"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/docstore/pkg/server"
"git.giftfish.de/ston1th/godrop/v2"
"github.com/urfave/cli"
@ -26,6 +28,8 @@ const (
defRunUser = "docstore"
defRunGroup = "docstore"
defTimeout = 600
)
var (
@ -55,7 +59,7 @@ func initServer(cfg core.Config) (err error) {
return
}
if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock unveil"); err != nil {
if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock proc exec unveil"); err != nil {
return
}
@ -64,6 +68,14 @@ func initServer(cfg core.Config) (err error) {
return
}
log.InitLogger(cfg)
cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir)
scanner, err := scan.New(cfg.DataDir, cfg.Langs, time.Second*time.Duration(cfg.CMDTimeout))
if err != nil {
return errors.New("scanner: " + err.Error())
}
err = godrop.Unveil(cfg.RunDir, "rwc")
if err != nil {
return
@ -73,10 +85,7 @@ func initServer(cfg core.Config) (err error) {
return
}
log.InitLogger(cfg)
cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir)
srv := server.NewHTTPServer(cfg, l)
srv := server.NewHTTPServer(cfg, l, scanner)
err = srv.Start()
if err != nil {
return
@ -124,6 +133,7 @@ func Run(version string) {
Usage: "start docstore server",
Flags: defaultFlags(serverFlags()),
Action: func(c *cli.Context) error {
config.Langs = c.StringSlice("langs")
if err := loadConfig(); err != nil {
stdlog.Fatal("server: ", err)
}
@ -235,6 +245,16 @@ func serverFlags() []cli.Flag {
Usage: "static hmac secret (at least 64 chars, used for JWT signing)",
Destination: &config.Secret,
},
cli.StringSliceFlag{
Name: "langs",
Usage: "languages for tesseract ocr",
},
cli.IntFlag{
Name: "timeout, t",
Value: defTimeout,
Usage: "timeout in seconds for executed teseract and pdftotext commands",
Destination: &config.CMDTimeout,
},
cli.BoolFlag{
Name: "foreground, f",
Usage: "do not fork into the background",

8
pkg/core/helper.go
View file

@ -3,6 +3,7 @@
package core
import (
"sync"
"time"
)
@ -10,6 +11,13 @@ const (
fileFmt = "20060102_150405"
)
var m sync.Mutex
func FileName() string {
m.Lock()
defer func() {
time.Sleep(time.Millisecond * 1100)
m.Unlock()
}()
return time.Now().Format(fileFmt)
}

25
pkg/core/types.go
View file

@ -24,6 +24,7 @@ type Path struct {
Name string
Abs string
Flag bool
Scan bool
}
type Paths []Path
@ -33,15 +34,17 @@ func (p Paths) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Paths) Less(i, j int) bool { return p[i].Name < p[j].Name }
type Config struct {
RunDir string `json:"run_dir,omitempty"`
DataDir string `json:"-"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
ListenAddr string `json:"listen_addr,omitempty"`
LogFile string `json:"log_file,omitempty"`
Version string `json:"-"`
Secret string `json:"secret,omitempty"`
Foreground bool `json:"foreground,omitempty"`
SecureCookie bool `json:"secure_cookie,omitempty"`
Debug bool `json:"debug,omitempty"`
RunDir string `json:"run_dir,omitempty"`
DataDir string `json:"-"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
ListenAddr string `json:"listen_addr,omitempty"`
LogFile string `json:"log_file,omitempty"`
Version string `json:"-"`
Secret string `json:"secret,omitempty"`
Langs []string `json:"langs,omitempty"`
CMDTimeout int `json:"cmd_timeout,omitempty"`
Foreground bool `json:"foreground,omitempty"`
SecureCookie bool `json:"secure_cookie,omitempty"`
Debug bool `json:"debug,omitempty"`
}

35
pkg/fs/file.go
View file

@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
)
type Mode int
@ -78,20 +79,40 @@ func (fs *Filesystem) Mkdir(path, name string) error {
return os.Mkdir(filepath.Join(fs.Base, dirpath), DirMode)
}
func (fs *Filesystem) Move(path, dest, name string) (string, error) {
type Moved struct {
Old string
New string
}
func (fs *Filesystem) Move(path, dest, name string) (moved []Moved, newfile string, err error) {
if name == "" {
name = File(path)
}
fullpath := filepath.Join(fs.Base, Clean(path))
path = Clean(path)
fullpath := filepath.Join(fs.Base, path)
if fullpath == fs.Base {
return "", errors.New("the root path can not be moved")
err = errors.New("the root path can not be moved")
return
}
newfile := filepath.Join(Clean(dest), Clean(name))
err := checkPath(newfile)
newfile = filepath.Join(Clean(dest), Clean(name))
err = checkPath(newfile)
if err != nil {
return "", err
return
}
return newfile, os.Rename(fullpath, filepath.Join(fs.Base, newfile))
fi, err := os.Stat(fullpath)
if err != nil {
return
}
m := fi.Mode()
if m.IsDir() {
for _, f := range fs.RecursiveFiles(path) {
moved = append(moved, Moved{f, strings.Replace(f, path, newfile, 1)})
}
} else if m.IsRegular() {
moved = append(moved, Moved{path, newfile})
}
err = os.Rename(fullpath, filepath.Join(fs.Base, newfile))
return
}
func (fs *Filesystem) Delete(path string) (removed []string, err error) {

10
pkg/fs/fs.go
View file

@ -10,6 +10,7 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
)
const (
@ -25,6 +26,10 @@ type Directory struct {
type Filesystem struct {
Base string
// protects scan
m sync.RWMutex
scan map[string]struct{}
}
func NewFilesystem(base string) (fs *Filesystem, err error) {
@ -48,7 +53,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) {
err = errors.New("base dir '" + base + "' is not a directory")
return
}
fs = &Filesystem{base}
fs = &Filesystem{Base: base, scan: make(map[string]struct{})}
return
}
@ -75,7 +80,8 @@ func (fs *Filesystem) ReadDir(path string) (d *Directory) {
continue
}
if m.IsRegular() {
d.Files = append(d.Files, core.Path{Name: n, Abs: filepath.Join(path, n)})
abs := filepath.Join(path, n)
d.Files = append(d.Files, core.Path{Name: n, Abs: abs, Scan: fs.CheckScan(abs)})
}
}
sort.Sort(d.Dirs)

26
pkg/fs/scan.go Normal file
View file

@ -0,0 +1,26 @@
package fs
import "errors"
func (fs *Filesystem) AddScan(path string) error {
if fs.CheckScan(path) {
return errors.New("file '" + path + "' is already scanning")
}
fs.m.Lock()
defer fs.m.Unlock()
fs.scan[path] = struct{}{}
return nil
}
func (fs *Filesystem) CheckScan(path string) (ok bool) {
fs.m.RLock()
defer fs.m.RUnlock()
_, ok = fs.scan[path]
return
}
func (fs *Filesystem) RemoveScan(path string) {
fs.m.Lock()
defer fs.m.Unlock()
delete(fs.scan, path)
}

7
pkg/scan/args.go Normal file
View file

@ -0,0 +1,7 @@
// Copyright (C) 2019 Marius Schellenberger
// +build !openbsd
package scan
const tesseract_oem = "2"

7
pkg/scan/args_openbsd.go Normal file
View file

@ -0,0 +1,7 @@
// Copyright (C) 2019 Marius Schellenberger
// +build openbsd
package scan
const tesseract_oem = "1"

15
pkg/scan/scanner.go
View file

@ -56,11 +56,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er
if err != nil {
return
}
err = godrop.Unveil(tcmd, "rx")
pcmd, err = exec.LookPath("pdftotext")
if err != nil {
return
}
pcmd, err = exec.LookPath("pdftotext")
err = godrop.Unveil(tcmd, "rx")
if err != nil {
return
}
@ -73,11 +73,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er
}
for i, l := range langs {
lang += l
if i < len(langs) {
if i < len(langs)-1 {
lang += "+"
}
}
args := []string{"-l", lang, "--oem", "2"}
args := []string{"-l", lang, "--oem", tesseract_oem}
s = &Scanner{
base,
timeout,
@ -98,14 +98,14 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
f *os.File
)
p, _ := filepath.Split(file)
filename = filepath.Join(p, core.FileName()+pdfExt)
pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...)
pdfCmd.Env = s.t.env
log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args)
pdf, err = core.Timeout(pdfCmd, s.timeout)
pdf, err = timeout(pdfCmd, s.timeout)
if err != nil {
return
}
filename = filepath.Join(p, core.FileName()+pdfExt)
pdffile = filepath.Join(s.base, filename)
f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, fs.FileMode)
if err != nil {
@ -124,8 +124,9 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
}
txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...)
log.Debugf("%s %v", txtCmd.Path, txtCmd.Args)
txt, err := core.Timeout(txtCmd, s.timeout)
txt, err := timeout(txtCmd, s.timeout)
if err != nil {
log.Debugf("killed", err)
return
}
if len(txt) <= 2 {

14
pkg/core/timeout.go → pkg/scan/timeout.go
View file

@ -1,14 +1,14 @@
// Copyright (C) 2019 Marius Schellenberger
package core
package scan
import (
"errors"
"fmt"
"os/exec"
"time"
)
func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
func timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
done := make(chan error)
out := make(chan []byte)
go func() {
@ -19,17 +19,17 @@ func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
select {
case <-time.After(t):
e := cmd.Process.Kill()
err = <-done
<-out
if e != nil {
<-done
err = e
return
} else {
err = fmt.Errorf("command '%s' timed out: %s", cmd.Path, err)
}
err = <-done
case e := <-done:
exerr, ok := e.(*exec.ExitError)
if ok {
err = errors.New(e.Error() + "\n" + string(exerr.Stderr))
err = fmt.Errorf("%s\n%s", e, string(exerr.Stderr))
} else {
err = e
}

24
pkg/server/handler.go
View file

@ -179,11 +179,22 @@ func indexHandler(ctx *Context) {
case "GET":
earlyErr := make(chan error)
go func() {
err := ctx.Srv.FS.AddScan(path)
if err != nil {
select {
case earlyErr <- err:
default:
}
return
}
defer ctx.Srv.FS.RemoveScan(path)
file, txt, err := ctx.Srv.Scanner.Scan(path)
if err != nil {
select {
case earlyErr <- err:
default:
}
log.Printf("scanner: %s: %s", path, err)
return
}
id, err := ctx.Srv.DB.Index.Add(txt)
@ -306,15 +317,17 @@ func moveHandler(ctx *Context) {
}
name := ctx.Form("name")
dest := ctx.Form("destination")
newfile, err := ctx.Srv.FS.Move(path, dest, name)
moved, newfile, err := ctx.Srv.FS.Move(path, dest, name)
if err != nil {
ctx.Error(err)
return
}
err = ctx.Srv.DB.MoveFile(path, newfile)
if err != nil {
ctx.Error(err)
return
for _, m := range moved {
log.Debugf("move: %s to %s", m.Old, m.New)
err = ctx.Srv.DB.MoveFile(m.Old, m.New)
if err != nil {
log.Printf("move: %s: %s", m, err)
}
}
ctx.Redirect(newfile, http.StatusFound)
}
@ -411,6 +424,7 @@ func dirHandler(ctx *Context) {
Name: path,
Abs: core.RawPrefix + path,
Flag: ctx.Srv.DB.IsIndexed(path),
Scan: ctx.Srv.FS.CheckScan(path),
}
ctx.Data.Paths = fs.Paths(path)
ctx.Exec()

7
pkg/server/server.go
View file

@ -40,10 +40,11 @@ type HTTPServer struct {
res map[string][]byte
}
func NewHTTPServer(cfg core.Config, l net.Listener) (srv *HTTPServer) {
func NewHTTPServer(cfg core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPServer) {
srv = &HTTPServer{
Config: cfg,
listener: l,
Scanner: s,
}
srv.handler = srv.buildRoutes()
srv.loadTemplates()
@ -56,10 +57,6 @@ func (s *HTTPServer) Start() (err error) {
if err != nil {
return errors.New("fs: " + err.Error())
}
s.Scanner, err = scan.New(s.Config.DataDir, nil, time.Second*30)
if err != nil {
return errors.New("tesseract: " + err.Error())
}
s.DB, err = db.New(s.Config)
if err != nil {
return errors.New("db: " + err.Error())

10
pkg/server/templates.go
View file

@ -67,7 +67,7 @@ const (
{{if .Data}}
{{range $item := .Data.Dirs}}
<tr>
<th><a href="{{$item.Abs}}">{{$item.Name}}/</a></th>
<th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
<td>Directory</td>
<td><div class="btn-group">
<a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
@ -80,9 +80,13 @@ const (
<th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
<td>File</td>
<td><div class="btn-group">
{{if $item.Scan}}
<a href="#" class="btn btn-sm btn-warning">Scanning..</a>
{{else}}
{{if not $item.Flag}}
<a href="/index{{$item.Abs}}" class="btn btn-sm btn-success">Index</a>
{{end}}
{{end}}
<a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
</div></td>
@ -104,9 +108,13 @@ const (
{{end}}
</ol>
<div class="btn-group btn-breadcrumb">
{{if .Data.Scan}}
<a href="#" class="btn btn-sm btn-warning">Scanning..</a>
{{else}}
{{if not .Data.Flag}}
<a href="/index{{.Data.Name}}" class="btn btn-sm btn-success">Index</a>
{{end}}
{{end}}
<a href="/move{{.Data.Name}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{.Data.Name}}" class="btn btn-sm btn-danger">Delete</a>
</div>