some bug fixes

pkg/cmd/cmd.go

@@ -4,9 +4,11 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"git.giftfish.de/ston1th/docstore/pkg/core"
 	"git.giftfish.de/ston1th/docstore/pkg/db"
 	"git.giftfish.de/ston1th/docstore/pkg/log"
+	"git.giftfish.de/ston1th/docstore/pkg/scan"
 	"git.giftfish.de/ston1th/docstore/pkg/server"
 	"git.giftfish.de/ston1th/godrop/v2"
 	"github.com/urfave/cli"
@@ -26,6 +28,8 @@ const (
 
 	defRunUser  = "docstore"
 	defRunGroup = "docstore"
+
+	defTimeout = 600
 )
 
 var (
@@ -55,7 +59,7 @@ func initServer(cfg core.Config) (err error) {
 		return
 	}
 
-	if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock unveil"); err != nil {
+	if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock proc exec unveil"); err != nil {
 		return
 	}
 
@@ -64,6 +68,14 @@ func initServer(cfg core.Config) (err error) {
 		return
 	}
 
+	log.InitLogger(cfg)
+
+	cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir)
+
+	scanner, err := scan.New(cfg.DataDir, cfg.Langs, time.Second*time.Duration(cfg.CMDTimeout))
+	if err != nil {
+		return errors.New("scanner: " + err.Error())
+	}
 	err = godrop.Unveil(cfg.RunDir, "rwc")
 	if err != nil {
 		return
@@ -73,10 +85,7 @@ func initServer(cfg core.Config) (err error) {
 		return
 	}
 
-	log.InitLogger(cfg)
+	srv := server.NewHTTPServer(cfg, l, scanner)
-
-	cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir)
-	srv := server.NewHTTPServer(cfg, l)
 	err = srv.Start()
 	if err != nil {
 		return
@@ -124,6 +133,7 @@ func Run(version string) {
 			Usage: "start docstore server",
 			Flags: defaultFlags(serverFlags()),
 			Action: func(c *cli.Context) error {
+				config.Langs = c.StringSlice("langs")
 				if err := loadConfig(); err != nil {
 					stdlog.Fatal("server: ", err)
 				}
@@ -235,6 +245,16 @@ func serverFlags() []cli.Flag {
 			Usage:       "static hmac secret (at least 64 chars, used for JWT signing)",
 			Destination: &config.Secret,
 		},
+		cli.StringSliceFlag{
+			Name:  "langs",
+			Usage: "languages for tesseract ocr",
+		},
+		cli.IntFlag{
+			Name:        "timeout, t",
+			Value:       defTimeout,
+			Usage:       "timeout in seconds for executed teseract and pdftotext commands",
+			Destination: &config.CMDTimeout,
+		},
 		cli.BoolFlag{
 			Name:        "foreground, f",
 			Usage:       "do not fork into the background",

pkg/core/helper.go

@@ -3,6 +3,7 @@
 package core
 
 import (
+	"sync"
 	"time"
 )
 
@@ -10,6 +11,13 @@ const (
 	fileFmt = "20060102_150405"
 )
 
+var m sync.Mutex
+
 func FileName() string {
+	m.Lock()
+	defer func() {
+		time.Sleep(time.Millisecond * 1100)
+		m.Unlock()
+	}()
 	return time.Now().Format(fileFmt)
 }

pkg/core/types.go

@@ -24,6 +24,7 @@ type Path struct {
 	Name string
 	Abs  string
 	Flag bool
+	Scan bool
 }
 
 type Paths []Path
@@ -33,15 +34,17 @@ func (p Paths) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 func (p Paths) Less(i, j int) bool { return p[i].Name < p[j].Name }
 
 type Config struct {
-	RunDir       string `json:"run_dir,omitempty"`
+	RunDir       string   `json:"run_dir,omitempty"`
-	DataDir      string `json:"-"`
+	DataDir      string   `json:"-"`
-	User         string `json:"user,omitempty"`
+	User         string   `json:"user,omitempty"`
-	Group        string `json:"group,omitempty"`
+	Group        string   `json:"group,omitempty"`
-	ListenAddr   string `json:"listen_addr,omitempty"`
+	ListenAddr   string   `json:"listen_addr,omitempty"`
-	LogFile      string `json:"log_file,omitempty"`
+	LogFile      string   `json:"log_file,omitempty"`
-	Version      string `json:"-"`
+	Version      string   `json:"-"`
-	Secret       string `json:"secret,omitempty"`
+	Secret       string   `json:"secret,omitempty"`
-	Foreground   bool   `json:"foreground,omitempty"`
+	Langs        []string `json:"langs,omitempty"`
-	SecureCookie bool   `json:"secure_cookie,omitempty"`
+	CMDTimeout   int      `json:"cmd_timeout,omitempty"`
-	Debug        bool   `json:"debug,omitempty"`
+	Foreground   bool     `json:"foreground,omitempty"`
+	SecureCookie bool     `json:"secure_cookie,omitempty"`
+	Debug        bool     `json:"debug,omitempty"`
 }

pkg/fs/file.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 type Mode int
@@ -78,20 +79,40 @@ func (fs *Filesystem) Mkdir(path, name string) error {
 	return os.Mkdir(filepath.Join(fs.Base, dirpath), DirMode)
 }
 
-func (fs *Filesystem) Move(path, dest, name string) (string, error) {
+type Moved struct {
+	Old string
+	New string
+}
+
+func (fs *Filesystem) Move(path, dest, name string) (moved []Moved, newfile string, err error) {
 	if name == "" {
 		name = File(path)
 	}
-	fullpath := filepath.Join(fs.Base, Clean(path))
+	path = Clean(path)
+	fullpath := filepath.Join(fs.Base, path)
 	if fullpath == fs.Base {
-		return "", errors.New("the root path can not be moved")
+		err = errors.New("the root path can not be moved")
+		return
 	}
-	newfile := filepath.Join(Clean(dest), Clean(name))
+	newfile = filepath.Join(Clean(dest), Clean(name))
-	err := checkPath(newfile)
+	err = checkPath(newfile)
 	if err != nil {
-		return "", err
+		return
 	}
-	return newfile, os.Rename(fullpath, filepath.Join(fs.Base, newfile))
+	fi, err := os.Stat(fullpath)
+	if err != nil {
+		return
+	}
+	m := fi.Mode()
+	if m.IsDir() {
+		for _, f := range fs.RecursiveFiles(path) {
+			moved = append(moved, Moved{f, strings.Replace(f, path, newfile, 1)})
+		}
+	} else if m.IsRegular() {
+		moved = append(moved, Moved{path, newfile})
+	}
+	err = os.Rename(fullpath, filepath.Join(fs.Base, newfile))
+	return
 }
 
 func (fs *Filesystem) Delete(path string) (removed []string, err error) {

pkg/fs/fs.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 )
 
 const (
@@ -25,6 +26,10 @@ type Directory struct {
 
 type Filesystem struct {
 	Base string
+
+	// protects scan
+	m    sync.RWMutex
+	scan map[string]struct{}
 }
 
 func NewFilesystem(base string) (fs *Filesystem, err error) {
@@ -48,7 +53,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) {
 		err = errors.New("base dir '" + base + "' is not a directory")
 		return
 	}
-	fs = &Filesystem{base}
+	fs = &Filesystem{Base: base, scan: make(map[string]struct{})}
 	return
 }
 
@@ -75,7 +80,8 @@ func (fs *Filesystem) ReadDir(path string) (d *Directory) {
 			continue
 		}
 		if m.IsRegular() {
-			d.Files = append(d.Files, core.Path{Name: n, Abs: filepath.Join(path, n)})
+			abs := filepath.Join(path, n)
+			d.Files = append(d.Files, core.Path{Name: n, Abs: abs, Scan: fs.CheckScan(abs)})
 		}
 	}
 	sort.Sort(d.Dirs)

pkg/fs/scan.go

@@ -0,0 +1,26 @@
+package fs
+
+import "errors"
+
+func (fs *Filesystem) AddScan(path string) error {
+	if fs.CheckScan(path) {
+		return errors.New("file '" + path + "' is already scanning")
+	}
+	fs.m.Lock()
+	defer fs.m.Unlock()
+	fs.scan[path] = struct{}{}
+	return nil
+}
+
+func (fs *Filesystem) CheckScan(path string) (ok bool) {
+	fs.m.RLock()
+	defer fs.m.RUnlock()
+	_, ok = fs.scan[path]
+	return
+}
+
+func (fs *Filesystem) RemoveScan(path string) {
+	fs.m.Lock()
+	defer fs.m.Unlock()
+	delete(fs.scan, path)
+}

pkg/scan/args.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+// +build !openbsd
+
+package scan
+
+const tesseract_oem = "2"

pkg/scan/args_openbsd.go

@@ -0,0 +1,7 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+// +build openbsd
+
+package scan
+
+const tesseract_oem = "1"

pkg/scan/scanner.go

@@ -56,11 +56,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er
 	if err != nil {
 		return
 	}
-	err = godrop.Unveil(tcmd, "rx")
+	pcmd, err = exec.LookPath("pdftotext")
 	if err != nil {
 		return
 	}
-	pcmd, err = exec.LookPath("pdftotext")
+	err = godrop.Unveil(tcmd, "rx")
 	if err != nil {
 		return
 	}
@@ -73,11 +73,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er
 	}
 	for i, l := range langs {
 		lang += l
-		if i < len(langs) {
+		if i < len(langs)-1 {
 			lang += "+"
 		}
 	}
-	args := []string{"-l", lang, "--oem", "2"}
+	args := []string{"-l", lang, "--oem", tesseract_oem}
 	s = &Scanner{
 		base,
 		timeout,
@@ -98,14 +98,14 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
 			f   *os.File
 		)
 		p, _ := filepath.Split(file)
-		filename = filepath.Join(p, core.FileName()+pdfExt)
 		pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...)
 		pdfCmd.Env = s.t.env
 		log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args)
-		pdf, err = core.Timeout(pdfCmd, s.timeout)
+		pdf, err = timeout(pdfCmd, s.timeout)
 		if err != nil {
 			return
 		}
+		filename = filepath.Join(p, core.FileName()+pdfExt)
 		pdffile = filepath.Join(s.base, filename)
 		f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, fs.FileMode)
 		if err != nil {
@@ -124,8 +124,9 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
 	}
 	txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...)
 	log.Debugf("%s %v", txtCmd.Path, txtCmd.Args)
-	txt, err := core.Timeout(txtCmd, s.timeout)
+	txt, err := timeout(txtCmd, s.timeout)
 	if err != nil {
+		log.Debugf("killed", err)
 		return
 	}
 	if len(txt) <= 2 {

pkg/scan/timeout.go

@@ -1,14 +1,14 @@
 // Copyright (C) 2019 Marius Schellenberger
 
-package core
+package scan
 
 import (
-	"errors"
+	"fmt"
 	"os/exec"
 	"time"
 )
 
-func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
+func timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
 	done := make(chan error)
 	out := make(chan []byte)
 	go func() {
@@ -19,17 +19,17 @@ func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) {
 	select {
 	case <-time.After(t):
 		e := cmd.Process.Kill()
+		err = <-done
 		<-out
 		if e != nil {
-			<-done
 			err = e
-			return
+		} else {
+			err = fmt.Errorf("command '%s' timed out: %s", cmd.Path, err)
 		}
-		err = <-done
 	case e := <-done:
 		exerr, ok := e.(*exec.ExitError)
 		if ok {
-			err = errors.New(e.Error() + "\n" + string(exerr.Stderr))
+			err = fmt.Errorf("%s\n%s", e, string(exerr.Stderr))
 		} else {
 			err = e
 		}

pkg/server/handler.go

@@ -179,11 +179,22 @@ func indexHandler(ctx *Context) {
 	case "GET":
 		earlyErr := make(chan error)
 		go func() {
+			err := ctx.Srv.FS.AddScan(path)
+			if err != nil {
+				select {
+				case earlyErr <- err:
+				default:
+				}
+				return
+			}
+			defer ctx.Srv.FS.RemoveScan(path)
 			file, txt, err := ctx.Srv.Scanner.Scan(path)
 			if err != nil {
 				select {
 				case earlyErr <- err:
+				default:
 				}
+				log.Printf("scanner: %s: %s", path, err)
 				return
 			}
 			id, err := ctx.Srv.DB.Index.Add(txt)
@@ -306,15 +317,17 @@ func moveHandler(ctx *Context) {
 		}
 		name := ctx.Form("name")
 		dest := ctx.Form("destination")
-		newfile, err := ctx.Srv.FS.Move(path, dest, name)
+		moved, newfile, err := ctx.Srv.FS.Move(path, dest, name)
 		if err != nil {
 			ctx.Error(err)
 			return
 		}
-		err = ctx.Srv.DB.MoveFile(path, newfile)
+		for _, m := range moved {
-		if err != nil {
+			log.Debugf("move: %s to %s", m.Old, m.New)
-			ctx.Error(err)
+			err = ctx.Srv.DB.MoveFile(m.Old, m.New)
-			return
+			if err != nil {
+				log.Printf("move: %s: %s", m, err)
+			}
 		}
 		ctx.Redirect(newfile, http.StatusFound)
 	}
@@ -411,6 +424,7 @@ func dirHandler(ctx *Context) {
 				Name: path,
 				Abs:  core.RawPrefix + path,
 				Flag: ctx.Srv.DB.IsIndexed(path),
+				Scan: ctx.Srv.FS.CheckScan(path),
 			}
 			ctx.Data.Paths = fs.Paths(path)
 			ctx.Exec()

pkg/server/server.go

@@ -40,10 +40,11 @@ type HTTPServer struct {
 	res   map[string][]byte
 }
 
-func NewHTTPServer(cfg core.Config, l net.Listener) (srv *HTTPServer) {
+func NewHTTPServer(cfg core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPServer) {
 	srv = &HTTPServer{
 		Config:   cfg,
 		listener: l,
+		Scanner:  s,
 	}
 	srv.handler = srv.buildRoutes()
 	srv.loadTemplates()
@@ -56,10 +57,6 @@ func (s *HTTPServer) Start() (err error) {
 	if err != nil {
 		return errors.New("fs: " + err.Error())
 	}
-	s.Scanner, err = scan.New(s.Config.DataDir, nil, time.Second*30)
-	if err != nil {
-		return errors.New("tesseract: " + err.Error())
-	}
 	s.DB, err = db.New(s.Config)
 	if err != nil {
 		return errors.New("db: " + err.Error())

pkg/server/templates.go

@@ -67,7 +67,7 @@ const (
         {{if .Data}}
         {{range $item := .Data.Dirs}}
         <tr>
-            <th><a href="{{$item.Abs}}">{{$item.Name}}/</a></th>
+            <th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
             <td>Directory</td>
             <td><div class="btn-group">
                 <a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
@@ -80,9 +80,13 @@ const (
             <th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
             <td>File</td>
             <td><div class="btn-group">
+                {{if $item.Scan}}
+                <a href="#" class="btn btn-sm btn-warning">Scanning..</a>
+                {{else}}
                 {{if not $item.Flag}}
                 <a href="/index{{$item.Abs}}" class="btn btn-sm btn-success">Index</a>
                 {{end}}
+                {{end}}
                 <a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
                 <a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
             </div></td>
@@ -104,9 +108,13 @@ const (
         {{end}}
     </ol>
     <div class="btn-group btn-breadcrumb">
+        {{if .Data.Scan}}
+        <a href="#" class="btn btn-sm btn-warning">Scanning..</a>
+        {{else}}
         {{if not .Data.Flag}}
         <a href="/index{{.Data.Name}}" class="btn btn-sm btn-success">Index</a>
         {{end}}
+        {{end}}
         <a href="/move{{.Data.Name}}" class="btn btn-sm btn-primary">Move</a>
         <a href="/delete{{.Data.Name}}" class="btn btn-sm btn-danger">Delete</a>
     </div>

scripts/install_openbsd.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-pkg_add poppler tesseract tesseract-deu tesseract-eng
+# requires the xbase and xfont sets
+pkg_add poppler-utils tesseract tesseract-deu tesseract-eng
 
 # rc script
 cat <<'EOF'> /etc/rc.d/docstore

templates/dir.html

@@ -29,7 +29,7 @@
         {{if .Data}}
         {{range $item := .Data.Dirs}}
         <tr>
-            <th><a href="{{$item.Abs}}">{{$item.Name}}/</a></th>
+            <th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
             <td>Directory</td>
             <td><div class="btn-group">
                 <a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
@@ -42,9 +42,13 @@
             <th><a href="{{$item.Abs}}">{{$item.Name}}</a></th>
             <td>File</td>
             <td><div class="btn-group">
+                {{if $item.Scan}}
+                <a href="#" class="btn btn-sm btn-warning">Scanning..</a>
+                {{else}}
                 {{if not $item.Flag}}
                 <a href="/index{{$item.Abs}}" class="btn btn-sm btn-success">Index</a>
                 {{end}}
+                {{end}}
                 <a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
                 <a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
             </div></td>

templates/file.html

@@ -9,9 +9,13 @@
         {{end}}
     </ol>
     <div class="btn-group btn-breadcrumb">
+        {{if .Data.Scan}}
+        <a href="#" class="btn btn-sm btn-warning">Scanning..</a>
+        {{else}}
         {{if not .Data.Flag}}
         <a href="/index{{.Data.Name}}" class="btn btn-sm btn-success">Index</a>
         {{end}}
+        {{end}}
         <a href="/move{{.Data.Name}}" class="btn btn-sm btn-primary">Move</a>
         <a href="/delete{{.Data.Name}}" class="btn btn-sm btn-danger">Delete</a>
     </div>