diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 5d97898..c0c4cbb 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -4,9 +4,11 @@ package cmd import ( "encoding/json" + "errors" "git.giftfish.de/ston1th/docstore/pkg/core" "git.giftfish.de/ston1th/docstore/pkg/db" "git.giftfish.de/ston1th/docstore/pkg/log" + "git.giftfish.de/ston1th/docstore/pkg/scan" "git.giftfish.de/ston1th/docstore/pkg/server" "git.giftfish.de/ston1th/godrop/v2" "github.com/urfave/cli" @@ -26,6 +28,8 @@ const ( defRunUser = "docstore" defRunGroup = "docstore" + + defTimeout = 600 ) var ( @@ -55,7 +59,7 @@ func initServer(cfg core.Config) (err error) { return } - if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock unveil"); err != nil { + if err = godrop.PledgePromises("stdio rpath wpath cpath inet fattr flock proc exec unveil"); err != nil { return } @@ -64,6 +68,14 @@ func initServer(cfg core.Config) (err error) { return } + log.InitLogger(cfg) + + cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir) + + scanner, err := scan.New(cfg.DataDir, cfg.Langs, time.Second*time.Duration(cfg.CMDTimeout)) + if err != nil { + return errors.New("scanner: " + err.Error()) + } err = godrop.Unveil(cfg.RunDir, "rwc") if err != nil { return @@ -73,10 +85,7 @@ func initServer(cfg core.Config) (err error) { return } - log.InitLogger(cfg) - - cfg.DataDir = filepath.Join(cfg.RunDir, core.DataDir) - srv := server.NewHTTPServer(cfg, l) + srv := server.NewHTTPServer(cfg, l, scanner) err = srv.Start() if err != nil { return @@ -124,6 +133,7 @@ func Run(version string) { Usage: "start docstore server", Flags: defaultFlags(serverFlags()), Action: func(c *cli.Context) error { + config.Langs = c.StringSlice("langs") if err := loadConfig(); err != nil { stdlog.Fatal("server: ", err) } @@ -235,6 +245,16 @@ func serverFlags() []cli.Flag { Usage: "static hmac secret (at least 64 chars, used for JWT signing)", Destination: &config.Secret, }, + cli.StringSliceFlag{ + Name: "langs", + Usage: "languages for tesseract ocr", + }, + cli.IntFlag{ + Name: "timeout, t", + Value: defTimeout, + Usage: "timeout in seconds for executed teseract and pdftotext commands", + Destination: &config.CMDTimeout, + }, cli.BoolFlag{ Name: "foreground, f", Usage: "do not fork into the background", diff --git a/pkg/core/helper.go b/pkg/core/helper.go index 7da268a..fb403d2 100644 --- a/pkg/core/helper.go +++ b/pkg/core/helper.go @@ -3,6 +3,7 @@ package core import ( + "sync" "time" ) @@ -10,6 +11,13 @@ const ( fileFmt = "20060102_150405" ) +var m sync.Mutex + func FileName() string { + m.Lock() + defer func() { + time.Sleep(time.Millisecond * 1100) + m.Unlock() + }() return time.Now().Format(fileFmt) } diff --git a/pkg/core/types.go b/pkg/core/types.go index 20b1802..adafcbc 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -24,6 +24,7 @@ type Path struct { Name string Abs string Flag bool + Scan bool } type Paths []Path @@ -33,15 +34,17 @@ func (p Paths) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Paths) Less(i, j int) bool { return p[i].Name < p[j].Name } type Config struct { - RunDir string `json:"run_dir,omitempty"` - DataDir string `json:"-"` - User string `json:"user,omitempty"` - Group string `json:"group,omitempty"` - ListenAddr string `json:"listen_addr,omitempty"` - LogFile string `json:"log_file,omitempty"` - Version string `json:"-"` - Secret string `json:"secret,omitempty"` - Foreground bool `json:"foreground,omitempty"` - SecureCookie bool `json:"secure_cookie,omitempty"` - Debug bool `json:"debug,omitempty"` + RunDir string `json:"run_dir,omitempty"` + DataDir string `json:"-"` + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + ListenAddr string `json:"listen_addr,omitempty"` + LogFile string `json:"log_file,omitempty"` + Version string `json:"-"` + Secret string `json:"secret,omitempty"` + Langs []string `json:"langs,omitempty"` + CMDTimeout int `json:"cmd_timeout,omitempty"` + Foreground bool `json:"foreground,omitempty"` + SecureCookie bool `json:"secure_cookie,omitempty"` + Debug bool `json:"debug,omitempty"` } diff --git a/pkg/fs/file.go b/pkg/fs/file.go index 252f1d2..8c9c9fd 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" ) type Mode int @@ -78,20 +79,40 @@ func (fs *Filesystem) Mkdir(path, name string) error { return os.Mkdir(filepath.Join(fs.Base, dirpath), DirMode) } -func (fs *Filesystem) Move(path, dest, name string) (string, error) { +type Moved struct { + Old string + New string +} + +func (fs *Filesystem) Move(path, dest, name string) (moved []Moved, newfile string, err error) { if name == "" { name = File(path) } - fullpath := filepath.Join(fs.Base, Clean(path)) + path = Clean(path) + fullpath := filepath.Join(fs.Base, path) if fullpath == fs.Base { - return "", errors.New("the root path can not be moved") + err = errors.New("the root path can not be moved") + return } - newfile := filepath.Join(Clean(dest), Clean(name)) - err := checkPath(newfile) + newfile = filepath.Join(Clean(dest), Clean(name)) + err = checkPath(newfile) if err != nil { - return "", err + return } - return newfile, os.Rename(fullpath, filepath.Join(fs.Base, newfile)) + fi, err := os.Stat(fullpath) + if err != nil { + return + } + m := fi.Mode() + if m.IsDir() { + for _, f := range fs.RecursiveFiles(path) { + moved = append(moved, Moved{f, strings.Replace(f, path, newfile, 1)}) + } + } else if m.IsRegular() { + moved = append(moved, Moved{path, newfile}) + } + err = os.Rename(fullpath, filepath.Join(fs.Base, newfile)) + return } func (fs *Filesystem) Delete(path string) (removed []string, err error) { diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 6ba6085..4413130 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sort" "strings" + "sync" ) const ( @@ -25,6 +26,10 @@ type Directory struct { type Filesystem struct { Base string + + // protects scan + m sync.RWMutex + scan map[string]struct{} } func NewFilesystem(base string) (fs *Filesystem, err error) { @@ -48,7 +53,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) { err = errors.New("base dir '" + base + "' is not a directory") return } - fs = &Filesystem{base} + fs = &Filesystem{Base: base, scan: make(map[string]struct{})} return } @@ -75,7 +80,8 @@ func (fs *Filesystem) ReadDir(path string) (d *Directory) { continue } if m.IsRegular() { - d.Files = append(d.Files, core.Path{Name: n, Abs: filepath.Join(path, n)}) + abs := filepath.Join(path, n) + d.Files = append(d.Files, core.Path{Name: n, Abs: abs, Scan: fs.CheckScan(abs)}) } } sort.Sort(d.Dirs) diff --git a/pkg/fs/scan.go b/pkg/fs/scan.go new file mode 100644 index 0000000..b16163e --- /dev/null +++ b/pkg/fs/scan.go @@ -0,0 +1,26 @@ +package fs + +import "errors" + +func (fs *Filesystem) AddScan(path string) error { + if fs.CheckScan(path) { + return errors.New("file '" + path + "' is already scanning") + } + fs.m.Lock() + defer fs.m.Unlock() + fs.scan[path] = struct{}{} + return nil +} + +func (fs *Filesystem) CheckScan(path string) (ok bool) { + fs.m.RLock() + defer fs.m.RUnlock() + _, ok = fs.scan[path] + return +} + +func (fs *Filesystem) RemoveScan(path string) { + fs.m.Lock() + defer fs.m.Unlock() + delete(fs.scan, path) +} diff --git a/pkg/scan/args.go b/pkg/scan/args.go new file mode 100644 index 0000000..ad1ab73 --- /dev/null +++ b/pkg/scan/args.go @@ -0,0 +1,7 @@ +// Copyright (C) 2019 Marius Schellenberger + +// +build !openbsd + +package scan + +const tesseract_oem = "2" diff --git a/pkg/scan/args_openbsd.go b/pkg/scan/args_openbsd.go new file mode 100644 index 0000000..4f866f4 --- /dev/null +++ b/pkg/scan/args_openbsd.go @@ -0,0 +1,7 @@ +// Copyright (C) 2019 Marius Schellenberger + +// +build openbsd + +package scan + +const tesseract_oem = "1" diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index 2d9ec19..f9f96f9 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -56,11 +56,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er if err != nil { return } - err = godrop.Unveil(tcmd, "rx") + pcmd, err = exec.LookPath("pdftotext") if err != nil { return } - pcmd, err = exec.LookPath("pdftotext") + err = godrop.Unveil(tcmd, "rx") if err != nil { return } @@ -73,11 +73,11 @@ func New(base string, langs []string, timeout time.Duration) (s *Scanner, err er } for i, l := range langs { lang += l - if i < len(langs) { + if i < len(langs)-1 { lang += "+" } } - args := []string{"-l", lang, "--oem", "2"} + args := []string{"-l", lang, "--oem", tesseract_oem} s = &Scanner{ base, timeout, @@ -98,14 +98,14 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) { f *os.File ) p, _ := filepath.Split(file) - filename = filepath.Join(p, core.FileName()+pdfExt) pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...) pdfCmd.Env = s.t.env log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args) - pdf, err = core.Timeout(pdfCmd, s.timeout) + pdf, err = timeout(pdfCmd, s.timeout) if err != nil { return } + filename = filepath.Join(p, core.FileName()+pdfExt) pdffile = filepath.Join(s.base, filename) f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, fs.FileMode) if err != nil { @@ -124,8 +124,9 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) { } txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...) log.Debugf("%s %v", txtCmd.Path, txtCmd.Args) - txt, err := core.Timeout(txtCmd, s.timeout) + txt, err := timeout(txtCmd, s.timeout) if err != nil { + log.Debugf("killed", err) return } if len(txt) <= 2 { diff --git a/pkg/core/timeout.go b/pkg/scan/timeout.go similarity index 66% rename from pkg/core/timeout.go rename to pkg/scan/timeout.go index 3e758f9..8a84bc0 100644 --- a/pkg/core/timeout.go +++ b/pkg/scan/timeout.go @@ -1,14 +1,14 @@ // Copyright (C) 2019 Marius Schellenberger -package core +package scan import ( - "errors" + "fmt" "os/exec" "time" ) -func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) { +func timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) { done := make(chan error) out := make(chan []byte) go func() { @@ -19,17 +19,17 @@ func Timeout(cmd *exec.Cmd, t time.Duration) (output []byte, err error) { select { case <-time.After(t): e := cmd.Process.Kill() + err = <-done <-out if e != nil { - <-done err = e - return + } else { + err = fmt.Errorf("command '%s' timed out: %s", cmd.Path, err) } - err = <-done case e := <-done: exerr, ok := e.(*exec.ExitError) if ok { - err = errors.New(e.Error() + "\n" + string(exerr.Stderr)) + err = fmt.Errorf("%s\n%s", e, string(exerr.Stderr)) } else { err = e } diff --git a/pkg/server/handler.go b/pkg/server/handler.go index 55c116e..06fd7cb 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -179,11 +179,22 @@ func indexHandler(ctx *Context) { case "GET": earlyErr := make(chan error) go func() { + err := ctx.Srv.FS.AddScan(path) + if err != nil { + select { + case earlyErr <- err: + default: + } + return + } + defer ctx.Srv.FS.RemoveScan(path) file, txt, err := ctx.Srv.Scanner.Scan(path) if err != nil { select { case earlyErr <- err: + default: } + log.Printf("scanner: %s: %s", path, err) return } id, err := ctx.Srv.DB.Index.Add(txt) @@ -306,15 +317,17 @@ func moveHandler(ctx *Context) { } name := ctx.Form("name") dest := ctx.Form("destination") - newfile, err := ctx.Srv.FS.Move(path, dest, name) + moved, newfile, err := ctx.Srv.FS.Move(path, dest, name) if err != nil { ctx.Error(err) return } - err = ctx.Srv.DB.MoveFile(path, newfile) - if err != nil { - ctx.Error(err) - return + for _, m := range moved { + log.Debugf("move: %s to %s", m.Old, m.New) + err = ctx.Srv.DB.MoveFile(m.Old, m.New) + if err != nil { + log.Printf("move: %s: %s", m, err) + } } ctx.Redirect(newfile, http.StatusFound) } @@ -411,6 +424,7 @@ func dirHandler(ctx *Context) { Name: path, Abs: core.RawPrefix + path, Flag: ctx.Srv.DB.IsIndexed(path), + Scan: ctx.Srv.FS.CheckScan(path), } ctx.Data.Paths = fs.Paths(path) ctx.Exec() diff --git a/pkg/server/server.go b/pkg/server/server.go index 7da6c44..6480ccc 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -40,10 +40,11 @@ type HTTPServer struct { res map[string][]byte } -func NewHTTPServer(cfg core.Config, l net.Listener) (srv *HTTPServer) { +func NewHTTPServer(cfg core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPServer) { srv = &HTTPServer{ Config: cfg, listener: l, + Scanner: s, } srv.handler = srv.buildRoutes() srv.loadTemplates() @@ -56,10 +57,6 @@ func (s *HTTPServer) Start() (err error) { if err != nil { return errors.New("fs: " + err.Error()) } - s.Scanner, err = scan.New(s.Config.DataDir, nil, time.Second*30) - if err != nil { - return errors.New("tesseract: " + err.Error()) - } s.DB, err = db.New(s.Config) if err != nil { return errors.New("db: " + err.Error()) diff --git a/pkg/server/templates.go b/pkg/server/templates.go index 697c3d7..c576dd9 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -67,7 +67,7 @@ const ( {{if .Data}} {{range $item := .Data.Dirs}} - {{$item.Name}}/ + {{$item.Name}} Directory
Move @@ -80,9 +80,13 @@ const ( {{$item.Name}} File
+ {{if $item.Scan}} + Scanning.. + {{else}} {{if not $item.Flag}} Index {{end}} + {{end}} Move Delete
@@ -104,9 +108,13 @@ const ( {{end}}
+ {{if .Data.Scan}} + Scanning.. + {{else}} {{if not .Data.Flag}} Index {{end}} + {{end}} Move Delete
diff --git a/scripts/install_openbsd.sh b/scripts/install_openbsd.sh index a49a706..2863929 100755 --- a/scripts/install_openbsd.sh +++ b/scripts/install_openbsd.sh @@ -1,6 +1,7 @@ #!/bin/sh -pkg_add poppler tesseract tesseract-deu tesseract-eng +# requires the xbase and xfont sets +pkg_add poppler-utils tesseract tesseract-deu tesseract-eng # rc script cat <<'EOF'> /etc/rc.d/docstore diff --git a/templates/dir.html b/templates/dir.html index 9af6773..26466ef 100644 --- a/templates/dir.html +++ b/templates/dir.html @@ -29,7 +29,7 @@ {{if .Data}} {{range $item := .Data.Dirs}} - {{$item.Name}}/ + {{$item.Name}} Directory
Move @@ -42,9 +42,13 @@ {{$item.Name}} File
+ {{if $item.Scan}} + Scanning.. + {{else}} {{if not $item.Flag}} Index {{end}} + {{end}} Move Delete
diff --git a/templates/file.html b/templates/file.html index b5ed5c9..03f2c6d 100644 --- a/templates/file.html +++ b/templates/file.html @@ -9,9 +9,13 @@ {{end}}
+ {{if .Data.Scan}} + Scanning.. + {{else}} {{if not .Data.Flag}} Index {{end}} + {{end}} Move Delete