more work done

pkg/scan/helper.go

@@ -0,0 +1,10 @@
+package scan
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+func isPDF(path string) bool {
+	return strings.ToLower(filepath.Ext(path)) == pdfExt
+}

pkg/scan/pdf.go

@@ -0,0 +1,6 @@
+package scan
+
+type PDF struct {
+	cmd  string
+	args []string
+}

pkg/scan/scanner.go

@@ -0,0 +1,124 @@
+package scan
+
+import (
+	"errors"
+	"fmt"
+	"git.giftfish.de/ston1th/docstore/pkg/core"
+	"git.giftfish.de/ston1th/docstore/pkg/log"
+	"git.giftfish.de/ston1th/godrop/v2"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"time"
+)
+
+const pdfExt = ".pdf"
+
+var (
+	wsr = regexp.MustCompile(`[\t\f ]+`)
+	nlr = regexp.MustCompile(`[\n]{3,}`)
+	ws  = []byte(" ")
+	nl  = []byte("\n")
+)
+
+type Scanner struct {
+	base    string
+	timeout time.Duration
+	t       Tesseract
+	p       PDF
+}
+
+func New(base string, langs []string, timeout time.Duration) (s *Scanner, err error) {
+	var (
+		tcmd string
+		pcmd string
+		lang string
+	)
+	tcmd, err = exec.LookPath("tesseract")
+	if err != nil {
+		return
+	}
+	err = godrop.Unveil(tcmd, "rx")
+	if err != nil {
+		return
+	}
+	pcmd, err = exec.LookPath("pdftotext")
+	if err != nil {
+		return
+	}
+	err = godrop.Unveil(pcmd, "rx")
+	if err != nil {
+		return
+	}
+	if langs == nil {
+		langs = defaultLangs
+	}
+	for i, l := range langs {
+		lang += l
+		if i < len(langs) {
+			lang += "+"
+		}
+	}
+	args := []string{"-l", lang, "--oem", "2"}
+	s = &Scanner{
+		base,
+		timeout,
+		Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", runtime.NumCPU())}},
+		PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}},
+	}
+	return
+}
+
+func (s *Scanner) Scan(file string) (filename, text string, err error) {
+	file = filepath.Clean(file)
+	scanfile := filepath.Join(s.base, file)
+	var pdffile string
+	pdf := isPDF(scanfile)
+	if !pdf {
+		var (
+			pdf []byte
+			f   *os.File
+		)
+		p, _ := filepath.Split(file)
+		filename = filepath.Join(p, core.FileName()+pdfExt)
+		pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...)
+		pdfCmd.Env = s.t.env
+		log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args)
+		pdf, err = core.Timeout(pdfCmd, s.timeout)
+		if err != nil {
+			return
+		}
+		pdffile = filepath.Join(s.base, filename)
+		f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, 0640)
+		if err != nil {
+			f.Close()
+			return
+		}
+		_, err = f.Write(pdf)
+		if err != nil {
+			f.Close()
+			return
+		}
+		f.Close()
+	} else {
+		pdffile = scanfile
+		filename = file
+	}
+	txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...)
+	log.Debugf("%s %v", txtCmd.Path, txtCmd.Args)
+	txt, err := core.Timeout(txtCmd, s.timeout)
+	if err != nil {
+		return
+	}
+	if len(txt) <= 2 {
+		err = errors.New("pdf does not contain any text information")
+		return
+	}
+	text = string(nlr.ReplaceAll(wsr.ReplaceAll(txt, ws), nl))
+	if !pdf {
+		err = os.Remove(scanfile)
+	}
+	return
+}

pkg/scan/tesseract.go

@@ -0,0 +1,9 @@
+package scan
+
+var defaultLangs = []string{"eng", "deu"}
+
+type Tesseract struct {
+	cmd  string
+	args []string
+	env  []string
+}