diff --git a/.gitignore b/.gitignore index a4694b8..2f4283f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -gowiki +docstore tmp run.sh TODO.txt diff --git a/README.md b/README.md index eb96770..a15d39d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# GoWiki - a simple wiki engine written in Go +# DocStore - document server and indexing engine ## Building @@ -24,4 +24,4 @@ GOOS=openbsd make vendor ## Install -See the `scripts/` directory on how to install gowiki on Linux or OpenBSD. +See the `scripts/` directory on how to install docstore on OpenBSD. diff --git a/docstore b/docstore deleted file mode 100755 index 981b967..0000000 Binary files a/docstore and /dev/null differ diff --git a/pkg/db/index.go b/pkg/db/index.go index 343e5f2..ec54219 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -67,6 +67,9 @@ func (db *DB) MoveFile(op, np string) error { } func (db *DB) DeleteFile(path string) (id string, err error) { + if !db.IsIndexed(path) { + return + } id, err = db.GetID(path) if err != nil { return diff --git a/pkg/fs/file.go b/pkg/fs/file.go index dce9997..18584b8 100644 --- a/pkg/fs/file.go +++ b/pkg/fs/file.go @@ -56,7 +56,7 @@ func (fs *Filesystem) Delete(path string) (removed []string, err error) { if fullpath == fs.Base { return nil, errors.New("the root path can not be deleted") } - fi, err := os.Stat(path) + fi, err := os.Stat(fullpath) if err != nil { return } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index ab90599..193d145 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -116,5 +116,8 @@ func Paths(path string) (p core.Paths) { } p = append(p, core.Path{Name: v, Abs: filepath.Join(p[i-1].Abs, v)}) } + if len(p) != 0 { + p[len(p)-1].Indexed = true + } return } diff --git a/pkg/scan/helper.go b/pkg/scan/helper.go new file mode 100644 index 0000000..7567d0f --- /dev/null +++ b/pkg/scan/helper.go @@ -0,0 +1,10 @@ +package scan + +import ( + "path/filepath" + "strings" +) + +func isPDF(path string) bool { + return strings.ToLower(filepath.Ext(path)) == pdfExt +} diff --git a/pkg/scan/pdf.go b/pkg/scan/pdf.go new file mode 100644 index 0000000..6fe939f --- /dev/null +++ b/pkg/scan/pdf.go @@ -0,0 +1,6 @@ +package scan + +type PDF struct { + cmd string + args []string +} diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go new file mode 100644 index 0000000..899fe26 --- /dev/null +++ b/pkg/scan/scanner.go @@ -0,0 +1,124 @@ +package scan + +import ( + "errors" + "fmt" + "git.giftfish.de/ston1th/docstore/pkg/core" + "git.giftfish.de/ston1th/docstore/pkg/log" + "git.giftfish.de/ston1th/godrop/v2" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "time" +) + +const pdfExt = ".pdf" + +var ( + wsr = regexp.MustCompile(`[\t\f ]+`) + nlr = regexp.MustCompile(`[\n]{3,}`) + ws = []byte(" ") + nl = []byte("\n") +) + +type Scanner struct { + base string + timeout time.Duration + t Tesseract + p PDF +} + +func New(base string, langs []string, timeout time.Duration) (s *Scanner, err error) { + var ( + tcmd string + pcmd string + lang string + ) + tcmd, err = exec.LookPath("tesseract") + if err != nil { + return + } + err = godrop.Unveil(tcmd, "rx") + if err != nil { + return + } + pcmd, err = exec.LookPath("pdftotext") + if err != nil { + return + } + err = godrop.Unveil(pcmd, "rx") + if err != nil { + return + } + if langs == nil { + langs = defaultLangs + } + for i, l := range langs { + lang += l + if i < len(langs) { + lang += "+" + } + } + args := []string{"-l", lang, "--oem", "2"} + s = &Scanner{ + base, + timeout, + Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", runtime.NumCPU())}}, + PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}}, + } + return +} + +func (s *Scanner) Scan(file string) (filename, text string, err error) { + file = filepath.Clean(file) + scanfile := filepath.Join(s.base, file) + var pdffile string + pdf := isPDF(scanfile) + if !pdf { + var ( + pdf []byte + f *os.File + ) + p, _ := filepath.Split(file) + filename = filepath.Join(p, core.FileName()+pdfExt) + pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...) + pdfCmd.Env = s.t.env + log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args) + pdf, err = core.Timeout(pdfCmd, s.timeout) + if err != nil { + return + } + pdffile = filepath.Join(s.base, filename) + f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, 0640) + if err != nil { + f.Close() + return + } + _, err = f.Write(pdf) + if err != nil { + f.Close() + return + } + f.Close() + } else { + pdffile = scanfile + filename = file + } + txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...) + log.Debugf("%s %v", txtCmd.Path, txtCmd.Args) + txt, err := core.Timeout(txtCmd, s.timeout) + if err != nil { + return + } + if len(txt) <= 2 { + err = errors.New("pdf does not contain any text information") + return + } + text = string(nlr.ReplaceAll(wsr.ReplaceAll(txt, ws), nl)) + if !pdf { + err = os.Remove(scanfile) + } + return +} diff --git a/pkg/scan/tesseract.go b/pkg/scan/tesseract.go new file mode 100644 index 0000000..fc667cf --- /dev/null +++ b/pkg/scan/tesseract.go @@ -0,0 +1,9 @@ +package scan + +var defaultLangs = []string{"eng", "deu"} + +type Tesseract struct { + cmd string + args []string + env []string +} diff --git a/pkg/server/handler.go b/pkg/server/handler.go index ef3dd53..9883170 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -8,6 +8,7 @@ import ( "git.giftfish.de/ston1th/docstore/pkg/log" "git.giftfish.de/ston1th/docstore/pkg/otp" "git.giftfish.de/ston1th/jwt/v3" + "html/template" "net/http" "path/filepath" "strings" @@ -178,10 +179,13 @@ func indexHandler(ctx *Context) { path := strings.TrimPrefix(ctx.Path(), core.IndexPrefix) switch ctx.Method() { case "GET": + earlyErr := make(chan error) go func() { - file, txt, err := ctx.Srv.Tesseract.Scan(path) + file, txt, err := ctx.Srv.Scanner.Scan(path) if err != nil { - log.Printf("scan: %s: %s", path, err) + select { + case earlyErr <- err: + } return } id, err := ctx.Srv.DB.Index.Add(txt, "pdf") @@ -195,6 +199,19 @@ func indexHandler(ctx *Context) { } }() p, _ := filepath.Split(path) + p = filepath.Dir(p) + select { + case err := <-earlyErr: + ctx.Template("iseHandler") + ctx.Data = webData{ + Data: template.HTML(strings.ReplaceAll(err.Error(), "\n", "
")), + Path: p, + } + ctx.Status(http.StatusInternalServerError) + ctx.Exec() + return + case <-time.After(time.Second): + } ctx.Redirect(p, http.StatusFound) } } @@ -231,35 +248,35 @@ func moveHandler(ctx *Context) { func deleteHandler(ctx *Context) { path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix) + mode, err := ctx.Srv.FS.Mode(path) + if err != nil { + ctx.Error(err) + return + } + switch mode { + case fs.IsFile: + ctx.Data = webData{ + Title: "Delete File", + BodyTitle: "Delete File", + } + case fs.IsDir: + ctx.Data = webData{ + Title: "Delete Directory", + BodyTitle: "Delete Directory", + } + default: + ctx.Data = webData{ + Title: "Delete", + BodyTitle: "Delete", + } + ctx.Error(noSuchFile) + return + } + ctx.Data.Path = path + ctx.Data.Data = mode ctx.Template("deleteHandler") switch ctx.Method() { case "GET": - mode, err := ctx.Srv.FS.Mode(path) - if err != nil { - ctx.Error(err) - return - } - switch mode { - case fs.IsFile: - ctx.Data = webData{ - Title: "Delete File", - BodyTitle: "Delete File", - } - case fs.IsDir: - ctx.Data = webData{ - Title: "Delete Directory", - BodyTitle: "Delete Directory", - } - default: - ctx.Data = webData{ - Title: "Delete", - BodyTitle: "Delete", - } - ctx.Error(noSuchFile) - return - } - ctx.Data.Path = path - ctx.Data.Data = mode ctx.Exec() case "POST": files, err := ctx.Srv.FS.Delete(path) @@ -272,21 +289,22 @@ func deleteHandler(ctx *Context) { if err != nil { log.Printf("db: delete: %s %s", f, err) } + if id == "" { + continue + } err = ctx.Srv.DB.Index.Delete(id) if err != nil { log.Printf("index: delete: %s %s", f, err) } } - ctx.Redirect(core.IndexURI, http.StatusFound) + p, _ := filepath.Split(path) + p = filepath.Dir(p) + ctx.Redirect(p, http.StatusFound) } } func dirHandler(ctx *Context) { ctx.Template("dirHandler") - ctx.Data = webData{ - Title: "Directory Viewer", - BodyTitle: "Directory Viewer", - } path := ctx.Path() switch ctx.Method() { case "GET": @@ -297,12 +315,18 @@ func dirHandler(ctx *Context) { } switch mode { case fs.IsDir: + ctx.Data = webData{ + Title: "Directory Viewer", + } dir := ctx.Srv.FS.ReadDir(path) ctx.Srv.DB.GetIndexed(dir.Files) ctx.Data.Data = dir ctx.Data.Paths = fs.Paths(path) ctx.Exec() case fs.IsFile: + ctx.Data = webData{ + Title: "Document Viewer", + } ctx.Template("fileHandler") ctx.Data.Data = core.RawPrefix + path ctx.Data.Paths = fs.Paths(path) diff --git a/pkg/server/server.go b/pkg/server/server.go index 4a67705..7da6c44 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,7 +10,7 @@ import ( "git.giftfish.de/ston1th/docstore/pkg/db" "git.giftfish.de/ston1th/docstore/pkg/fs" "git.giftfish.de/ston1th/docstore/pkg/log" - "git.giftfish.de/ston1th/docstore/pkg/tesseract" + "git.giftfish.de/ston1th/docstore/pkg/scan" "git.giftfish.de/ston1th/godrop/v2" "git.giftfish.de/ston1th/jwt/v3" "github.com/gorilla/mux" @@ -31,10 +31,10 @@ type HTTPServer struct { srv *http.Server handler http.Handler - DB *db.DB - JWT *jwt.JWT - Tesseract *tesseract.Tesseract - FS *fs.Filesystem + DB *db.DB + JWT *jwt.JWT + Scanner *scan.Scanner + FS *fs.Filesystem templ map[string]*template.Template res map[string][]byte @@ -56,7 +56,7 @@ func (s *HTTPServer) Start() (err error) { if err != nil { return errors.New("fs: " + err.Error()) } - s.Tesseract, err = tesseract.NewTesseract(s.Config.DataDir, nil, time.Second*30) + s.Scanner, err = scan.New(s.Config.DataDir, nil, time.Second*30) if err != nil { return errors.New("tesseract: " + err.Error()) } diff --git a/pkg/server/templates.go b/pkg/server/templates.go index bf64c2f..15118c2 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -28,7 +28,8 @@ const ( {{if eq .Data 1}}
This will also remove all files and directories below.

{{end}}
- + + Back
@@ -36,23 +37,21 @@ const ( {{end}}` dir = `{{define "body"}} -
-
-

{{.BodyTitle}}

-
-
-root /  -{{range $item := .Paths}} -{{$item.Name}} /  -{{end}} +
    +
  1. +
  2. root
    3. + {{range $item := .Paths}} +
  3. {{if $item.Indexed}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    4. + {{end}} +
- + @@ -60,23 +59,26 @@ const ( {{if .Data.Dirs}} {{range $item := .Data.Dirs}} - - + + {{end}} {{end}} {{if .Data.Files}} {{range $item := .Data.Files}} - + {{end}} @@ -86,16 +88,14 @@ const ( {{end}}` file = `{{define "body"}} -
-
-

{{.BodyTitle}}

-
-
-root -{{range $i, $item := .Paths}} - / {{$item.Name}} -{{end}} +
    +
  1. +
  2. root
    3. + {{range $item := .Paths}} +
  3. {{if $item.Indexed}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    4. + {{end}} +
@@ -124,7 +124,7 @@ const ( DocStore | {{.Title}} - + @@ -153,6 +153,7 @@ const (

© DocStore {{.Version}} Request: {{.Time}}ms

@@ -161,6 +162,13 @@ const (
` + ise = `{{define "body"}} +
+

Error!

+

{{.Data}}

+ Back +
+{{end}}` login = `{{define "body"}}
@@ -566,7 +574,10 @@ html, body, .container, .content { margin: 0 auto -60px; } .push { - height: 60px; + height: 20px; +} +.full-width { + width: 100%; } .footer-wrapper { position: relative; @@ -614,6 +625,7 @@ func (s *HTTPServer) loadTemplates() { s.templ["deleteHandler"] = parse(index, menu, delete) s.templ["notFoundHandler"] = parse(index, menu, notFound) s.templ["forbiddenHandler"] = parse(index, menu, forbidden) + s.templ["iseHandler"] = parse(index, menu, ise) s.templ["loginHandler"] = parse(index, menu, login) s.templ["loginTotpHandler"] = parse(index, menu, loginTotp) s.templ["searchHandler"] = parse(index, menu, search) diff --git a/pkg/tesseract/tesseract.go b/pkg/tesseract/tesseract.go deleted file mode 100644 index 33a419d..0000000 --- a/pkg/tesseract/tesseract.go +++ /dev/null @@ -1,67 +0,0 @@ -package tesseract - -import ( - "git.giftfish.de/ston1th/docstore/pkg/core" - "os" - "os/exec" - "path/filepath" - "time" -) - -const DefaultLang = "eng+deu" - -type Tesseract struct { - cmd string - args []string - base string - timeout time.Duration -} - -func NewTesseract(base string, langs []string, timeout time.Duration) (t *Tesseract, err error) { - var ( - cmd string - lang string - ) - cmd, err = exec.LookPath("tesseract") - if err != nil { - return - } - for i, l := range langs { - lang += l - if i < len(langs) { - lang += "+" - } - } - if lang == "" { - lang = DefaultLang - } - args := []string{"-l", lang, "--oem", "2"} - t = &Tesseract{cmd, args, base, timeout} - return -} - -func (t *Tesseract) Scan(file string) (filename, text string, err error) { - scanfile := filepath.Join(t.base, file) - p, _ := filepath.Split(file) - filename = filepath.Join(p, core.FileName()+".pdf") - txt, err := core.Timeout(exec.Command(t.cmd, append(t.args, []string{scanfile, "stdout", "txt"}...)...), t.timeout) - if err != nil { - return - } - text = string(txt) - pdf, err := core.Timeout(exec.Command(t.cmd, append(t.args, []string{scanfile, "stdout", "pdf"}...)...), t.timeout) - if err != nil { - return - } - f, err := os.OpenFile(filepath.Join(t.base, filename), os.O_RDWR|os.O_CREATE, 0640) - if err != nil { - return - } - defer f.Close() - _, err = f.Write(pdf) - if err != nil { - return - } - err = os.Remove(scanfile) - return -} diff --git a/scripts/gowiki_init b/scripts/gowiki_init deleted file mode 100644 index c75e064..0000000 --- a/scripts/gowiki_init +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: gowiki -# Required-Start: $syslog $network -# Required-Stop: $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: GoWiki -# Description: GoWiki -### END INIT INFO - -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="wiki engine" -NAME=gowiki -DAEMON=/usr/bin/${NAME} -DAEMON_OPTS="-s" -PID=$(ps aux|grep "${DAEMON}"|awk '!/grep/{print $2}') - -. /lib/lsb/init-functions - -do_start() { - ${DAEMON} ${DAEMON_OPTS} - log_end_msg ${?} -} - -do_stop() { - kill ${PID} - return "${?}" -} - -case "${1}" in - start) - log_action_msg "Starting ${DESC}" "${NAME}" - do_start - ;; - stop) - log_daemon_msg "Stopping ${DESC}" "${NAME}" - do_stop - ;; - restart) - log_daemon_msg "Restarting ${DESC}" "${NAME}" - do_stop - if [ ${?} -ne 0 ]; then - log_end_msg 1 - fi - do_start - ;; - status) - status_of_proc -p ${PID} "${DAEMON}" "${NAME}" && exit 0 || exit ${?} - ;; - *) - echo "Usage: service ${NAME} {start|stop|restart|status}" - exit 3 - ;; -esac diff --git a/scripts/install_linux.sh b/scripts/install_linux.sh deleted file mode 100755 index 49b526a..0000000 --- a/scripts/install_linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -useradd wiki -mkdir /var/{lib,log}/gowiki -chown wiki: /var/{lib,log}/gowiki diff --git a/scripts/install_openbsd.sh b/scripts/install_openbsd.sh index df29e47..d1e8719 100755 --- a/scripts/install_openbsd.sh +++ b/scripts/install_openbsd.sh @@ -1,32 +1,34 @@ #!/bin/sh +pkg_add poppler tesseract tesseract-deu tesseract-eng + # rc script -cat <<'EOF'> /etc/rc.d/gowiki +cat <<'EOF'> /etc/rc.d/docstore #!/bin/sh -daemon="/usr/local/sbin/gowiki" +daemon="/usr/local/sbin/docstore" . /etc/rc.d/rc.subr rc_cmd $1 EOF -chmod 555 /etc/rc.d/gowiki -rcctl enable gowiki -rcctl set gowiki flags server -c /var/wiki/gowiki.conf +chmod 555 /etc/rc.d/docstore +rcctl enable docstore +rcctl set docstore flags server -c /var/wiki/docstore.conf # user useradd -m -b /var -s /sbin/nologin wiki # config -cat < /var/wiki/gowiki.conf +cat < /var/wiki/docstore.conf { "data_dir": "/var/wiki", "secret": "$(openssl rand -hex 32)", "secure_cookie": true } EOF -chmod 640 /var/wiki/gowiki.conf -chgrp wiki /var/wiki/gowiki.conf +chmod 640 /var/wiki/docstore.conf +chgrp wiki /var/wiki/docstore.conf # start -rcctl start gowiki +rcctl start docstore diff --git a/templates.sh b/templates.sh index b12779c..06bc144 100755 --- a/templates.sh +++ b/templates.sh @@ -86,6 +86,7 @@ func (s *HTTPServer) loadTemplates() { s.templ["deleteHandler"] = parse(index, menu, delete) s.templ["notFoundHandler"] = parse(index, menu, notFound) s.templ["forbiddenHandler"] = parse(index, menu, forbidden) + s.templ["iseHandler"] = parse(index, menu, ise) s.templ["loginHandler"] = parse(index, menu, login) s.templ["loginTotpHandler"] = parse(index, menu, loginTotp) s.templ["searchHandler"] = parse(index, menu, search) diff --git a/templates/custom.css b/templates/custom.css index c943c74..a0444ad 100644 --- a/templates/custom.css +++ b/templates/custom.css @@ -139,7 +139,10 @@ html, body, .container, .content { margin: 0 auto -60px; } .push { - height: 60px; + height: 20px; +} +.full-width { + width: 100%; } .footer-wrapper { position: relative; diff --git a/templates/delete.html b/templates/delete.html index 620d2cb..f76bd52 100644 --- a/templates/delete.html +++ b/templates/delete.html @@ -13,7 +13,8 @@ {{if eq .Data 1}}
This will also remove all files and directories below.

{{end}}
- + + Back
diff --git a/templates/dir.html b/templates/dir.html index 6ca41ea..935c612 100644 --- a/templates/dir.html +++ b/templates/dir.html @@ -1,21 +1,19 @@ {{define "body"}} -
-
-

{{.BodyTitle}}

-
-
-root /  -{{range $item := .Paths}} -{{$item.Name}} /  -{{end}} +
    +
  1. +
  2. root
    3. + {{range $item := .Paths}} +
  3. {{if $item.Indexed}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    4. + {{end}} +
PathType Options
{{$item.Name}}/MoveDirectory
+ Move + Delete +
{{$item.Name}}File
{{if not $item.Indexed}} Index {{end}} Move - Delete + Delete
- + @@ -23,23 +21,26 @@ {{if .Data.Dirs}} {{range $item := .Data.Dirs}} - - + + {{end}} {{end}} {{if .Data.Files}} {{range $item := .Data.Files}} - + {{end}} diff --git a/templates/file.html b/templates/file.html index ab97ffb..f862a4b 100644 --- a/templates/file.html +++ b/templates/file.html @@ -1,14 +1,12 @@ {{define "body"}} -
-
-

{{.BodyTitle}}

-
-
-root -{{range $i, $item := .Paths}} - / {{$item.Name}} -{{end}} +
    +
  1. +
  2. root
    3. + {{range $item := .Paths}} +
  3. {{if $item.Indexed}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    4. + {{end}} +
diff --git a/templates/index.html b/templates/index.html index db21ea6..9dd438b 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,7 +4,7 @@ DocStore | {{.Title}} - + @@ -33,6 +33,7 @@

© DocStore {{.Version}} Request: {{.Time}}ms

diff --git a/templates/ise.html b/templates/ise.html new file mode 100644 index 0000000..32e85f7 --- /dev/null +++ b/templates/ise.html @@ -0,0 +1,7 @@ +{{define "body"}} +
+

Error!

+

{{.Data}}

+ Back +
+{{end}}
PathType Options
{{$item.Name}}/MoveDirectory
+ Move + Delete +
{{$item.Name}}File
{{if not $item.Indexed}} Index {{end}} Move - Delete + Delete