ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

more work done

Browse source
This commit is contained in:
ston1th 2019-05-01 22:25:23 +02:00
commit abc6159044
24 changed files with 309 additions and 230 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pkg/db/index.go
View file

@ -67,6 +67,9 @@ func (db *DB) MoveFile(op, np string) error {
}
func (db *DB) DeleteFile(path string) (id string, err error) {
if !db.IsIndexed(path) {
return
}
id, err = db.GetID(path)
if err != nil {
return

2
pkg/fs/file.go
View file

@ -56,7 +56,7 @@ func (fs *Filesystem) Delete(path string) (removed []string, err error) {
if fullpath == fs.Base {
return nil, errors.New("the root path can not be deleted")
}
fi, err := os.Stat(path)
fi, err := os.Stat(fullpath)
if err != nil {
return
}

3
pkg/fs/fs.go
View file

@ -116,5 +116,8 @@ func Paths(path string) (p core.Paths) {
}
p = append(p, core.Path{Name: v, Abs: filepath.Join(p[i-1].Abs, v)})
}
if len(p) != 0 {
p[len(p)-1].Indexed = true
}
return
}

10
pkg/scan/helper.go Normal file
View file

@ -0,0 +1,10 @@
package scan
import (
"path/filepath"
"strings"
)
func isPDF(path string) bool {
return strings.ToLower(filepath.Ext(path)) == pdfExt
}

6
pkg/scan/pdf.go Normal file
View file

@ -0,0 +1,6 @@
package scan
type PDF struct {
cmd string
args []string
}

124
pkg/scan/scanner.go Normal file
View file

@ -0,0 +1,124 @@
package scan
import (
"errors"
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/godrop/v2"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"time"
)
const pdfExt = ".pdf"
var (
wsr = regexp.MustCompile(`[\t\f ]+`)
nlr = regexp.MustCompile(`[\n]{3,}`)
ws = []byte(" ")
nl = []byte("\n")
)
type Scanner struct {
base string
timeout time.Duration
t Tesseract
p PDF
}
func New(base string, langs []string, timeout time.Duration) (s *Scanner, err error) {
var (
tcmd string
pcmd string
lang string
)
tcmd, err = exec.LookPath("tesseract")
if err != nil {
return
}
err = godrop.Unveil(tcmd, "rx")
if err != nil {
return
}
pcmd, err = exec.LookPath("pdftotext")
if err != nil {
return
}
err = godrop.Unveil(pcmd, "rx")
if err != nil {
return
}
if langs == nil {
langs = defaultLangs
}
for i, l := range langs {
lang += l
if i < len(langs) {
lang += "+"
}
}
args := []string{"-l", lang, "--oem", "2"}
s = &Scanner{
base,
timeout,
Tesseract{tcmd, args, []string{fmt.Sprintf("OMP_THREAD_LIMIT=%d", runtime.NumCPU())}},
PDF{pcmd, []string{"-layout", "-nopgbrk", "-eol", "unix"}},
}
return
}
func (s *Scanner) Scan(file string) (filename, text string, err error) {
file = filepath.Clean(file)
scanfile := filepath.Join(s.base, file)
var pdffile string
pdf := isPDF(scanfile)
if !pdf {
var (
pdf []byte
f *os.File
)
p, _ := filepath.Split(file)
filename = filepath.Join(p, core.FileName()+pdfExt)
pdfCmd := exec.Command(s.t.cmd, append(s.t.args, []string{scanfile, "stdout", "pdf"}...)...)
pdfCmd.Env = s.t.env
log.Debugf("%v %s %v", pdfCmd.Env, pdfCmd.Path, pdfCmd.Args)
pdf, err = core.Timeout(pdfCmd, s.timeout)
if err != nil {
return
}
pdffile = filepath.Join(s.base, filename)
f, err = os.OpenFile(pdffile, os.O_RDWR|os.O_CREATE, 0640)
if err != nil {
f.Close()
return
}
_, err = f.Write(pdf)
if err != nil {
f.Close()
return
}
f.Close()
} else {
pdffile = scanfile
filename = file
}
txtCmd := exec.Command(s.p.cmd, append(s.p.args, []string{pdffile, "-"}...)...)
log.Debugf("%s %v", txtCmd.Path, txtCmd.Args)
txt, err := core.Timeout(txtCmd, s.timeout)
if err != nil {
return
}
if len(txt) <= 2 {
err = errors.New("pdf does not contain any text information")
return
}
text = string(nlr.ReplaceAll(wsr.ReplaceAll(txt, ws), nl))
if !pdf {
err = os.Remove(scanfile)
}
return
}

9
pkg/scan/tesseract.go Normal file
View file

@ -0,0 +1,9 @@
package scan
var defaultLangs = []string{"eng", "deu"}
type Tesseract struct {
cmd string
args []string
env []string
}

90
pkg/server/handler.go
View file

@ -8,6 +8,7 @@ import (
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/otp"
"git.giftfish.de/ston1th/jwt/v3"
"html/template"
"net/http"
"path/filepath"
"strings"
@ -178,10 +179,13 @@ func indexHandler(ctx *Context) {
path := strings.TrimPrefix(ctx.Path(), core.IndexPrefix)
switch ctx.Method() {
case "GET":
earlyErr := make(chan error)
go func() {
file, txt, err := ctx.Srv.Tesseract.Scan(path)
file, txt, err := ctx.Srv.Scanner.Scan(path)
if err != nil {
log.Printf("scan: %s: %s", path, err)
select {
case earlyErr <- err:
}
return
}
id, err := ctx.Srv.DB.Index.Add(txt, "pdf")
@ -195,6 +199,19 @@ func indexHandler(ctx *Context) {
}
}()
p, _ := filepath.Split(path)
p = filepath.Dir(p)
select {
case err := <-earlyErr:
ctx.Template("iseHandler")
ctx.Data = webData{
Data: template.HTML(strings.ReplaceAll(err.Error(), "\n", "<br>")),
Path: p,
}
ctx.Status(http.StatusInternalServerError)
ctx.Exec()
return
case <-time.After(time.Second):
}
ctx.Redirect(p, http.StatusFound)
}
}
@ -231,35 +248,35 @@ func moveHandler(ctx *Context) {
func deleteHandler(ctx *Context) {
path := strings.TrimPrefix(ctx.Path(), core.DeletePrefix)
mode, err := ctx.Srv.FS.Mode(path)
if err != nil {
ctx.Error(err)
return
}
switch mode {
case fs.IsFile:
ctx.Data = webData{
Title: "Delete File",
BodyTitle: "Delete File",
}
case fs.IsDir:
ctx.Data = webData{
Title: "Delete Directory",
BodyTitle: "Delete Directory",
}
default:
ctx.Data = webData{
Title: "Delete",
BodyTitle: "Delete",
}
ctx.Error(noSuchFile)
return
}
ctx.Data.Path = path
ctx.Data.Data = mode
ctx.Template("deleteHandler")
switch ctx.Method() {
case "GET":
mode, err := ctx.Srv.FS.Mode(path)
if err != nil {
ctx.Error(err)
return
}
switch mode {
case fs.IsFile:
ctx.Data = webData{
Title: "Delete File",
BodyTitle: "Delete File",
}
case fs.IsDir:
ctx.Data = webData{
Title: "Delete Directory",
BodyTitle: "Delete Directory",
}
default:
ctx.Data = webData{
Title: "Delete",
BodyTitle: "Delete",
}
ctx.Error(noSuchFile)
return
}
ctx.Data.Path = path
ctx.Data.Data = mode
ctx.Exec()
case "POST":
files, err := ctx.Srv.FS.Delete(path)
@ -272,21 +289,22 @@ func deleteHandler(ctx *Context) {
if err != nil {
log.Printf("db: delete: %s %s", f, err)
}
if id == "" {
continue
}
err = ctx.Srv.DB.Index.Delete(id)
if err != nil {
log.Printf("index: delete: %s %s", f, err)
}
}
ctx.Redirect(core.IndexURI, http.StatusFound)
p, _ := filepath.Split(path)
p = filepath.Dir(p)
ctx.Redirect(p, http.StatusFound)
}
}
func dirHandler(ctx *Context) {
ctx.Template("dirHandler")
ctx.Data = webData{
Title: "Directory Viewer",
BodyTitle: "Directory Viewer",
}
path := ctx.Path()
switch ctx.Method() {
case "GET":
@ -297,12 +315,18 @@ func dirHandler(ctx *Context) {
}
switch mode {
case fs.IsDir:
ctx.Data = webData{
Title: "Directory Viewer",
}
dir := ctx.Srv.FS.ReadDir(path)
ctx.Srv.DB.GetIndexed(dir.Files)
ctx.Data.Data = dir
ctx.Data.Paths = fs.Paths(path)
ctx.Exec()
case fs.IsFile:
ctx.Data = webData{
Title: "Document Viewer",
}
ctx.Template("fileHandler")
ctx.Data.Data = core.RawPrefix + path
ctx.Data.Paths = fs.Paths(path)

12
pkg/server/server.go
View file

@ -10,7 +10,7 @@ import (
"git.giftfish.de/ston1th/docstore/pkg/db"
"git.giftfish.de/ston1th/docstore/pkg/fs"
"git.giftfish.de/ston1th/docstore/pkg/log"
"git.giftfish.de/ston1th/docstore/pkg/tesseract"
"git.giftfish.de/ston1th/docstore/pkg/scan"
"git.giftfish.de/ston1th/godrop/v2"
"git.giftfish.de/ston1th/jwt/v3"
"github.com/gorilla/mux"
@ -31,10 +31,10 @@ type HTTPServer struct {
srv *http.Server
handler http.Handler
DB *db.DB
JWT *jwt.JWT
Tesseract *tesseract.Tesseract
FS *fs.Filesystem
DB *db.DB
JWT *jwt.JWT
Scanner *scan.Scanner
FS *fs.Filesystem
templ map[string]*template.Template
res map[string][]byte
@ -56,7 +56,7 @@ func (s *HTTPServer) Start() (err error) {
if err != nil {
return errors.New("fs: " + err.Error())
}
s.Tesseract, err = tesseract.NewTesseract(s.Config.DataDir, nil, time.Second*30)
s.Scanner, err = scan.New(s.Config.DataDir, nil, time.Second*30)
if err != nil {
return errors.New("tesseract: " + err.Error())
}

64
pkg/server/templates.go
View file

@ -28,7 +28,8 @@ const (
{{if eq .Data 1}}<br>This will also remove all files and directories below.</p>{{end}}
<form class="form-horizontal" action="/delete{{.Path}}" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<button class="btn btn-sm btn-primary" type="submit">Delete</button>
<button class="btn btn-sm btn-danger" type="submit">Delete</button>
<a href="{{.Path}}" class="btn btn-sm btn-primary">Back</a>
</form>
</div>
</div>
@ -36,23 +37,21 @@ const (
</div>
{{end}}`
dir = `{{define "body"}}
<div class="page-header">
<div class="row">
<h2>{{.BodyTitle}}</h2>
</div>
</div>
<div class="row">
<a href="/">root</a>&nbsp;/&nbsp;
{{range $item := .Paths}}
<a href="{{$item.Abs}}">{{$item.Name}}</a>&nbsp;/&nbsp;
{{end}}
<ol class="breadcrumb full-width">
<li class="breadcrumb-item"></li>
<li class="breadcrumb-item"><a href="/">root</a></li>
{{range $item := .Paths}}
<li class="breadcrumb-item{{if $item.Indexed}} active{{end}}">{{if $item.Indexed}}{{$item.Name}}{{else}}<a href="{{$item.Abs}}">{{$item.Name}}</a>{{end}}</li>
{{end}}
</ol>
</div>
<div class="row">
<table class="table table-stripped table-hover">
<thead>
<tr>
<th></th>
<th>Path</th>
<th>Type</th>
<th>Options</th>
</tr>
</thead>
@ -60,23 +59,26 @@ const (
{{if .Data.Dirs}}
{{range $item := .Data.Dirs}}
<tr>
<td><input type="checkbox" name="cb" value="{{$item.Name}}"></td>
<td><a href="{{$item.Abs}}">{{$item.Name}}/</a></td>
<td>Move</td>
<td>Directory</td>
<td><div class="btn-group">
<a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
</div></td>
</tr>
{{end}}
{{end}}
{{if .Data.Files}}
{{range $item := .Data.Files}}
<tr>
<td><input type="checkbox" name="cb" value="{{$item.Name}}"></td>
<td><a href="{{$item.Abs}}">{{$item.Name}}</a></td>
<td>File</td>
<td><div class="btn-group">
{{if not $item.Indexed}}
<a href="/index{{$item.Abs}}" class="btn btn-sm btn-primary">Index</a>
{{end}}
<a href="/move{{$item.Abs}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{$item.Abs}}" class="btn btn-sm btn-primary">Delete</a>
<a href="/delete{{$item.Abs}}" class="btn btn-sm btn-danger">Delete</a>
</div></td>
</tr>
{{end}}
@ -86,16 +88,14 @@ const (
</div>
{{end}}`
file = `{{define "body"}}
<div class="page-header">
<div class="row">
<h2>{{.BodyTitle}}</h2>
</div>
</div>
<div class="row">
<a href="/">root</a>
{{range $i, $item := .Paths}}
&nbsp;/&nbsp;<a href="{{$item.Abs}}">{{$item.Name}}</a>
{{end}}
<ol class="breadcrumb full-width">
<li class="breadcrumb-item"></li>
<li class="breadcrumb-item"><a href="/">root</a></li>
{{range $item := .Paths}}
<li class="breadcrumb-item{{if $item.Indexed}} active{{end}}">{{if $item.Indexed}}{{$item.Name}}{{else}}<a href="{{$item.Abs}}">{{$item.Name}}</a>{{end}}</li>
{{end}}
</ol>
</div>
<div class="row">
<embed src="{{.Data}}" width="100%" height="700px" toolbar="1" statusbar="1" navpanes="1"></embed>
@ -124,7 +124,7 @@ const (
<title>DocStore | {{.Title}}</title>
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" integrity="sha256-bgotk/IPq4Yvt6s8AQGPTM5ZjPjQ6rrBRa2EyxpnFSc="></link>
<link rel="stylesheet" type="text/css" href="/bootstrap.css" media="screen" integrity="sha256-NeLmQ7cX66J4MdtgGlG3O/TgvsSyP1b9WbaQtxYZUbQ="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-5/Vbh+c9BC042HeSbJXZZVIc1dD2b9cy6uH9LFAOSfo="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-iYnSZSZLepCN7vabI6deUcbcw+1UNCmKhXlJ75O+S6Y="></link>
<meta name="theme-color" content="#375a7f">
<meta name="description" content="{{.Title}}">
</head>
@ -153,6 +153,7 @@ const (
<div class="col">
<ul class="list-unstyled">
<li class="float-md-right"><a href="#top">Back to top</a></li>
<li><a href="https://git.giftfish.de/ston1th/docstore">Git</a></li>
</ul>
<p>© DocStore {{.Version}} Request: <strong>{{.Time}}ms</strong></p>
</div>
@ -161,6 +162,13 @@ const (
</div>
</body>
</html>`
ise = `{{define "body"}}
<div class="alert alert-danger msg">
<h4>Error!</h4>
<p>{{.Data}}</p>
<a href="{{.Path}}" class="btn btn-sm btn-primary">Back</a>
</div>
{{end}}`
login = `{{define "body"}}
<div class="page-header">
<div class="row">
@ -566,7 +574,10 @@ html, body, .container, .content {
margin: 0 auto -60px;
}
.push {
height: 60px;
height: 20px;
}
.full-width {
width: 100%;
}
.footer-wrapper {
position: relative;
@ -614,6 +625,7 @@ func (s *HTTPServer) loadTemplates() {
s.templ["deleteHandler"] = parse(index, menu, delete)
s.templ["notFoundHandler"] = parse(index, menu, notFound)
s.templ["forbiddenHandler"] = parse(index, menu, forbidden)
s.templ["iseHandler"] = parse(index, menu, ise)
s.templ["loginHandler"] = parse(index, menu, login)
s.templ["loginTotpHandler"] = parse(index, menu, loginTotp)
s.templ["searchHandler"] = parse(index, menu, search)

67
pkg/tesseract/tesseract.go
View file

@ -1,67 +0,0 @@
package tesseract
import (
"git.giftfish.de/ston1th/docstore/pkg/core"
"os"
"os/exec"
"path/filepath"
"time"
)
const DefaultLang = "eng+deu"
type Tesseract struct {
cmd string
args []string
base string
timeout time.Duration
}
func NewTesseract(base string, langs []string, timeout time.Duration) (t *Tesseract, err error) {
var (
cmd string
lang string
)
cmd, err = exec.LookPath("tesseract")
if err != nil {
return
}
for i, l := range langs {
lang += l
if i < len(langs) {
lang += "+"
}
}
if lang == "" {
lang = DefaultLang
}
args := []string{"-l", lang, "--oem", "2"}
t = &Tesseract{cmd, args, base, timeout}
return
}
func (t *Tesseract) Scan(file string) (filename, text string, err error) {
scanfile := filepath.Join(t.base, file)
p, _ := filepath.Split(file)
filename = filepath.Join(p, core.FileName()+".pdf")
txt, err := core.Timeout(exec.Command(t.cmd, append(t.args, []string{scanfile, "stdout", "txt"}...)...), t.timeout)
if err != nil {
return
}
text = string(txt)
pdf, err := core.Timeout(exec.Command(t.cmd, append(t.args, []string{scanfile, "stdout", "pdf"}...)...), t.timeout)
if err != nil {
return
}
f, err := os.OpenFile(filepath.Join(t.base, filename), os.O_RDWR|os.O_CREATE, 0640)
if err != nil {
return
}
defer f.Close()
_, err = f.Write(pdf)
if err != nil {
return
}
err = os.Remove(scanfile)
return
}