ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

some improvements

Browse source
This commit is contained in:
ston1th 2019-08-26 19:58:31 +02:00
commit 58a0dc36d5
25 changed files with 233 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

16
pkg/core/const.go
View file

@ -12,14 +12,14 @@ const (
TagsURI = "/tags"
UserURI = "/user"
RawPrefix = "/raw"
IndexPrefix = "/index"
ReindexPrefix = "/reindex"
UploadPrefix = "/upload"
NewPrefix = "/new"
MovePrefix = "/move"
TagsPrefix = "/tags"
DeletePrefix = "/delete"
RawPrefix = "/raw"
IndexPrefix = "/index"
RetagPrefix = "/retag"
UploadPrefix = "/upload"
NewPrefix = "/new"
MovePrefix = "/move"
TagsPrefix = "/tags"
DeletePrefix = "/delete"
Favicon = "/favicon.ico"
BootstrapCSS = "/bootstrap.css"

4
pkg/core/types.go
View file

@ -30,6 +30,10 @@ type Tag struct {
type Tags []Tag
func (t Tags) Len() int { return len(t) }
func (t Tags) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t Tags) Less(i, j int) bool { return t[i].Name < t[j].Name }
type RTag struct {
Name string
Regex *regexp.Regexp

14
pkg/db/index.go
View file

@ -19,7 +19,7 @@ const (
pathPrefix = "path/"
)
func (db *DB) Reindex(path string, tags core.RTags) (err error) {
func (db *DB) Retag(path string, tags core.RTags) (err error) {
if tags == nil {
tags, err = db.GetAllRTags()
if err != nil {
@ -30,20 +30,20 @@ func (db *DB) Reindex(path string, tags core.RTags) (err error) {
if err != nil {
return
}
log.Debugf("db: reindexing: %s", path)
log.Debugf("db: retagging: %s", path)
doc, err := db.Index.Get(id)
if err != nil {
return
}
doc.Tags = tags.Match(doc.Text)
err = db.Index.Reindex(id, doc)
err = db.Index.Update(id, doc)
if err != nil {
return
}
return db.SetTag(path, doc.Tags)
}
func (db *DB) ReindexAll() (err error) {
func (db *DB) RetagAll() (err error) {
var paths []string
err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
paths = append(paths, k)
@ -57,12 +57,12 @@ func (db *DB) ReindexAll() (err error) {
return
}
for _, p := range paths {
e := db.Reindex(p, tags)
e := db.Retag(p, tags)
if e != nil {
if err == nil {
err = errors.New("reindexAll had errors")
err = errors.New("retagAll had errors")
}
log.Printf("db: error reindexing %s: %s", p, e)
log.Printf("db: error retagging %s: %s", p, e)
}
}
return

2
pkg/db/migration.go
View file

@ -27,7 +27,7 @@ var migrators = []migrator{
if err != nil {
return
}
return db.ReindexAll()
return db.RetagAll()
},
}

2
pkg/db/tags.go
View file

@ -5,6 +5,7 @@ package db
import (
"git.giftfish.de/ston1th/docstore/pkg/core"
"regexp"
"sort"
)
const (
@ -38,6 +39,7 @@ func (db *DB) GetAllTags() (tags core.Tags, err error) {
}
return nil
})
sort.Sort(tags)
return
}

3
pkg/fs/fs.go
View file

@ -26,7 +26,6 @@ type Directory struct {
type Filesystem struct {
Base string
Log *log.ScanLog
// protects scan
m sync.RWMutex
@ -54,7 +53,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) {
err = errors.New("base dir '" + base + "' is not a directory")
return
}
fs = &Filesystem{Base: base, Log: log.NewScanLog(0), scan: make(map[string]struct{})}
fs = &Filesystem{Base: base, scan: make(map[string]struct{})}
return
}

2
pkg/fs/helper.go
View file

@ -30,7 +30,7 @@ var reserved = []string{
core.UserURI,
core.RawPrefix,
core.IndexPrefix,
core.ReindexPrefix,
core.RetagPrefix,
core.UploadPrefix,
core.NewPrefix,
core.MovePrefix,

2
pkg/index/index.go
View file

@ -78,7 +78,7 @@ func (i *Index) Add(text string, tags []string) (id string, err error) {
return
}
func (i *Index) Reindex(id string, doc *Document) error {
func (i *Index) Update(id string, doc *Document) error {
return i.i.Index(id, doc)
}

71
pkg/scan/helper.go
View file

@ -3,24 +3,52 @@
package scan
import (
"bytes"
"net/http"
"os"
"path/filepath"
"strings"
)
const (
pdfContentType = "application/pdf"
maxSniffSize = 512
pdfT = "application/pdf"
jpegT = "image/jpeg"
pngT = "image/png"
bmpT = "image/bmp"
gifT = "image/gif"
webpT = "image/webp"
pnmT = "image/x-portable-anymap"
jp2T = "image/jp2"
tiffT = "image/tiff"
maxSniffSize = 512
)
func isPDF(path string) bool {
// TODO
// return checkContentType(path) || checkExtension(path)
return checkContentType(path)
type extendedSig struct {
sig []byte
ct string
}
func checkContentType(path string) bool {
var extendedSigs = []extendedSig{
{[]byte("P1\x0A"), pnmT},
{[]byte("P2\x0A"), pnmT},
{[]byte("P3\x0A"), pnmT},
{[]byte("P4\x0A"), pnmT},
{[]byte("P5\x0A"), pnmT},
{[]byte("P6\x0A"), pnmT},
{[]byte("P7\x0A"), pnmT},
{[]byte("\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x00\x00\x00\x14\x66\x74\x79\x70\x6A\x70\x32"), jp2T},
{[]byte("\x4d\x4d\x00\x2a"), tiffT},
{[]byte("\x49\x49\x2a\x00"), tiffT},
}
func extendedContentType(buf []byte) string {
for _, sig := range extendedSigs {
if bytes.HasPrefix(buf, sig.sig) {
return sig.ct
}
}
return "application/octet-stream"
}
func isPDF(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
@ -31,9 +59,28 @@ func checkContentType(path string) bool {
if err != nil {
return false
}
return http.DetectContentType(buf) == pdfContentType
return http.DetectContentType(buf) == pdfT
}
func checkExtension(path string) bool {
return strings.ToLower(filepath.Ext(path)) == pdfExt
func isImage(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
buf := make([]byte, maxSniffSize)
_, err = f.Read(buf)
if err != nil {
return false
}
switch http.DetectContentType(buf) {
case jpegT, pngT, bmpT, gifT, webpT:
return true
default:
switch extendedContentType(buf) {
case pnmT, jp2T, tiffT:
return true
}
}
return false
}

4
pkg/scan/scanner.go
View file

@ -106,6 +106,10 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) {
var pdffile string
pdf := isPDF(scanfile)
if !pdf {
if !isImage(scanfile) {
err = errors.New("error: input file is not an image")
return
}
var (
pdf []byte
f *os.File

13
pkg/server/context.go
View file

@ -69,13 +69,14 @@ type webData struct {
NotFound string
User string
Token string
Msg string
Search string
User string
Token string
Msg string
Query string
Path string
Paths core.Paths
Tags []string
Data interface{}
}
@ -209,6 +210,10 @@ func (c *Context) Form(name string) string {
return c.Request.PostFormValue(name)
}
func (c *Context) GetForm(name string) string {
return c.Request.FormValue(name)
}
func (c *Context) Var(name string) (ret string) {
ret, _ = mux.Vars(c.Request)[name]
return

75
pkg/server/handler.go
View file

@ -173,30 +173,30 @@ func rawHandler(ctx *Context) {
http.ServeContent(ctx.Response, ctx.Request, fi.Name(), fi.ModTime(), f)
}
func scanFile(ctx *Context, path string) (err error) {
err = ctx.Srv.FS.AddScan(path)
func scanFile(srv *HTTPServer, path string) (err error) {
err = srv.FS.AddScan(path)
if err != nil {
return
}
go func() {
defer ctx.Srv.FS.RemoveScan(path)
l := ctx.Srv.FS.Log
file, txt, err := ctx.Srv.Scanner.Scan(path)
defer srv.FS.RemoveScan(path)
l := srv.Log
file, txt, err := srv.Scanner.Scan(path)
if err != nil {
l.Printf("scanner: %s: %s", path, err)
return
}
tags, err := ctx.Srv.DB.GetAllRTags()
tags, err := srv.DB.GetAllRTags()
if err != nil {
l.Printf("getTags: %s: %s", path, err)
}
found := tags.Match(txt)
id, err := ctx.Srv.DB.Index.Add(txt, found)
id, err := srv.DB.Index.Add(txt, found)
if err != nil {
l.Printf("index: %s: %s", path, err)
return
}
err = ctx.Srv.DB.NewFile(id, file, found)
err = srv.DB.NewFile(id, file, found)
if err != nil {
l.Printf("newFile: %s: %s", path, err)
}
@ -214,14 +214,19 @@ func indexHandler(ctx *Context) {
path := ctx.Path()
if strings.HasPrefix(path, core.IndexPrefix) {
fpath = strings.TrimPrefix(path, core.IndexPrefix)
err = scanFile(ctx, fpath)
err = scanFile(ctx.Srv, fpath)
}
if strings.HasPrefix(path, core.ReindexPrefix) {
fpath = strings.TrimPrefix(path, core.ReindexPrefix)
if strings.HasPrefix(path, core.RetagPrefix) {
fpath = strings.TrimPrefix(path, core.RetagPrefix)
if fpath == "" {
err = ctx.Srv.DB.ReindexAll()
go func() {
err := ctx.Srv.DB.RetagAll()
if err != nil {
ctx.Srv.Log.Printf("retagAll: %s", err)
}
}()
} else {
err = ctx.Srv.DB.Reindex(fpath, nil)
err = ctx.Srv.DB.Retag(fpath, nil)
}
}
p, _ := filepath.Split(fpath)
@ -237,6 +242,19 @@ func indexHandler(ctx *Context) {
return
}
ctx.Redirect(p, http.StatusFound)
case "POST":
if ctx.Path() != core.IndexPrefix+"/all" || !ctx.CheckXsrf() {
ctx.Redirect(core.IndexURI, http.StatusFound)
return
}
go func() {
for _, f := range ctx.Srv.FS.RecursiveFiles("/") {
if !ctx.Srv.DB.IsIndexed(f) {
scanFile(ctx.Srv, f)
}
}
}()
ctx.Redirect(core.IndexURI, http.StatusFound)
}
}
@ -457,13 +475,21 @@ func dirHandler(ctx *Context) {
Title: "Document Viewer | " + path,
}
ctx.Template("fileHandler")
f := ctx.Srv.DB.IsIndexed(path)
ctx.Data.Data = core.Path{
Name: path,
Abs: core.RawPrefix + path,
Flag: ctx.Srv.DB.IsIndexed(path),
Flag: f,
Scan: ctx.Srv.FS.CheckScan(path),
}
ctx.Data.Paths = fs.Paths(path)
if f {
ctx.Data.Tags, err = ctx.Srv.DB.GetTag(path)
if err != nil {
ctx.Error(err)
return
}
}
ctx.Exec()
default:
p := fs.Clean(path)
@ -573,19 +599,18 @@ func searchHandler(ctx *Context) {
Title: "Search",
BodyTitle: "Search",
}
switch ctx.Method() {
case "GET":
ctx.Exec()
case "POST":
if !ctx.CheckXsrf() {
m := ctx.Method()
switch m {
case "GET", "POST":
if m == "POST" && !ctx.CheckXsrf() {
return
}
s := ctx.Form("search")
ctx.Data.Search = s
if s == "" {
s = `""`
q := ctx.GetForm("query")
ctx.Data.Query = q
if q == "" {
q = `""`
}
res, err := ctx.Srv.DB.Index.Search(s)
res, err := ctx.Srv.DB.Index.Search(q)
if err != nil {
ctx.Error(err)
return
@ -706,7 +731,7 @@ func logsHandler(ctx *Context) {
}
switch ctx.Method() {
case "GET":
ctx.Data.Data = ctx.Srv.FS.Log.Logs()
ctx.Data.Data = ctx.Srv.Log.Logs()
ctx.Exec()
}
}

4
pkg/server/routes.go
View file

@ -48,10 +48,10 @@ var prefixRoutes = []route{
jwtHandler(
authHandler(
indexHandler)),
[]string{"GET"},
[]string{"GET", "POST"},
},
{
core.ReindexPrefix,
core.RetagPrefix,
jwtHandler(
authHandler(
indexHandler)),

2
pkg/server/server.go
View file

@ -35,6 +35,7 @@ type HTTPServer struct {
JWT *jwt.JWT
Scanner *scan.Scanner
FS *fs.Filesystem
Log *log.ScanLog
templ map[string]*template.Template
res map[string][]byte
@ -45,6 +46,7 @@ func NewHTTPServer(cfg core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPS
Config: cfg,
listener: l,
Scanner: s,
Log: log.NewScanLog(0),
}
srv.handler = srv.buildRoutes()
srv.loadTemplates()

59
pkg/server/templates.go
View file

@ -46,17 +46,21 @@ const (
<li class="breadcrumb-item{{if $item.Flag}} active{{end}}">{{if $item.Flag}}{{$item.Name}}{{else}}<a href="{{$item.Abs}}">{{$item.Name}}</a>{{end}}</li>
{{end}}
</ol>
<div class="btn-group btn-breadcrumb">
<a href="/upload{{.Path}}" class="btn btn-sm btn-success">Upload</a>
<a href="/new{{.Path}}" class="btn btn-sm btn-primary">New Directory</a>
{{$pl := len .Paths}}
{{if eq $pl 0}}
<a href="/reindex" class="btn btn-sm btn-warning">Reindex All</a>
{{else}}
<a href="/move{{.Path}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{.Path}}" class="btn btn-sm btn-danger">Delete</a>
{{end}}
</div>
<form action="/index/all" method="post">
<input type="hidden" name="token" value="{{.Token}}">
<div class="btn-group btn-breadcrumb">
<a href="/upload{{.Path}}" class="btn btn-sm btn-success">Upload</a>
<a href="/new{{.Path}}" class="btn btn-sm btn-primary">New Directory</a>
{{$pl := len .Paths}}
{{if eq $pl 0}}
<a href="/retag" class="btn btn-sm btn-warning">Retag All</a>
<button class="btn btn-sm btn-warning" type="submit">Index All</button>
{{else}}
<a href="/move{{.Path}}" class="btn btn-sm btn-primary">Move</a>
<a href="/delete{{.Path}}" class="btn btn-sm btn-danger">Delete</a>
{{end}}
</div>
</form>
</div>
</div>
<div class="row">
@ -88,7 +92,7 @@ const (
<td>File</td>
<td>
{{range $t := $item.Tags}}
<code>{{$t}}</code>
<a href="/search?query=tags:{{$t}}" class="btn badge btn-primary">{{$t}}</a>
{{end}}
</td>
<td><div class="btn-group">
@ -136,7 +140,7 @@ const (
<a href="#" class="btn btn-sm btn-warning">Scanning</a>
{{else}}
{{if .Data.Flag}}
<a href="/reindex{{.Data.Name}}" class="btn btn-sm btn-primary">Reindex</a>
<a href="/retag{{.Data.Name}}" class="btn btn-sm btn-primary">Retag</a>
{{else}}
<a href="/index{{.Data.Name}}" class="btn btn-sm btn-success">Index</a>
{{end}}
@ -147,6 +151,16 @@ const (
</div>
</div>
</div>
{{if .Tags}}
<div class="row">
<div class="alert alert-secondary full-width">
Tags
{{range $t := .Tags}}
<a href="/search?query=tags:{{$t}}" class="btn badge btn-primary tag">{{$t}}</a>
{{end}}
</div>
</div>
{{end}}
<div class="row">
<embed src="{{.Data.Abs}}" width="100%" height="700px" toolbar="1" statusbar="1" navpanes="1"></embed>
</div>
@ -174,7 +188,7 @@ const (
<title>DocStore | {{.Title}}</title>
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" integrity="sha256-bgotk/IPq4Yvt6s8AQGPTM5ZjPjQ6rrBRa2EyxpnFSc="></link>
<link rel="stylesheet" type="text/css" href="/bootstrap.css" media="screen" integrity="sha256-3YeveuySvBgp2GItUS8eV1R/9T1zcPMyvw22SyVf8qo="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-TjpqFXPvQzoakyE489hpNO1YK+wUAVd/AdwqpNz1gwc="></link>
<link rel="stylesheet" type="text/css" href="/custom.css" media="screen" integrity="sha256-6jxgK5/GbgoDJ3zQ2bcYXgSUsOvQ1IF18Krslbwlk7w="></link>
<meta name="theme-color" content="#375a7f">
<meta name="description" content="{{.Title}}">
</head>
@ -298,7 +312,7 @@ const (
{{if .Login}}
<form class="form-inline" method="post" action="/search">
<input type="hidden" name="token" value="{{.Token}}">
<input type="text" class="form-control mr-sm-2 menu" placeholder="Search" name="search" value="{{.Search}}" autocomplete="off">
<input type="text" class="form-control mr-sm-2 menu" placeholder="Search" name="query" value="{{.Query}}" autocomplete="off">
<button class="btn btn-sm btn-info" type="submit">Search</button>
</form>
{{end}}
@ -357,7 +371,7 @@ const (
<input class="form-control input-sm" type="text" id="name" name="name" autocomplete="off" value="{{.Data}}" autofocus required>
</div>
<button class="btn btn-sm btn-primary" type="submit">Create</button>
<a href="/" class="btn btn-sm btn-primary">Back</a>
<a href="{{.Path}}" class="btn btn-sm btn-primary">Back</a>
</form>
</div>
</div>
@ -422,7 +436,7 @@ const (
<input type="hidden" name="token" value="{{.Token}}">
<div class="form-group">
<div class="input-group">
<input type="text" class="form-control input-sm" placeholder="Search" name="search" value="{{.Search}}" autocomplete="off" autofocus>
<input type="text" class="form-control input-sm" placeholder="Query" name="query" value="{{.Query}}" autocomplete="off" autofocus>
<div class="input-group-append">
<button class="btn btn-sm btn-primary" type="submit">Search</button>
</div>
@ -437,7 +451,7 @@ const (
<thead>
<tr>
<th>File</th>
<th>Tags</th>
<th>Tag</th>
</tr>
</thead>
<tbody>
@ -449,7 +463,9 @@ const (
<br><pre class="wrap">{{$item.HTML}}</pre>
{{end}}
</td>
<td>{{$item.Tags}}</td>
<td>
{{if $item.Tags}}<a href="/search?query=tags:{{$item.Tags}}" class="btn badge btn-primary">{{$item.Tags}}</a>{{end}}
</td>
</tr>
{{end}}
</tbody>
@ -547,7 +563,7 @@ created:&lt;="2016-09-21"</code></pre>
{{range $item := .Data}}
{{$len := len $item.Regex}}
<tr>
<td>{{$item.Name}}</td>
<td><a href="/search?query=tags:{{$item.Name}}" class="btn badge btn-primary tag">{{$item.Name}}</a></td>
<td><code>{{printf "%.50s" $item.Regex}}{{if gt $len 50}}...{{end}}</code></td>
<td><div class="btn-group">
<a href="/tags/edit/{{$item.Name}}" class="btn btn-sm btn-primary">Edit</a>
@ -728,6 +744,9 @@ body {
a.dark {
color: #222222;
}
a.tag {
text-decoration: none !important;
}
a.para {
text-decoration: none;
font-size: 18px;