diff --git a/Makefile b/Makefile index 3e78265..393e6e8 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ CC=go BUILD=build -v VERSION=$(shell cat VERSION) -GCFLAGS=-gcflags '-e' +GCFLAGS=-gcflags 'all=-e' LDFLAGS=-ldflags '-X main.version=$(VERSION) -s -w' PROGRAM=docstore ENV=CGO_ENABLED=0 GO111MODULE=on @@ -21,7 +21,7 @@ vendor: clean generate gofmt $(ENV) $(CC) $(BUILD) -mod=vendor $(LDFLAGS) $(PROGRAM): clean generate gofmt - $(ENV) $(CC) $(BUILD) $(LDFLAGS) + $(ENV) $(CC) $(BUILD) $(GCFLAGS) $(LDFLAGS) clean: $(CC) clean -x diff --git a/README.md b/README.md index 38f3726..085ac6c 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,9 @@ See the `scripts/` directory on how to install docstore on Linux or OpenBSD. ## Features * Fulltext indexing of documents + * The `Index All` button tries add every file not already indexed to the index * Custom regex based tags (see https://golang.org/pkg/regexp/syntax/) + * The `Retag All` button will reapply the current tags to all indexed documents * Password hashing using BCrypt * 2FA using TOTP * Content Security Policy diff --git a/pkg/core/const.go b/pkg/core/const.go index 53540b9..d2f0c9a 100644 --- a/pkg/core/const.go +++ b/pkg/core/const.go @@ -12,14 +12,14 @@ const ( TagsURI = "/tags" UserURI = "/user" - RawPrefix = "/raw" - IndexPrefix = "/index" - ReindexPrefix = "/reindex" - UploadPrefix = "/upload" - NewPrefix = "/new" - MovePrefix = "/move" - TagsPrefix = "/tags" - DeletePrefix = "/delete" + RawPrefix = "/raw" + IndexPrefix = "/index" + RetagPrefix = "/retag" + UploadPrefix = "/upload" + NewPrefix = "/new" + MovePrefix = "/move" + TagsPrefix = "/tags" + DeletePrefix = "/delete" Favicon = "/favicon.ico" BootstrapCSS = "/bootstrap.css" diff --git a/pkg/core/types.go b/pkg/core/types.go index e950efb..169c9a5 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -30,6 +30,10 @@ type Tag struct { type Tags []Tag +func (t Tags) Len() int { return len(t) } +func (t Tags) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t Tags) Less(i, j int) bool { return t[i].Name < t[j].Name } + type RTag struct { Name string Regex *regexp.Regexp diff --git a/pkg/db/index.go b/pkg/db/index.go index 77e51b0..a6c2033 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -19,7 +19,7 @@ const ( pathPrefix = "path/" ) -func (db *DB) Reindex(path string, tags core.RTags) (err error) { +func (db *DB) Retag(path string, tags core.RTags) (err error) { if tags == nil { tags, err = db.GetAllRTags() if err != nil { @@ -30,20 +30,20 @@ func (db *DB) Reindex(path string, tags core.RTags) (err error) { if err != nil { return } - log.Debugf("db: reindexing: %s", path) + log.Debugf("db: retagging: %s", path) doc, err := db.Index.Get(id) if err != nil { return } doc.Tags = tags.Match(doc.Text) - err = db.Index.Reindex(id, doc) + err = db.Index.Update(id, doc) if err != nil { return } return db.SetTag(path, doc.Tags) } -func (db *DB) ReindexAll() (err error) { +func (db *DB) RetagAll() (err error) { var paths []string err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error { paths = append(paths, k) @@ -57,12 +57,12 @@ func (db *DB) ReindexAll() (err error) { return } for _, p := range paths { - e := db.Reindex(p, tags) + e := db.Retag(p, tags) if e != nil { if err == nil { - err = errors.New("reindexAll had errors") + err = errors.New("retagAll had errors") } - log.Printf("db: error reindexing %s: %s", p, e) + log.Printf("db: error retagging %s: %s", p, e) } } return diff --git a/pkg/db/migration.go b/pkg/db/migration.go index 90a7562..72d67e0 100644 --- a/pkg/db/migration.go +++ b/pkg/db/migration.go @@ -27,7 +27,7 @@ var migrators = []migrator{ if err != nil { return } - return db.ReindexAll() + return db.RetagAll() }, } diff --git a/pkg/db/tags.go b/pkg/db/tags.go index 19d3d3f..7c5e17f 100644 --- a/pkg/db/tags.go +++ b/pkg/db/tags.go @@ -5,6 +5,7 @@ package db import ( "git.giftfish.de/ston1th/docstore/pkg/core" "regexp" + "sort" ) const ( @@ -38,6 +39,7 @@ func (db *DB) GetAllTags() (tags core.Tags, err error) { } return nil }) + sort.Sort(tags) return } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 5e9e7fc..4413130 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -26,7 +26,6 @@ type Directory struct { type Filesystem struct { Base string - Log *log.ScanLog // protects scan m sync.RWMutex @@ -54,7 +53,7 @@ func NewFilesystem(base string) (fs *Filesystem, err error) { err = errors.New("base dir '" + base + "' is not a directory") return } - fs = &Filesystem{Base: base, Log: log.NewScanLog(0), scan: make(map[string]struct{})} + fs = &Filesystem{Base: base, scan: make(map[string]struct{})} return } diff --git a/pkg/fs/helper.go b/pkg/fs/helper.go index 4d7c3a4..6d3cba3 100644 --- a/pkg/fs/helper.go +++ b/pkg/fs/helper.go @@ -30,7 +30,7 @@ var reserved = []string{ core.UserURI, core.RawPrefix, core.IndexPrefix, - core.ReindexPrefix, + core.RetagPrefix, core.UploadPrefix, core.NewPrefix, core.MovePrefix, diff --git a/pkg/index/index.go b/pkg/index/index.go index 68dfc24..6047a3a 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -78,7 +78,7 @@ func (i *Index) Add(text string, tags []string) (id string, err error) { return } -func (i *Index) Reindex(id string, doc *Document) error { +func (i *Index) Update(id string, doc *Document) error { return i.i.Index(id, doc) } diff --git a/pkg/scan/helper.go b/pkg/scan/helper.go index 21d732e..906529a 100644 --- a/pkg/scan/helper.go +++ b/pkg/scan/helper.go @@ -3,24 +3,52 @@ package scan import ( + "bytes" "net/http" "os" - "path/filepath" - "strings" ) const ( - pdfContentType = "application/pdf" - maxSniffSize = 512 + pdfT = "application/pdf" + jpegT = "image/jpeg" + pngT = "image/png" + bmpT = "image/bmp" + gifT = "image/gif" + webpT = "image/webp" + pnmT = "image/x-portable-anymap" + jp2T = "image/jp2" + tiffT = "image/tiff" + maxSniffSize = 512 ) -func isPDF(path string) bool { - // TODO - // return checkContentType(path) || checkExtension(path) - return checkContentType(path) +type extendedSig struct { + sig []byte + ct string } -func checkContentType(path string) bool { +var extendedSigs = []extendedSig{ + {[]byte("P1\x0A"), pnmT}, + {[]byte("P2\x0A"), pnmT}, + {[]byte("P3\x0A"), pnmT}, + {[]byte("P4\x0A"), pnmT}, + {[]byte("P5\x0A"), pnmT}, + {[]byte("P6\x0A"), pnmT}, + {[]byte("P7\x0A"), pnmT}, + {[]byte("\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x00\x00\x00\x14\x66\x74\x79\x70\x6A\x70\x32"), jp2T}, + {[]byte("\x4d\x4d\x00\x2a"), tiffT}, + {[]byte("\x49\x49\x2a\x00"), tiffT}, +} + +func extendedContentType(buf []byte) string { + for _, sig := range extendedSigs { + if bytes.HasPrefix(buf, sig.sig) { + return sig.ct + } + } + return "application/octet-stream" +} + +func isPDF(path string) bool { f, err := os.Open(path) if err != nil { return false @@ -31,9 +59,28 @@ func checkContentType(path string) bool { if err != nil { return false } - return http.DetectContentType(buf) == pdfContentType + return http.DetectContentType(buf) == pdfT } -func checkExtension(path string) bool { - return strings.ToLower(filepath.Ext(path)) == pdfExt +func isImage(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, maxSniffSize) + _, err = f.Read(buf) + if err != nil { + return false + } + switch http.DetectContentType(buf) { + case jpegT, pngT, bmpT, gifT, webpT: + return true + default: + switch extendedContentType(buf) { + case pnmT, jp2T, tiffT: + return true + } + } + return false } diff --git a/pkg/scan/scanner.go b/pkg/scan/scanner.go index ca01f7d..cfd79ae 100644 --- a/pkg/scan/scanner.go +++ b/pkg/scan/scanner.go @@ -106,6 +106,10 @@ func (s *Scanner) Scan(file string) (filename, text string, err error) { var pdffile string pdf := isPDF(scanfile) if !pdf { + if !isImage(scanfile) { + err = errors.New("error: input file is not an image") + return + } var ( pdf []byte f *os.File diff --git a/pkg/server/context.go b/pkg/server/context.go index 77409bb..12221c2 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -69,13 +69,14 @@ type webData struct { NotFound string - User string - Token string - Msg string - Search string + User string + Token string + Msg string + Query string Path string Paths core.Paths + Tags []string Data interface{} } @@ -209,6 +210,10 @@ func (c *Context) Form(name string) string { return c.Request.PostFormValue(name) } +func (c *Context) GetForm(name string) string { + return c.Request.FormValue(name) +} + func (c *Context) Var(name string) (ret string) { ret, _ = mux.Vars(c.Request)[name] return diff --git a/pkg/server/handler.go b/pkg/server/handler.go index e2b251e..5856077 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -173,30 +173,30 @@ func rawHandler(ctx *Context) { http.ServeContent(ctx.Response, ctx.Request, fi.Name(), fi.ModTime(), f) } -func scanFile(ctx *Context, path string) (err error) { - err = ctx.Srv.FS.AddScan(path) +func scanFile(srv *HTTPServer, path string) (err error) { + err = srv.FS.AddScan(path) if err != nil { return } go func() { - defer ctx.Srv.FS.RemoveScan(path) - l := ctx.Srv.FS.Log - file, txt, err := ctx.Srv.Scanner.Scan(path) + defer srv.FS.RemoveScan(path) + l := srv.Log + file, txt, err := srv.Scanner.Scan(path) if err != nil { l.Printf("scanner: %s: %s", path, err) return } - tags, err := ctx.Srv.DB.GetAllRTags() + tags, err := srv.DB.GetAllRTags() if err != nil { l.Printf("getTags: %s: %s", path, err) } found := tags.Match(txt) - id, err := ctx.Srv.DB.Index.Add(txt, found) + id, err := srv.DB.Index.Add(txt, found) if err != nil { l.Printf("index: %s: %s", path, err) return } - err = ctx.Srv.DB.NewFile(id, file, found) + err = srv.DB.NewFile(id, file, found) if err != nil { l.Printf("newFile: %s: %s", path, err) } @@ -214,14 +214,19 @@ func indexHandler(ctx *Context) { path := ctx.Path() if strings.HasPrefix(path, core.IndexPrefix) { fpath = strings.TrimPrefix(path, core.IndexPrefix) - err = scanFile(ctx, fpath) + err = scanFile(ctx.Srv, fpath) } - if strings.HasPrefix(path, core.ReindexPrefix) { - fpath = strings.TrimPrefix(path, core.ReindexPrefix) + if strings.HasPrefix(path, core.RetagPrefix) { + fpath = strings.TrimPrefix(path, core.RetagPrefix) if fpath == "" { - err = ctx.Srv.DB.ReindexAll() + go func() { + err := ctx.Srv.DB.RetagAll() + if err != nil { + ctx.Srv.Log.Printf("retagAll: %s", err) + } + }() } else { - err = ctx.Srv.DB.Reindex(fpath, nil) + err = ctx.Srv.DB.Retag(fpath, nil) } } p, _ := filepath.Split(fpath) @@ -237,6 +242,19 @@ func indexHandler(ctx *Context) { return } ctx.Redirect(p, http.StatusFound) + case "POST": + if ctx.Path() != core.IndexPrefix+"/all" || !ctx.CheckXsrf() { + ctx.Redirect(core.IndexURI, http.StatusFound) + return + } + go func() { + for _, f := range ctx.Srv.FS.RecursiveFiles("/") { + if !ctx.Srv.DB.IsIndexed(f) { + scanFile(ctx.Srv, f) + } + } + }() + ctx.Redirect(core.IndexURI, http.StatusFound) } } @@ -457,13 +475,21 @@ func dirHandler(ctx *Context) { Title: "Document Viewer | " + path, } ctx.Template("fileHandler") + f := ctx.Srv.DB.IsIndexed(path) ctx.Data.Data = core.Path{ Name: path, Abs: core.RawPrefix + path, - Flag: ctx.Srv.DB.IsIndexed(path), + Flag: f, Scan: ctx.Srv.FS.CheckScan(path), } ctx.Data.Paths = fs.Paths(path) + if f { + ctx.Data.Tags, err = ctx.Srv.DB.GetTag(path) + if err != nil { + ctx.Error(err) + return + } + } ctx.Exec() default: p := fs.Clean(path) @@ -573,19 +599,18 @@ func searchHandler(ctx *Context) { Title: "Search", BodyTitle: "Search", } - switch ctx.Method() { - case "GET": - ctx.Exec() - case "POST": - if !ctx.CheckXsrf() { + m := ctx.Method() + switch m { + case "GET", "POST": + if m == "POST" && !ctx.CheckXsrf() { return } - s := ctx.Form("search") - ctx.Data.Search = s - if s == "" { - s = `""` + q := ctx.GetForm("query") + ctx.Data.Query = q + if q == "" { + q = `""` } - res, err := ctx.Srv.DB.Index.Search(s) + res, err := ctx.Srv.DB.Index.Search(q) if err != nil { ctx.Error(err) return @@ -706,7 +731,7 @@ func logsHandler(ctx *Context) { } switch ctx.Method() { case "GET": - ctx.Data.Data = ctx.Srv.FS.Log.Logs() + ctx.Data.Data = ctx.Srv.Log.Logs() ctx.Exec() } } diff --git a/pkg/server/routes.go b/pkg/server/routes.go index fbc4213..f610448 100644 --- a/pkg/server/routes.go +++ b/pkg/server/routes.go @@ -48,10 +48,10 @@ var prefixRoutes = []route{ jwtHandler( authHandler( indexHandler)), - []string{"GET"}, + []string{"GET", "POST"}, }, { - core.ReindexPrefix, + core.RetagPrefix, jwtHandler( authHandler( indexHandler)), diff --git a/pkg/server/server.go b/pkg/server/server.go index 6480ccc..93afdae 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -35,6 +35,7 @@ type HTTPServer struct { JWT *jwt.JWT Scanner *scan.Scanner FS *fs.Filesystem + Log *log.ScanLog templ map[string]*template.Template res map[string][]byte @@ -45,6 +46,7 @@ func NewHTTPServer(cfg core.Config, l net.Listener, s *scan.Scanner) (srv *HTTPS Config: cfg, listener: l, Scanner: s, + Log: log.NewScanLog(0), } srv.handler = srv.buildRoutes() srv.loadTemplates() diff --git a/pkg/server/templates.go b/pkg/server/templates.go index b546bbe..b7e24aa 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -46,17 +46,21 @@ const (
  • {{if $item.Flag}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    • {{end}} -
    - Upload - New Directory - {{$pl := len .Paths}} - {{if eq $pl 0}} - Reindex All - {{else}} - Move - Delete - {{end}} -
    +
    + +
    + Upload + New Directory + {{$pl := len .Paths}} + {{if eq $pl 0}} + Retag All + + {{else}} + Move + Delete + {{end}} +
    +
    @@ -88,7 +92,7 @@ const ( File {{range $t := $item.Tags}} - {{$t}} + {{$t}} {{end}}
    @@ -136,7 +140,7 @@ const ( Scanning {{else}} {{if .Data.Flag}} - Reindex + Retag {{else}} Index {{end}} @@ -147,6 +151,16 @@ const (
    +{{if .Tags}} +
    +
    + Tags + {{range $t := .Tags}} + {{$t}} + {{end}} +
    +
    +{{end}}
    @@ -174,7 +188,7 @@ const ( DocStore | {{.Title}} - + @@ -298,7 +312,7 @@ const ( {{if .Login}}
    - +
    {{end}} @@ -357,7 +371,7 @@ const ( - Back + Back @@ -422,7 +436,7 @@ const (
    - +
    @@ -437,7 +451,7 @@ const ( File - Tags + Tag @@ -449,7 +463,9 @@ const ( 
    {{$item.HTML}}
    {{end}} - {{$item.Tags}} + + {{if $item.Tags}}{{$item.Tags}}{{end}} + {{end}} @@ -547,7 +563,7 @@ created:<="2016-09-21" {{range $item := .Data}} {{$len := len $item.Regex}} - {{$item.Name}} + {{$item.Name}} {{printf "%.50s" $item.Regex}}{{if gt $len 50}}...{{end}}
    Edit @@ -728,6 +744,9 @@ body { a.dark { color: #222222; } +a.tag { + text-decoration: none !important; +} a.para { text-decoration: none; font-size: 18px; diff --git a/templates/custom.css b/templates/custom.css index 54f0822..2b564f8 100644 --- a/templates/custom.css +++ b/templates/custom.css @@ -31,6 +31,9 @@ body { a.dark { color: #222222; } +a.tag { + text-decoration: none !important; +} a.para { text-decoration: none; font-size: 18px; diff --git a/templates/dir.html b/templates/dir.html index b2ff7ba..97b7cba 100644 --- a/templates/dir.html +++ b/templates/dir.html @@ -8,17 +8,21 @@
  • {{if $item.Flag}}{{$item.Name}}{{else}}{{$item.Name}}{{end}}
    • {{end}} -
    - Upload - New Directory - {{$pl := len .Paths}} - {{if eq $pl 0}} - Reindex All - {{else}} - Move - Delete - {{end}} -
    +
    + +
    + Upload + New Directory + {{$pl := len .Paths}} + {{if eq $pl 0}} + Retag All + + {{else}} + Move + Delete + {{end}} +
    +
    @@ -50,7 +54,7 @@ File {{range $t := $item.Tags}} - {{$t}} + {{$t}} {{end}}
    diff --git a/templates/file.html b/templates/file.html index e414100..2931b3e 100644 --- a/templates/file.html +++ b/templates/file.html @@ -13,7 +13,7 @@ Scanning {{else}} {{if .Data.Flag}} - Reindex + Retag {{else}} Index {{end}} @@ -24,6 +24,16 @@
    +{{if .Tags}} +
    +
    + Tags + {{range $t := .Tags}} + {{$t}} + {{end}} +
    +
    +{{end}}
    diff --git a/templates/index.html b/templates/index.html index d1fd8c7..17005b7 100644 --- a/templates/index.html +++ b/templates/index.html @@ -4,7 +4,7 @@ DocStore | {{.Title}} - + diff --git a/templates/menu.html b/templates/menu.html index 7b4c5ec..f680e3d 100644 --- a/templates/menu.html +++ b/templates/menu.html @@ -9,7 +9,7 @@ {{if .Login}}
    - +
    {{end}} diff --git a/templates/newDir.html b/templates/newDir.html index d4b6627..7e82ef7 100644 --- a/templates/newDir.html +++ b/templates/newDir.html @@ -14,7 +14,7 @@ - Back + Back diff --git a/templates/search.html b/templates/search.html index 4192622..1158011 100644 --- a/templates/search.html +++ b/templates/search.html @@ -10,7 +10,7 @@
    - +
    @@ -25,7 +25,7 @@ File - Tags + Tag @@ -37,7 +37,9 @@ 
    {{$item.HTML}}
    {{end}} - {{$item.Tags}} + + {{if $item.Tags}}{{$item.Tags}}{{end}} + {{end}} diff --git a/templates/tagsList.html b/templates/tagsList.html index 0b3d6df..aac289a 100644 --- a/templates/tagsList.html +++ b/templates/tagsList.html @@ -23,7 +23,7 @@ {{range $item := .Data}} {{$len := len $item.Regex}} - {{$item.Name}} + {{$item.Name}} {{printf "%.50s" $item.Regex}}{{if gt $len 50}}...{{end}}
    Edit