added reindexing

README.md

@@ -2,6 +2,10 @@
 
 ## Building
 
+**Requirements:**
+
+* Go (golang) >= 1.12
+
 Linux:
 
 ```
@@ -46,6 +50,19 @@ See the `scripts/` directory on how to install docstore on Linux or OpenBSD.
 }
 ```
 
+## Features
+
+* Fulltext indexing of documents
+* Custom regex based tags (see https://golang.org/pkg/regexp/syntax/)
+* Password hashing using BCrypt
+* 2FA using TOTP
+* Content Security Policy
+* No Javascript (yet)
+
+### DocStore on OpenBSD
+
+DocStore makes use of the OpenBSD native security features [pledge(2)](https://man.openbsd.org/pledge) and [unveil(2)](https://man.openbsd.org/unveil).
+
 ## Usage
 
 DocStore manages all files and directories under the `$run_dir/data` directory.

pkg/core/const.go

@@ -12,13 +12,14 @@ const (
 	TagsURI   = "/tags"
 	UserURI   = "/user"
 
-	RawPrefix    = "/raw"
+	RawPrefix     = "/raw"
-	IndexPrefix  = "/index"
+	IndexPrefix   = "/index"
-	UploadPrefix = "/upload"
+	ReindexPrefix = "/reindex"
-	NewPrefix    = "/new"
+	UploadPrefix  = "/upload"
-	MovePrefix   = "/move"
+	NewPrefix     = "/new"
-	TagsPrefix   = "/tags"
+	MovePrefix    = "/move"
-	DeletePrefix = "/delete"
+	TagsPrefix    = "/tags"
+	DeletePrefix  = "/delete"
 
 	Favicon      = "/favicon.ico"
 	BootstrapCSS = "/bootstrap.css"

pkg/db/index.go

@@ -3,14 +3,72 @@
 package db
 
 import (
+	"errors"
 	"git.giftfish.de/ston1th/docstore/pkg/core"
+	"git.giftfish.de/ston1th/docstore/pkg/log"
+	"git.giftfish.de/ston1th/docstore/pkg/tags"
 )
 
 const (
-	idPrefix   = "id/"
+	// schema:
+	// key: id/<path>
+	// value: <id>
+	idPrefix = "id/"
+	// schema:
+	// key: path/<id>
+	// value: <path>
 	pathPrefix = "path/"
 )
 
+func (db *DB) Reindex(path string, match []core.Tag) (err error) {
+	if match == nil {
+		match, err = db.GetAllTags()
+		if err != nil {
+			return
+		}
+	}
+	id, err := db.GetID(path)
+	if err != nil {
+		return
+	}
+	log.Debugf("db: reindexing: %s", path)
+	doc, err := db.Index.Get(id)
+	if err != nil {
+		return
+	}
+	doc.Tags = tags.Find(doc.Text, match)
+	err = db.Index.Reindex(id, doc)
+	if err != nil {
+		return
+	}
+	return db.SetTag(path, doc.Tags)
+}
+
+func (db *DB) ReindexAll() (err error) {
+	var paths []string
+	err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
+		paths = append(paths, k)
+		return nil
+	})
+	if err != nil {
+		return
+	}
+	match, err := db.GetAllTags()
+	if err != nil {
+		return
+	}
+	for _, p := range paths {
+		e := db.Reindex(p, match)
+		if e != nil {
+			if err == nil {
+				err = errors.New("reindexAll had errors")
+			}
+			log.Printf("db: error reindexing %s: %s", p, e)
+		}
+	}
+	return
+}
+
 func (db *DB) GetPaths(res []core.Result) {
 	for i, v := range res {
 		err := db.store.Get(pathPrefix+v.ID, &res[i].Path)
@@ -49,7 +107,7 @@ func (db *DB) DeleteID(path string) error {
 	return db.store.Delete(idPrefix + path)
 }
 
-func (db *DB) NewFile(id, path string, tags []string) (err error) {
+func (db *DB) NewFile(id, path string, tag []string) (err error) {
 	err = db.store.Set(idPrefix+path, id)
 	if err != nil {
 		return
@@ -58,21 +116,21 @@ func (db *DB) NewFile(id, path string, tags []string) (err error) {
 	if err != nil {
 		return
 	}
-	return db.SetTag(path, tags)
+	return db.SetTag(path, tag)
 }
 
 func (db *DB) MoveFile(op, np string) (err error) {
 	if !db.IsIndexed(op) {
 		return nil
 	}
-	id, tags, err := db.DeleteFile(op)
+	id, tag, err := db.DeleteFile(op)
 	if err != nil {
 		return
 	}
-	return db.NewFile(id, np, tags)
+	return db.NewFile(id, np, tag)
 }
 
-func (db *DB) DeleteFile(path string) (id string, tags []string, err error) {
+func (db *DB) DeleteFile(path string) (id string, tag []string, err error) {
 	if !db.IsIndexed(path) {
 		return
 	}
@@ -84,7 +142,7 @@ func (db *DB) DeleteFile(path string) (id string, tags []string, err error) {
 	if err != nil {
 		return
 	}
-	tags, err = db.UnsetTag(path)
+	tag, err = db.UnsetTag(path)
 	if err != nil {
 		return
 	}

pkg/db/migration.go

@@ -22,8 +22,12 @@ var migrators = []migrator{
 		// dummy placeholder
 		return nil
 	},
-	func(db *DB) error {
+	func(db *DB) (err error) {
-		return db.addDefaultTags()
+		err = db.addDefaultTags()
+		if err != nil {
+			return
+		}
+		return db.ReindexAll()
 	},
 }
 
@@ -43,7 +47,7 @@ func (db *DB) runMigrations() (err error) {
 	log.Printf("db: database version %d", version)
 	for version < currentVersion {
 		v := version + 1
-		log.Printf("db: running migration %d", v)
+		log.Printf("db: running migration version %d", v)
 		err = migrators[v](db)
 		if err != nil {
 			return
@@ -52,6 +56,8 @@ func (db *DB) runMigrations() (err error) {
 		if err != nil {
 			return
 		}
+		version = v
+		log.Printf("db: database version %d", version)
 	}
 	return
 }

pkg/fs/helper.go

@@ -30,6 +30,7 @@ var reserved = []string{
 	core.UserURI,
 	core.RawPrefix,
 	core.IndexPrefix,
+	core.ReindexPrefix,
 	core.UploadPrefix,
 	core.NewPrefix,
 	core.MovePrefix,

pkg/index/index.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"git.giftfish.de/ston1th/docstore/pkg/core"
 	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/document"
 	"github.com/blevesearch/bleve/mapping"
 	"html/template"
 	"os"
@@ -17,13 +18,13 @@ const maxSearchResult = 101
 
 const docType = "document"
 
-type document struct {
+type Document struct {
 	Text    string    `json:"text"`
 	Tags    []string  `json:"tags"`
 	Created time.Time `json:"created"`
 }
 
-func (d *document) Type() string {
+func (d *Document) Type() string {
 	return docType
 }
 
@@ -70,7 +71,34 @@ func (i *Index) Add(text string, tags []string) (id string, err error) {
 		}
 		id = newID()
 	}
-	err = i.i.Index(id, &document{text, tags, time.Now()})
+	err = i.i.Index(id, &Document{text, tags, time.Now()})
+	return
+}
+
+func (i *Index) Reindex(id string, doc *Document) error {
+	return i.i.Index(id, doc)
+}
+
+func (i *Index) Get(id string) (doc *Document, err error) {
+	d, err := i.i.Document(id)
+	if d == nil || err != nil {
+		return
+	}
+	doc = new(Document)
+	for _, f := range d.Fields {
+		n := f.Name()
+		switch n {
+		case "text":
+			text := document.NewTextField(n, f.ArrayPositions(), f.Value())
+			doc.Text = string(text.Value())
+		//case "tags":
+		//	tags := document.NewTextField(n, f.ArrayPositions(), f.Value())
+		//	doc.Tags = append(doc.Tags, string(tags.Value()))
+		case "created":
+			created := document.NewDateTimeFieldFromBytes(n, f.ArrayPositions(), f.Value())
+			doc.Created, _ = created.DateTime()
+		}
+	}
 	return
 }
 

pkg/server/context.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
+	"strconv"
 	"time"
 )
 
@@ -59,7 +60,7 @@ type Context struct {
 
 type webData struct {
 	Version string
-	Time    int64
+	Time    string
 
 	Title     string
 	BodyTitle string
@@ -98,7 +99,12 @@ func (c *Context) Exec() {
 	}
 	c.Data.User = c.User()
 	c.Data.Login = c.LoggedOn()
-	c.Data.Time = time.Since(c.Time).Nanoseconds() / 1e6
+
+	t := strconv.Itoa(int(time.Since(c.Time).Nanoseconds()/1e6)) + "ms"
+	if t == "0ms" {
+		t = strconv.Itoa(int(time.Since(c.Time).Nanoseconds()/1e5)) + "µs"
+	}
+	c.Data.Time = t
 
 	if c.Data.Msg != "" {
 		c.Status(http.StatusBadRequest)

pkg/server/handler.go

@@ -206,11 +206,26 @@ func scanFile(ctx *Context, path string) (err error) {
 }
 
 func indexHandler(ctx *Context) {
-	path := strings.TrimPrefix(ctx.Path(), core.IndexPrefix)
 	switch ctx.Method() {
 	case "GET":
-		err := scanFile(ctx, path)
+		var (
-		p, _ := filepath.Split(path)
+			err   error
+			fpath string
+		)
+		path := ctx.Path()
+		if strings.HasPrefix(path, core.IndexPrefix) {
+			fpath = strings.TrimPrefix(path, core.IndexPrefix)
+			err = scanFile(ctx, fpath)
+		}
+		if strings.HasPrefix(path, core.ReindexPrefix) {
+			fpath = strings.TrimPrefix(path, core.ReindexPrefix)
+			if fpath == "" {
+				err = ctx.Srv.DB.ReindexAll()
+			} else {
+				err = ctx.Srv.DB.Reindex(fpath, nil)
+			}
+		}
+		p, _ := filepath.Split(fpath)
 		p = filepath.Dir(p)
 		if err != nil {
 			ctx.Template("iseHandler")
@@ -429,10 +444,10 @@ func dirHandler(ctx *Context) {
 		}
 		switch mode {
 		case fs.IsDir:
-			ctx.Data = webData{
-				Title: "Directory Viewer",
-			}
 			dir := ctx.Srv.FS.ReadDir(path)
+			ctx.Data = webData{
+				Title: "Directory Viewer | " + path,
+			}
 			ctx.Srv.DB.GetIndexed(dir.Files)
 			ctx.Data.Path = path
 			ctx.Data.Data = dir
@@ -440,7 +455,7 @@ func dirHandler(ctx *Context) {
 			ctx.Exec()
 		case fs.IsFile:
 			ctx.Data = webData{
-				Title: "Document Viewer",
+				Title: "Document Viewer | " + path,
 			}
 			ctx.Template("fileHandler")
 			ctx.Data.Data = core.Path{
@@ -566,12 +581,12 @@ func searchHandler(ctx *Context) {
 		if !ctx.CheckXsrf() {
 			return
 		}
-		ctx.Data.Search = ctx.Form("search")
+		s := ctx.Form("search")
-		if ctx.Data.Search == "" {
+		ctx.Data.Search = s
-			ctx.Error("empty search request")
+		if s == "" {
-			return
+			s = `""`
 		}
-		res, err := ctx.Srv.DB.Index.Search(ctx.Data.Search)
+		res, err := ctx.Srv.DB.Index.Search(s)
 		if err != nil {
 			ctx.Error(err)
 			return

pkg/server/routes.go

@@ -48,7 +48,14 @@ var prefixRoutes = []route{
 		jwtHandler(
 			authHandler(
 				indexHandler)),
-		[]string{"GET", "POST"},
+		[]string{"GET"},
+	},
+	{
+		core.ReindexPrefix,
+		jwtHandler(
+			authHandler(
+				indexHandler)),
+		[]string{"GET"},
 	},
 	{
 		core.UploadPrefix,

templates/dir.html

@@ -10,9 +10,14 @@
     </ol>
     <div class="btn-group btn-breadcrumb">
         <a href="/upload{{.Path}}" class="btn btn-sm btn-success">Upload</a>
-        <a href="/new{{.Path}}" class="btn btn-sm btn-primary">New</a>
+        <a href="/new{{.Path}}" class="btn btn-sm btn-primary">New Directory</a>
+        {{$pl := len .Paths}}
+        {{if eq $pl 0}}
+        <a href="/reindex" class="btn btn-sm btn-warning">Reindex All</a>
+        {{else}}
         <a href="/move{{.Path}}" class="btn btn-sm btn-primary">Move</a>
         <a href="/delete{{.Path}}" class="btn btn-sm btn-danger">Delete</a>
+        {{end}}
     </div>
     </div>
 </div>

templates/file.html

@@ -12,7 +12,9 @@
         {{if .Data.Scan}}
         <a href="#" class="btn btn-sm btn-warning">Scanning</a>
         {{else}}
-        {{if not .Data.Flag}}
+        {{if .Data.Flag}}
+        <a href="/reindex{{.Data.Name}}" class="btn btn-sm btn-primary">Reindex</a>
+        {{else}}
         <a href="/index{{.Data.Name}}" class="btn btn-sm btn-success">Index</a>
         {{end}}
         {{end}}

templates/index.html

@@ -35,7 +35,7 @@
                             <li class="float-md-right"><a href="#top">Back to top</a></li>
                             <li><a href="https://git.giftfish.de/ston1th/docstore">Git</a></li>
                         </ul>
-                        <p>© DocStore {{.Version}} Request: <strong>{{.Time}}ms</strong></p>
+                        <p>© DocStore {{.Version}} Request: <strong>{{.Time}}</strong></p>
                     </div>
                 </div>
             </footer>

templates/search.html

@@ -40,10 +40,23 @@
         <td>{{$item.Tags}}</td>
     </tr>
     {{end}}
-    {{else}}
-    <b>No search results found.</b>
-    {{end}}
     </tbody>
     </table>
+    {{else}}
+        <b>No search results found.</b>
+</div>
+<div class="row mt-3">
+    <div class="col col-nopad">
+        <div class="alert alert-primary">
+            <h4>Query Syntax</h4>
+            <p>Fields</p>
+            <pre><code>tags:insurance
+tags:insurance or tags:invoice
+created:&lt;="2016-09-21"</code></pre>
+            <p>Fulltext</p>
+            <pre><code>some text here</code></pre>
+        </div>
+    </div>
+    {{end}}
 </div>
 {{end}}