better indexing and classifier

pkg/core/types.go

@@ -12,9 +12,10 @@ type User struct {
 }
 
 type Result struct {
-	ID   string
+	ID      string
-	Path string
+	Path    string
-	HTML template.HTML
+	Keyword string
+	HTML    template.HTML
 }
 
 type Path struct {

pkg/db/index.go

@@ -59,6 +59,9 @@ func (db *DB) NewFile(id, path string) (err error) {
 }
 
 func (db *DB) MoveFile(op, np string) error {
+	if !db.IsIndexed(op) {
+		return nil
+	}
 	id, err := db.DeleteFile(op)
 	if err != nil {
 		return err

pkg/fs/fs.go

@@ -25,12 +25,17 @@ type Filesystem struct {
 }
 
 func NewFilesystem(base string) (fs *Filesystem, err error) {
+	if !filepath.IsAbs(base) {
+		err = errors.New("base dir '" + base + "' is not an absolute path")
+		return
+	}
 	fi, err := os.Stat(base)
 	if err != nil {
 		return
 	}
 	if !fi.IsDir() {
 		err = errors.New("base dir '" + base + "' is not a directory")
+		return
 	}
 	fs = &Filesystem{base}
 	return
@@ -68,7 +73,7 @@ func (fs *Filesystem) ReadDir(path string) (d *Directory) {
 }
 
 func (fs *Filesystem) Recursive(p string) (paths core.Paths) {
-	p = Clean(p)
+	p, _ = filepath.Split(Clean(p))
 	if len(p) >= 2 {
 		p = p[:len(p)-1]
 	}

pkg/index/classify.go

@@ -0,0 +1,30 @@
+package index
+
+import (
+	"regexp"
+)
+
+var classes = []class{
+	{regexp.MustCompile(`(?i)[^e]rechnung|quittung|beleg|invoice|bill|check`), []string{"invoice", "rechnung"}},
+	{regexp.MustCompile(`(?i)steuer|lohn|tax|salary`), []string{"tax", "steuer"}},
+	{regexp.MustCompile(`(?i)vertrag|kündigung|vereinbarung|contract|termination|agreement`), []string{"contract", "vertrag"}},
+	{regexp.MustCompile(`(?i)versicherung|versichert|insurance|insured`), []string{"insurance", "versicherung"}},
+}
+
+type class struct {
+	r *regexp.Regexp
+	k []string
+}
+
+func classify(text string) (keyword []string) {
+	keyword = []string{"", ""}
+	n := 0
+	for _, v := range classes {
+		num := len(v.r.FindAllString(text, -1))
+		if num > n && num >= 2 {
+			keyword = v.k
+			n = num
+		}
+	}
+	return
+}

pkg/index/index.go

@@ -7,16 +7,25 @@ import (
 	"git.giftfish.de/ston1th/docstore/pkg/core"
 	"git.giftfish.de/ston1th/godrop/v2"
 	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
 	"html/template"
 	"os"
+	"strings"
+	"time"
 )
 
 const maxSearchResult = 101
 
-type indexDocument struct {
+const docType = "document"
-	Text string `json:"text"`
+
-	Date string `json:"date"`
+type document struct {
-	Type string `json:"type"`
+	Text    string    `json:"text"`
+	Keyword []string  `json:"keyword"`
+	Created time.Time `json:"created"`
+}
+
+func (d *document) Type() string {
+	return docType
 }
 
 type Index struct {
@@ -30,8 +39,10 @@ func NewIndex(path string) (i *Index, err error) {
 		return
 	}
 	if _, err = os.Stat(path); os.IsNotExist(err) {
-		mapping := bleve.NewIndexMapping()
+		var m *mapping.IndexMappingImpl
-		bi, err = bleve.New(path, mapping)
+		m, err = createMapping()
+		//m := bleve.NewIndexMapping()
+		bi, err = bleve.New(path, m)
 		if err != nil {
 			err = errors.New("index: " + err.Error())
 			return
@@ -51,7 +62,7 @@ func (i *Index) Close() error {
 	return i.i.Close()
 }
 
-func (i *Index) Add(text, docType string) (id string, err error) {
+func (i *Index) Add(text string) (id string, err error) {
 	id = newID()
 	for {
 		doc, _ := i.i.Document(id)
@@ -60,7 +71,8 @@ func (i *Index) Add(text, docType string) (id string, err error) {
 		}
 		id = newID()
 	}
-	err = i.i.Index(id, indexDocument{text, core.Now(), docType})
+	k := classify(text)
+	err = i.i.Index(id, &document{text, k, time.Now()})
 	return
 }
 
@@ -86,12 +98,20 @@ func (i *Index) Search(search string) (results []core.Result, err error) {
 			continue
 		}
 		for fragField, frags := range hit.Fragments {
-			if fragField == "text" || fragField == "type" || fragField == "date" {
+			if fragField == "text" || fragField == "created" {
 				for _, f := range frags {
 					r.HTML += template.HTML(f)
 				}
 				break
 			}
+			if fragField == "keyword" {
+				var k string
+				for _, f := range frags {
+					k += f
+				}
+				r.Keyword = strings.ReplaceAll(strings.ReplaceAll(k, "<mark>", ""), "</mark>", "")
+				break
+			}
 		}
 		results = append(results, r)
 	}

pkg/index/mapping.go

@@ -0,0 +1,47 @@
+package index
+
+import (
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
+	"github.com/blevesearch/bleve/analysis/token/edgengram"
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+func createMapping() (m *mapping.IndexMappingImpl, err error) {
+	m = bleve.NewIndexMapping()
+
+	err = m.AddCustomTokenFilter("bigram_tokenfilter", map[string]interface{}{
+		"type": edgengram.Name,
+		"side": edgengram.FRONT,
+		"min":  3.0,
+		"max":  25.0,
+	})
+	if err != nil {
+		return
+	}
+	err = m.AddCustomAnalyzer("fulltext_ngram", map[string]interface{}{
+		"type":      custom.Name,
+		"tokenizer": unicode.Name,
+		"token_filters": []string{
+			lowercase.Name,
+			"bigram_tokenfilter",
+		},
+	})
+	if err != nil {
+		return
+	}
+
+	doc := bleve.NewDocumentMapping()
+	tm := bleve.NewTextFieldMapping()
+	tm.Analyzer = "fulltext_ngram"
+	doc.AddFieldMappingsAt("text", tm)
+	km := bleve.NewTextFieldMapping()
+	doc.AddFieldMappingsAt("keyword", km)
+	cm := bleve.NewDateTimeFieldMapping()
+	doc.AddFieldMappingsAt("created", cm)
+
+	m.AddDocumentMapping(docType, doc)
+	return
+}

pkg/server/handler.go

@@ -189,7 +189,7 @@ func indexHandler(ctx *Context) {
 				}
 				return
 			}
-			id, err := ctx.Srv.DB.Index.Add(txt, "pdf")
+			id, err := ctx.Srv.DB.Index.Add(txt)
 			if err != nil {
 				log.Printf("index: %s: %s", path, err)
 				return

pkg/server/templates.go

@@ -365,7 +365,7 @@ const (
     </div>
 </div>
 <div class="row mt-3">
-    <div class="col-xs-6">
+    <div class="col">
         <form class="form-horizontal" action="/search" method="post">
             <input type="hidden" name="token" value="{{.Token}}">
             <div class="form-group">
@@ -380,16 +380,31 @@ const (
     </div>
 </div>
 <div class="row mt-3">
-    <div class="col">
+    {{if .Data}}
-        {{if .Data}}
+    <table class="table table-stripped table-hover">
-        {{range $item := .Data}}
+    <thead>
+        <tr>
+            <th>File</th>
+            <th>Keyword</th>
+        </tr>
+    </thead>
+    <tbody>
+    {{range $item := .Data}}
+    <tr>
+        <td>
         <a class="mono" href="{{$item.Path}}"><b>{{$item.Path}}</b></a>
-        <pre class="wrap">{{$item.HTML}}</pre><br>
+        {{if $item.HTML}}
+        <br><pre class="wrap">{{$item.HTML}}</pre>
         {{end}}
-        {{else}}
+        </td>
-        <b>No search results found.</b>
+        <td>{{if $item.Keyword}}<span class="badge badge-primary">{{$item.Keyword}}</span>{{end}}</td>
-        {{end}}
+    </tr>
-    </div>
+    {{end}}
+    {{else}}
+    <b>No search results found.</b>
+    {{end}}
+    </tbody>
+    </table>
 </div>
 {{end}}`
 	upload = `{{define "body"}}

templates/search.html

@@ -5,7 +5,7 @@
     </div>
 </div>
 <div class="row mt-3">
-    <div class="col-xs-6">
+    <div class="col">
         <form class="form-horizontal" action="/search" method="post">
             <input type="hidden" name="token" value="{{.Token}}">
             <div class="form-group">
@@ -20,15 +20,30 @@
     </div>
 </div>
 <div class="row mt-3">
-    <div class="col">
+    {{if .Data}}
-        {{if .Data}}
+    <table class="table table-stripped table-hover">
-        {{range $item := .Data}}
+    <thead>
+        <tr>
+            <th>File</th>
+            <th>Keyword</th>
+        </tr>
+    </thead>
+    <tbody>
+    {{range $item := .Data}}
+    <tr>
+        <td>
         <a class="mono" href="{{$item.Path}}"><b>{{$item.Path}}</b></a>
-        <pre class="wrap">{{$item.HTML}}</pre><br>
+        {{if $item.HTML}}
+        <br><pre class="wrap">{{$item.HTML}}</pre>
         {{end}}
-        {{else}}
+        </td>
-        <b>No search results found.</b>
+        <td>{{if $item.Keyword}}<span class="badge badge-primary">{{$item.Keyword}}</span>{{end}}</td>
-        {{end}}
+    </tr>
-    </div>
+    {{end}}
+    {{else}}
+    <b>No search results found.</b>
+    {{end}}
+    </tbody>
+    </table>
 </div>
 {{end}}