better indexing and classifier

pkg/index/classify.go

@@ -0,0 +1,30 @@
+package index
+
+import (
+	"regexp"
+)
+
+var classes = []class{
+	{regexp.MustCompile(`(?i)[^e]rechnung|quittung|beleg|invoice|bill|check`), []string{"invoice", "rechnung"}},
+	{regexp.MustCompile(`(?i)steuer|lohn|tax|salary`), []string{"tax", "steuer"}},
+	{regexp.MustCompile(`(?i)vertrag|kündigung|vereinbarung|contract|termination|agreement`), []string{"contract", "vertrag"}},
+	{regexp.MustCompile(`(?i)versicherung|versichert|insurance|insured`), []string{"insurance", "versicherung"}},
+}
+
+type class struct {
+	r *regexp.Regexp
+	k []string
+}
+
+func classify(text string) (keyword []string) {
+	keyword = []string{"", ""}
+	n := 0
+	for _, v := range classes {
+		num := len(v.r.FindAllString(text, -1))
+		if num > n && num >= 2 {
+			keyword = v.k
+			n = num
+		}
+	}
+	return
+}

pkg/index/index.go

@@ -7,16 +7,25 @@ import (
 	"git.giftfish.de/ston1th/docstore/pkg/core"
 	"git.giftfish.de/ston1th/godrop/v2"
 	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/mapping"
 	"html/template"
 	"os"
+	"strings"
+	"time"
 )
 
 const maxSearchResult = 101
 
-type indexDocument struct {
-	Text string `json:"text"`
-	Date string `json:"date"`
-	Type string `json:"type"`
+const docType = "document"
+
+type document struct {
+	Text    string    `json:"text"`
+	Keyword []string  `json:"keyword"`
+	Created time.Time `json:"created"`
+}
+
+func (d *document) Type() string {
+	return docType
 }
 
 type Index struct {
@@ -30,8 +39,10 @@ func NewIndex(path string) (i *Index, err error) {
 		return
 	}
 	if _, err = os.Stat(path); os.IsNotExist(err) {
-		mapping := bleve.NewIndexMapping()
-		bi, err = bleve.New(path, mapping)
+		var m *mapping.IndexMappingImpl
+		m, err = createMapping()
+		//m := bleve.NewIndexMapping()
+		bi, err = bleve.New(path, m)
 		if err != nil {
 			err = errors.New("index: " + err.Error())
 			return
@@ -51,7 +62,7 @@ func (i *Index) Close() error {
 	return i.i.Close()
 }
 
-func (i *Index) Add(text, docType string) (id string, err error) {
+func (i *Index) Add(text string) (id string, err error) {
 	id = newID()
 	for {
 		doc, _ := i.i.Document(id)
@@ -60,7 +71,8 @@ func (i *Index) Add(text, docType string) (id string, err error) {
 		}
 		id = newID()
 	}
-	err = i.i.Index(id, indexDocument{text, core.Now(), docType})
+	k := classify(text)
+	err = i.i.Index(id, &document{text, k, time.Now()})
 	return
 }
 
@@ -86,12 +98,20 @@ func (i *Index) Search(search string) (results []core.Result, err error) {
 			continue
 		}
 		for fragField, frags := range hit.Fragments {
-			if fragField == "text" || fragField == "type" || fragField == "date" {
+			if fragField == "text" || fragField == "created" {
 				for _, f := range frags {
 					r.HTML += template.HTML(f)
 				}
 				break
 			}
+			if fragField == "keyword" {
+				var k string
+				for _, f := range frags {
+					k += f
+				}
+				r.Keyword = strings.ReplaceAll(strings.ReplaceAll(k, "<mark>", ""), "</mark>", "")
+				break
+			}
 		}
 		results = append(results, r)
 	}

pkg/index/mapping.go

@@ -0,0 +1,47 @@
+package index
+
+import (
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
+	"github.com/blevesearch/bleve/analysis/token/edgengram"
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+func createMapping() (m *mapping.IndexMappingImpl, err error) {
+	m = bleve.NewIndexMapping()
+
+	err = m.AddCustomTokenFilter("bigram_tokenfilter", map[string]interface{}{
+		"type": edgengram.Name,
+		"side": edgengram.FRONT,
+		"min":  3.0,
+		"max":  25.0,
+	})
+	if err != nil {
+		return
+	}
+	err = m.AddCustomAnalyzer("fulltext_ngram", map[string]interface{}{
+		"type":      custom.Name,
+		"tokenizer": unicode.Name,
+		"token_filters": []string{
+			lowercase.Name,
+			"bigram_tokenfilter",
+		},
+	})
+	if err != nil {
+		return
+	}
+
+	doc := bleve.NewDocumentMapping()
+	tm := bleve.NewTextFieldMapping()
+	tm.Analyzer = "fulltext_ngram"
+	doc.AddFieldMappingsAt("text", tm)
+	km := bleve.NewTextFieldMapping()
+	doc.AddFieldMappingsAt("keyword", km)
+	cm := bleve.NewDateTimeFieldMapping()
+	doc.AddFieldMappingsAt("created", cm)
+
+	m.AddDocumentMapping(docType, doc)
+	return
+}