better indexing and classifier

pkg/index/mapping.go

@@ -0,0 +1,47 @@
+package index
+
+import (
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/analysis/analyzer/custom"
+	"github.com/blevesearch/bleve/analysis/token/edgengram"
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+func createMapping() (m *mapping.IndexMappingImpl, err error) {
+	m = bleve.NewIndexMapping()
+
+	err = m.AddCustomTokenFilter("bigram_tokenfilter", map[string]interface{}{
+		"type": edgengram.Name,
+		"side": edgengram.FRONT,
+		"min":  3.0,
+		"max":  25.0,
+	})
+	if err != nil {
+		return
+	}
+	err = m.AddCustomAnalyzer("fulltext_ngram", map[string]interface{}{
+		"type":      custom.Name,
+		"tokenizer": unicode.Name,
+		"token_filters": []string{
+			lowercase.Name,
+			"bigram_tokenfilter",
+		},
+	})
+	if err != nil {
+		return
+	}
+
+	doc := bleve.NewDocumentMapping()
+	tm := bleve.NewTextFieldMapping()
+	tm.Analyzer = "fulltext_ngram"
+	doc.AddFieldMappingsAt("text", tm)
+	km := bleve.NewTextFieldMapping()
+	doc.AddFieldMappingsAt("keyword", km)
+	cm := bleve.NewDateTimeFieldMapping()
+	doc.AddFieldMappingsAt("created", cm)
+
+	m.AddDocumentMapping(docType, doc)
+	return
+}