diff --git a/pkg/core/types.go b/pkg/core/types.go index 03b6e15..7ed1290 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -12,9 +12,10 @@ type User struct { } type Result struct { - ID string - Path string - HTML template.HTML + ID string + Path string + Keyword string + HTML template.HTML } type Path struct { diff --git a/pkg/db/index.go b/pkg/db/index.go index 6e8a890..5b2955c 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -59,6 +59,9 @@ func (db *DB) NewFile(id, path string) (err error) { } func (db *DB) MoveFile(op, np string) error { + if !db.IsIndexed(op) { + return nil + } id, err := db.DeleteFile(op) if err != nil { return err diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index d082f7a..33c68ee 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -25,12 +25,17 @@ type Filesystem struct { } func NewFilesystem(base string) (fs *Filesystem, err error) { + if !filepath.IsAbs(base) { + err = errors.New("base dir '" + base + "' is not an absolute path") + return + } fi, err := os.Stat(base) if err != nil { return } if !fi.IsDir() { err = errors.New("base dir '" + base + "' is not a directory") + return } fs = &Filesystem{base} return @@ -68,7 +73,7 @@ func (fs *Filesystem) ReadDir(path string) (d *Directory) { } func (fs *Filesystem) Recursive(p string) (paths core.Paths) { - p = Clean(p) + p, _ = filepath.Split(Clean(p)) if len(p) >= 2 { p = p[:len(p)-1] } diff --git a/pkg/index/classify.go b/pkg/index/classify.go new file mode 100644 index 0000000..df5e248 --- /dev/null +++ b/pkg/index/classify.go @@ -0,0 +1,30 @@ +package index + +import ( + "regexp" +) + +var classes = []class{ + {regexp.MustCompile(`(?i)[^e]rechnung|quittung|beleg|invoice|bill|check`), []string{"invoice", "rechnung"}}, + {regexp.MustCompile(`(?i)steuer|lohn|tax|salary`), []string{"tax", "steuer"}}, + {regexp.MustCompile(`(?i)vertrag|kündigung|vereinbarung|contract|termination|agreement`), []string{"contract", "vertrag"}}, + {regexp.MustCompile(`(?i)versicherung|versichert|insurance|insured`), []string{"insurance", "versicherung"}}, +} + +type class struct { + r *regexp.Regexp + k []string +} + +func classify(text string) (keyword []string) { + keyword = []string{"", ""} + n := 0 + for _, v := range classes { + num := len(v.r.FindAllString(text, -1)) + if num > n && num >= 2 { + keyword = v.k + n = num + } + } + return +} diff --git a/pkg/index/index.go b/pkg/index/index.go index db47e9d..80613a5 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -7,16 +7,25 @@ import ( "git.giftfish.de/ston1th/docstore/pkg/core" "git.giftfish.de/ston1th/godrop/v2" "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" "html/template" "os" + "strings" + "time" ) const maxSearchResult = 101 -type indexDocument struct { - Text string `json:"text"` - Date string `json:"date"` - Type string `json:"type"` +const docType = "document" + +type document struct { + Text string `json:"text"` + Keyword []string `json:"keyword"` + Created time.Time `json:"created"` +} + +func (d *document) Type() string { + return docType } type Index struct { @@ -30,8 +39,10 @@ func NewIndex(path string) (i *Index, err error) { return } if _, err = os.Stat(path); os.IsNotExist(err) { - mapping := bleve.NewIndexMapping() - bi, err = bleve.New(path, mapping) + var m *mapping.IndexMappingImpl + m, err = createMapping() + //m := bleve.NewIndexMapping() + bi, err = bleve.New(path, m) if err != nil { err = errors.New("index: " + err.Error()) return @@ -51,7 +62,7 @@ func (i *Index) Close() error { return i.i.Close() } -func (i *Index) Add(text, docType string) (id string, err error) { +func (i *Index) Add(text string) (id string, err error) { id = newID() for { doc, _ := i.i.Document(id) @@ -60,7 +71,8 @@ func (i *Index) Add(text, docType string) (id string, err error) { } id = newID() } - err = i.i.Index(id, indexDocument{text, core.Now(), docType}) + k := classify(text) + err = i.i.Index(id, &document{text, k, time.Now()}) return } @@ -86,12 +98,20 @@ func (i *Index) Search(search string) (results []core.Result, err error) { continue } for fragField, frags := range hit.Fragments { - if fragField == "text" || fragField == "type" || fragField == "date" { + if fragField == "text" || fragField == "created" { for _, f := range frags { r.HTML += template.HTML(f) } break } + if fragField == "keyword" { + var k string + for _, f := range frags { + k += f + } + r.Keyword = strings.ReplaceAll(strings.ReplaceAll(k, "", ""), "", "") + break + } } results = append(results, r) } diff --git a/pkg/index/mapping.go b/pkg/index/mapping.go new file mode 100644 index 0000000..9c5d772 --- /dev/null +++ b/pkg/index/mapping.go @@ -0,0 +1,47 @@ +package index + +import ( + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/analysis/analyzer/custom" + "github.com/blevesearch/bleve/analysis/token/edgengram" + "github.com/blevesearch/bleve/analysis/token/lowercase" + "github.com/blevesearch/bleve/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/mapping" +) + +func createMapping() (m *mapping.IndexMappingImpl, err error) { + m = bleve.NewIndexMapping() + + err = m.AddCustomTokenFilter("bigram_tokenfilter", map[string]interface{}{ + "type": edgengram.Name, + "side": edgengram.FRONT, + "min": 3.0, + "max": 25.0, + }) + if err != nil { + return + } + err = m.AddCustomAnalyzer("fulltext_ngram", map[string]interface{}{ + "type": custom.Name, + "tokenizer": unicode.Name, + "token_filters": []string{ + lowercase.Name, + "bigram_tokenfilter", + }, + }) + if err != nil { + return + } + + doc := bleve.NewDocumentMapping() + tm := bleve.NewTextFieldMapping() + tm.Analyzer = "fulltext_ngram" + doc.AddFieldMappingsAt("text", tm) + km := bleve.NewTextFieldMapping() + doc.AddFieldMappingsAt("keyword", km) + cm := bleve.NewDateTimeFieldMapping() + doc.AddFieldMappingsAt("created", cm) + + m.AddDocumentMapping(docType, doc) + return +} diff --git a/pkg/server/handler.go b/pkg/server/handler.go index a7e4d6d..c4f23e6 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -189,7 +189,7 @@ func indexHandler(ctx *Context) { } return } - id, err := ctx.Srv.DB.Index.Add(txt, "pdf") + id, err := ctx.Srv.DB.Index.Add(txt) if err != nil { log.Printf("index: %s: %s", path, err) return diff --git a/pkg/server/templates.go b/pkg/server/templates.go index 7eb1b8d..efa101f 100644 --- a/pkg/server/templates.go +++ b/pkg/server/templates.go @@ -365,7 +365,7 @@ const (
| File | +Keyword | +
|---|---|
{{$item.Path}}
- {{$item.HTML}}+ {{if $item.HTML}} + {{$item.HTML}}
{{end}}
- {{else}}
- No search results found.
- {{end}}
-
+ |
+ {{if $item.Keyword}}{{$item.Keyword}}{{end}} | +
| File | +Keyword | +
|---|---|
{{$item.Path}}
- {{$item.HTML}}+ {{if $item.HTML}} + {{$item.HTML}}
{{end}}
- {{else}}
- No search results found.
- {{end}}
-
+ |
+ {{if $item.Keyword}}{{$item.Keyword}}{{end}} | +