From 5649acf6a85365d21c53ab6ee426c9f583772d17 Mon Sep 17 00:00:00 2001 From: ston1th Date: Tue, 27 Aug 2019 21:36:35 +0200 Subject: [PATCH] made indexAll and retagAll atomic and faster --- pkg/db/db.go | 6 ++-- pkg/db/helper.go | 12 ++++++++ pkg/db/index.go | 55 +----------------------------------- pkg/db/mutex.go | 32 +++++++++++++++++++++ pkg/db/tags.go | 66 +++++++++++++++++++++++++++++++++++++++++++ pkg/server/handler.go | 24 ++++++++++------ 6 files changed, 131 insertions(+), 64 deletions(-) create mode 100644 pkg/db/mutex.go diff --git a/pkg/db/db.go b/pkg/db/db.go index 8966941..b37b087 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -18,8 +18,10 @@ const ( const blevePath = "bleve" type DB struct { - store store.Store - Index *index.Index + rm countMutex + IndexMutex countMutex + store store.Store + Index *index.Index } func New(cfg core.Config) (db *DB, err error) { diff --git a/pkg/db/helper.go b/pkg/db/helper.go index 18a74ca..b931d92 100644 --- a/pkg/db/helper.go +++ b/pkg/db/helper.go @@ -27,3 +27,15 @@ func validatePassword(pw string) (b []byte) { b = hash.Sum(nil) return } + +func eq(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/pkg/db/index.go b/pkg/db/index.go index a6c2033..5feec42 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -2,11 +2,7 @@ package db -import ( - "errors" - "git.giftfish.de/ston1th/docstore/pkg/core" - "git.giftfish.de/ston1th/docstore/pkg/log" -) +import "git.giftfish.de/ston1th/docstore/pkg/core" const ( // schema: @@ -19,55 +15,6 @@ const ( pathPrefix = "path/" ) -func (db *DB) Retag(path string, tags core.RTags) (err error) { - if tags == nil { - tags, err = db.GetAllRTags() - if err != nil { - return - } - } - id, err := db.GetID(path) - if err != nil { - return - } - log.Debugf("db: retagging: %s", path) - doc, err := db.Index.Get(id) - if err != nil { - return - } - doc.Tags = tags.Match(doc.Text) - err = db.Index.Update(id, doc) - if err != nil { - return - } - return db.SetTag(path, doc.Tags) -} - -func (db *DB) RetagAll() (err error) { - var paths []string - err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error { - paths = append(paths, k) - return nil - }) - if err != nil { - return - } - tags, err := db.GetAllRTags() - if err != nil { - return - } - for _, p := range paths { - e := db.Retag(p, tags) - if e != nil { - if err == nil { - err = errors.New("retagAll had errors") - } - log.Printf("db: error retagging %s: %s", p, e) - } - } - return -} - func (db *DB) GetPaths(res core.Results) { for i, v := range res { err := db.store.Get(pathPrefix+v.ID, &res[i].Path) diff --git a/pkg/db/mutex.go b/pkg/db/mutex.go new file mode 100644 index 0000000..0742edd --- /dev/null +++ b/pkg/db/mutex.go @@ -0,0 +1,32 @@ +// Copyright (C) 2019 Marius Schellenberger + +package db + +import "sync/atomic" + +type countMutex struct { + mu uint64 + max uint64 + c uint64 +} + +func (cm *countMutex) Lock() (uint64, uint64, bool) { + if atomic.CompareAndSwapUint64(&cm.mu, 0, 1) { + return 0, 0, true + } + return atomic.LoadUint64(&cm.max), atomic.LoadUint64(&cm.c), false +} + +func (cm *countMutex) Set(max uint64) { + atomic.StoreUint64(&cm.max, max) +} + +func (cm *countMutex) Inc() { + atomic.AddUint64(&cm.c, 1) +} + +func (cm *countMutex) Unlock() { + atomic.StoreUint64(&cm.max, 0) + atomic.StoreUint64(&cm.c, 0) + atomic.StoreUint64(&cm.mu, 0) +} diff --git a/pkg/db/tags.go b/pkg/db/tags.go index 7c5e17f..7fd923e 100644 --- a/pkg/db/tags.go +++ b/pkg/db/tags.go @@ -3,7 +3,10 @@ package db import ( + "errors" + "fmt" "git.giftfish.de/ston1th/docstore/pkg/core" + "git.giftfish.de/ston1th/docstore/pkg/log" "regexp" "sort" ) @@ -30,6 +33,69 @@ func (db *DB) addDefaultTags() (err error) { return } +func (db *DB) Retag(path string, tags core.RTags) (err error) { + if tags == nil { + tags, err = db.GetAllRTags() + if err != nil { + return + } + } + id, err := db.GetID(path) + if err != nil { + return + } + t, err := db.GetTag(path) + if err != nil { + return + } + log.Debugf("db: retagging: %s", path) + doc, err := db.Index.Get(id) + if err != nil { + return + } + doc.Tags = tags.Match(doc.Text) + if eq(t, doc.Tags) { + log.Debugf("db: retagging: skipped %s", path) + return + } + err = db.Index.Update(id, doc) + if err != nil { + return + } + return db.SetTag(path, doc.Tags) +} + +func (db *DB) RetagAll() (err error) { + if m, c, ok := db.rm.Lock(); !ok { + return fmt.Errorf("retagging already in progress: %d/%d", c, m) + } + defer db.rm.Unlock() + var paths []string + err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error { + paths = append(paths, k) + return nil + }) + if err != nil { + return + } + tags, err := db.GetAllRTags() + if err != nil { + return + } + db.rm.Set(uint64(len(paths))) + for _, p := range paths { + e := db.Retag(p, tags) + if e != nil { + if err == nil { + err = errors.New("retagAll had errors") + } + log.Printf("db: error retagging %s: %s", p, e) + } + db.rm.Inc() + } + return +} + func (db *DB) GetAllTags() (tags core.Tags, err error) { err = db.store.ForEachPrefix(tagsPrefix, func(k string, _ []byte) error { var t core.Tag diff --git a/pkg/server/handler.go b/pkg/server/handler.go index 5856077..85f0589 100644 --- a/pkg/server/handler.go +++ b/pkg/server/handler.go @@ -247,13 +247,21 @@ func indexHandler(ctx *Context) { ctx.Redirect(core.IndexURI, http.StatusFound) return } - go func() { - for _, f := range ctx.Srv.FS.RecursiveFiles("/") { - if !ctx.Srv.DB.IsIndexed(f) { - scanFile(ctx.Srv, f) + if m, c, ok := ctx.Srv.DB.IndexMutex.Lock(); ok { + go func() { + defer ctx.Srv.DB.IndexMutex.Unlock() + files := ctx.Srv.FS.RecursiveFiles("/") + ctx.Srv.DB.IndexMutex.Set(uint64(len(files))) + for _, f := range files { + if !ctx.Srv.DB.IsIndexed(f) { + scanFile(ctx.Srv, f) + } + ctx.Srv.DB.IndexMutex.Inc() } - } - }() + }() + } else { + ctx.Srv.Log.Printf("indexAll: indexing already in progress: %d/%d", c, m) + } ctx.Redirect(core.IndexURI, http.StatusFound) } } @@ -726,8 +734,8 @@ func userTotpHandler(ctx *Context) { func logsHandler(ctx *Context) { ctx.Template("logsHandler") ctx.Data = webData{ - Title: "Scan Logs", - BodyTitle: "Scan Logs", + Title: "Logs", + BodyTitle: "Logs", } switch ctx.Method() { case "GET":