made indexAll and retagAll atomic and faster

pkg/db/db.go

@@ -18,6 +18,8 @@ const (
 const blevePath = "bleve"
 
 type DB struct {
+	rm         countMutex
+	IndexMutex countMutex
 	store      store.Store
 	Index      *index.Index
 }

pkg/db/helper.go

@@ -27,3 +27,15 @@ func validatePassword(pw string) (b []byte) {
 	b = hash.Sum(nil)
 	return
 }
+
+func eq(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}

pkg/db/index.go

@@ -2,11 +2,7 @@
 
 package db
 
-import (
+import "git.giftfish.de/ston1th/docstore/pkg/core"
-	"errors"
-	"git.giftfish.de/ston1th/docstore/pkg/core"
-	"git.giftfish.de/ston1th/docstore/pkg/log"
-)
 
 const (
 	// schema:
@@ -19,55 +15,6 @@ const (
 	pathPrefix = "path/"
 )
 
-func (db *DB) Retag(path string, tags core.RTags) (err error) {
-	if tags == nil {
-		tags, err = db.GetAllRTags()
-		if err != nil {
-			return
-		}
-	}
-	id, err := db.GetID(path)
-	if err != nil {
-		return
-	}
-	log.Debugf("db: retagging: %s", path)
-	doc, err := db.Index.Get(id)
-	if err != nil {
-		return
-	}
-	doc.Tags = tags.Match(doc.Text)
-	err = db.Index.Update(id, doc)
-	if err != nil {
-		return
-	}
-	return db.SetTag(path, doc.Tags)
-}
-
-func (db *DB) RetagAll() (err error) {
-	var paths []string
-	err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
-		paths = append(paths, k)
-		return nil
-	})
-	if err != nil {
-		return
-	}
-	tags, err := db.GetAllRTags()
-	if err != nil {
-		return
-	}
-	for _, p := range paths {
-		e := db.Retag(p, tags)
-		if e != nil {
-			if err == nil {
-				err = errors.New("retagAll had errors")
-			}
-			log.Printf("db: error retagging %s: %s", p, e)
-		}
-	}
-	return
-}
-
 func (db *DB) GetPaths(res core.Results) {
 	for i, v := range res {
 		err := db.store.Get(pathPrefix+v.ID, &res[i].Path)

pkg/db/mutex.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2019 Marius Schellenberger
+
+package db
+
+import "sync/atomic"
+
+type countMutex struct {
+	mu  uint64
+	max uint64
+	c   uint64
+}
+
+func (cm *countMutex) Lock() (uint64, uint64, bool) {
+	if atomic.CompareAndSwapUint64(&cm.mu, 0, 1) {
+		return 0, 0, true
+	}
+	return atomic.LoadUint64(&cm.max), atomic.LoadUint64(&cm.c), false
+}
+
+func (cm *countMutex) Set(max uint64) {
+	atomic.StoreUint64(&cm.max, max)
+}
+
+func (cm *countMutex) Inc() {
+	atomic.AddUint64(&cm.c, 1)
+}
+
+func (cm *countMutex) Unlock() {
+	atomic.StoreUint64(&cm.max, 0)
+	atomic.StoreUint64(&cm.c, 0)
+	atomic.StoreUint64(&cm.mu, 0)
+}

pkg/db/tags.go

@@ -3,7 +3,10 @@
 package db
 
 import (
+	"errors"
+	"fmt"
 	"git.giftfish.de/ston1th/docstore/pkg/core"
+	"git.giftfish.de/ston1th/docstore/pkg/log"
 	"regexp"
 	"sort"
 )
@@ -30,6 +33,69 @@ func (db *DB) addDefaultTags() (err error) {
 	return
 }
 
+func (db *DB) Retag(path string, tags core.RTags) (err error) {
+	if tags == nil {
+		tags, err = db.GetAllRTags()
+		if err != nil {
+			return
+		}
+	}
+	id, err := db.GetID(path)
+	if err != nil {
+		return
+	}
+	t, err := db.GetTag(path)
+	if err != nil {
+		return
+	}
+	log.Debugf("db: retagging: %s", path)
+	doc, err := db.Index.Get(id)
+	if err != nil {
+		return
+	}
+	doc.Tags = tags.Match(doc.Text)
+	if eq(t, doc.Tags) {
+		log.Debugf("db: retagging: skipped %s", path)
+		return
+	}
+	err = db.Index.Update(id, doc)
+	if err != nil {
+		return
+	}
+	return db.SetTag(path, doc.Tags)
+}
+
+func (db *DB) RetagAll() (err error) {
+	if m, c, ok := db.rm.Lock(); !ok {
+		return fmt.Errorf("retagging already in progress: %d/%d", c, m)
+	}
+	defer db.rm.Unlock()
+	var paths []string
+	err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
+		paths = append(paths, k)
+		return nil
+	})
+	if err != nil {
+		return
+	}
+	tags, err := db.GetAllRTags()
+	if err != nil {
+		return
+	}
+	db.rm.Set(uint64(len(paths)))
+	for _, p := range paths {
+		e := db.Retag(p, tags)
+		if e != nil {
+			if err == nil {
+				err = errors.New("retagAll had errors")
+			}
+			log.Printf("db: error retagging %s: %s", p, e)
+		}
+		db.rm.Inc()
+	}
+	return
+}
+
 func (db *DB) GetAllTags() (tags core.Tags, err error) {
 	err = db.store.ForEachPrefix(tagsPrefix, func(k string, _ []byte) error {
 		var t core.Tag

pkg/server/handler.go

@@ -247,13 +247,21 @@ func indexHandler(ctx *Context) {
 			ctx.Redirect(core.IndexURI, http.StatusFound)
 			return
 		}
+		if m, c, ok := ctx.Srv.DB.IndexMutex.Lock(); ok {
 			go func() {
-			for _, f := range ctx.Srv.FS.RecursiveFiles("/") {
+				defer ctx.Srv.DB.IndexMutex.Unlock()
+				files := ctx.Srv.FS.RecursiveFiles("/")
+				ctx.Srv.DB.IndexMutex.Set(uint64(len(files)))
+				for _, f := range files {
 					if !ctx.Srv.DB.IsIndexed(f) {
 						scanFile(ctx.Srv, f)
 					}
+					ctx.Srv.DB.IndexMutex.Inc()
 				}
 			}()
+		} else {
+			ctx.Srv.Log.Printf("indexAll: indexing already in progress: %d/%d", c, m)
+		}
 		ctx.Redirect(core.IndexURI, http.StatusFound)
 	}
 }
@@ -726,8 +734,8 @@ func userTotpHandler(ctx *Context) {
 func logsHandler(ctx *Context) {
 	ctx.Template("logsHandler")
 	ctx.Data = webData{
-		Title:     "Scan Logs",
+		Title:     "Logs",
-		BodyTitle: "Scan Logs",
+		BodyTitle: "Logs",
 	}
 	switch ctx.Method() {
 	case "GET":