ston1th/docstore
1
1
Fork 0
Code Issues 3 Releases Activity

made indexAll and retagAll atomic and faster

Browse source
This commit is contained in:
ston1th 2019-08-27 21:36:35 +02:00
commit 5649acf6a8
6 changed files with 131 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pkg/db/db.go
View file

@ -18,6 +18,8 @@ const (
const blevePath = "bleve"
type DB struct {
rm countMutex
IndexMutex countMutex
store store.Store
Index *index.Index
}

12
pkg/db/helper.go
View file

@ -27,3 +27,15 @@ func validatePassword(pw string) (b []byte) {
b = hash.Sum(nil)
return
}
func eq(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}

55
pkg/db/index.go
View file

@ -2,11 +2,7 @@
package db
import (
"errors"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
)
import "git.giftfish.de/ston1th/docstore/pkg/core"
const (
// schema:
@ -19,55 +15,6 @@ const (
pathPrefix = "path/"
)
func (db *DB) Retag(path string, tags core.RTags) (err error) {
if tags == nil {
tags, err = db.GetAllRTags()
if err != nil {
return
}
}
id, err := db.GetID(path)
if err != nil {
return
}
log.Debugf("db: retagging: %s", path)
doc, err := db.Index.Get(id)
if err != nil {
return
}
doc.Tags = tags.Match(doc.Text)
err = db.Index.Update(id, doc)
if err != nil {
return
}
return db.SetTag(path, doc.Tags)
}
func (db *DB) RetagAll() (err error) {
var paths []string
err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
paths = append(paths, k)
return nil
})
if err != nil {
return
}
tags, err := db.GetAllRTags()
if err != nil {
return
}
for _, p := range paths {
e := db.Retag(p, tags)
if e != nil {
if err == nil {
err = errors.New("retagAll had errors")
}
log.Printf("db: error retagging %s: %s", p, e)
}
}
return
}
func (db *DB) GetPaths(res core.Results) {
for i, v := range res {
err := db.store.Get(pathPrefix+v.ID, &res[i].Path)

32
pkg/db/mutex.go Normal file
View file

@ -0,0 +1,32 @@
// Copyright (C) 2019 Marius Schellenberger
package db
import "sync/atomic"
type countMutex struct {
mu uint64
max uint64
c uint64
}
func (cm *countMutex) Lock() (uint64, uint64, bool) {
if atomic.CompareAndSwapUint64(&cm.mu, 0, 1) {
return 0, 0, true
}
return atomic.LoadUint64(&cm.max), atomic.LoadUint64(&cm.c), false
}
func (cm *countMutex) Set(max uint64) {
atomic.StoreUint64(&cm.max, max)
}
func (cm *countMutex) Inc() {
atomic.AddUint64(&cm.c, 1)
}
func (cm *countMutex) Unlock() {
atomic.StoreUint64(&cm.max, 0)
atomic.StoreUint64(&cm.c, 0)
atomic.StoreUint64(&cm.mu, 0)
}

66
pkg/db/tags.go
View file

@ -3,7 +3,10 @@
package db
import (
"errors"
"fmt"
"git.giftfish.de/ston1th/docstore/pkg/core"
"git.giftfish.de/ston1th/docstore/pkg/log"
"regexp"
"sort"
)
@ -30,6 +33,69 @@ func (db *DB) addDefaultTags() (err error) {
return
}
func (db *DB) Retag(path string, tags core.RTags) (err error) {
if tags == nil {
tags, err = db.GetAllRTags()
if err != nil {
return
}
}
id, err := db.GetID(path)
if err != nil {
return
}
t, err := db.GetTag(path)
if err != nil {
return
}
log.Debugf("db: retagging: %s", path)
doc, err := db.Index.Get(id)
if err != nil {
return
}
doc.Tags = tags.Match(doc.Text)
if eq(t, doc.Tags) {
log.Debugf("db: retagging: skipped %s", path)
return
}
err = db.Index.Update(id, doc)
if err != nil {
return
}
return db.SetTag(path, doc.Tags)
}
func (db *DB) RetagAll() (err error) {
if m, c, ok := db.rm.Lock(); !ok {
return fmt.Errorf("retagging already in progress: %d/%d", c, m)
}
defer db.rm.Unlock()
var paths []string
err = db.store.ForEachPrefix(idPrefix, func(k string, _ []byte) error {
paths = append(paths, k)
return nil
})
if err != nil {
return
}
tags, err := db.GetAllRTags()
if err != nil {
return
}
db.rm.Set(uint64(len(paths)))
for _, p := range paths {
e := db.Retag(p, tags)
if e != nil {
if err == nil {
err = errors.New("retagAll had errors")
}
log.Printf("db: error retagging %s: %s", p, e)
}
db.rm.Inc()
}
return
}
func (db *DB) GetAllTags() (tags core.Tags, err error) {
err = db.store.ForEachPrefix(tagsPrefix, func(k string, _ []byte) error {
var t core.Tag

14
pkg/server/handler.go
View file

@ -247,13 +247,21 @@ func indexHandler(ctx *Context) {
ctx.Redirect(core.IndexURI, http.StatusFound)
return
}
if m, c, ok := ctx.Srv.DB.IndexMutex.Lock(); ok {
go func() {
for _, f := range ctx.Srv.FS.RecursiveFiles("/") {
defer ctx.Srv.DB.IndexMutex.Unlock()
files := ctx.Srv.FS.RecursiveFiles("/")
ctx.Srv.DB.IndexMutex.Set(uint64(len(files)))
for _, f := range files {
if !ctx.Srv.DB.IsIndexed(f) {
scanFile(ctx.Srv, f)
}
ctx.Srv.DB.IndexMutex.Inc()
}
}()
} else {
ctx.Srv.Log.Printf("indexAll: indexing already in progress: %d/%d", c, m)
}
ctx.Redirect(core.IndexURI, http.StatusFound)
}
}
@ -726,8 +734,8 @@ func userTotpHandler(ctx *Context) {
func logsHandler(ctx *Context) {
ctx.Template("logsHandler")
ctx.Data = webData{
Title: "Scan Logs",
BodyTitle: "Scan Logs",
Title: "Logs",
BodyTitle: "Logs",
}
switch ctx.Method() {
case "GET":