From e47a874711b9dcf76b0a620461e31c152353bc61 Mon Sep 17 00:00:00 2001 From: ston1th Date: Mon, 9 Sep 2019 08:27:51 +0200 Subject: [PATCH] initial db refactoring --- pkg/db/index.go | 151 +++++++++++++++++++++++++++-------------- pkg/store/boltstore.go | 79 ++++++++++++++++++++- pkg/store/helper.go | 5 ++ pkg/store/kvstore.go | 11 +++ pkg/store/store.go | 7 +- 5 files changed, 194 insertions(+), 59 deletions(-) create mode 100644 pkg/store/kvstore.go diff --git a/pkg/db/index.go b/pkg/db/index.go index 5feec42..1ed216c 100644 --- a/pkg/db/index.go +++ b/pkg/db/index.go @@ -16,82 +16,131 @@ const ( ) func (db *DB) GetPaths(res core.Results) { - for i, v := range res { - err := db.store.Get(pathPrefix+v.ID, &res[i].Path) - if err != nil { - res[i].Path = err.Error() + db.store.Tx(false, func(tx store.KVStore) error { + for i, v := range res { + err := tx.Get(pathPrefix+v.ID, &res[i].Path) + if err != nil { + res[i].Path = err.Error() + } } - } + }) } func (db *DB) GetIndexed(p core.Paths) { - for i, v := range p { - p[i].Flag = db.IsIndexed(v.Abs) - p[i].Tags, _ = db.GetTag(v.Abs) + db.store.Tx(false, func(tx store.KVStore) error { + for i, v := range p { + p[i].Flag = db.IsIndexed(tx, v.Abs) + p[i].Tags, _ = db.GetTag(tx, v.Abs) + } + }) +} + +func (db *DB) IsIndexed(tx store.KVStore, path string) bool { + f := func(tx store.KVStore) error { + return tx.Get(idPrefix+path, nil) } + if tx != nil { + return f(tx) == nil + } + return f(db.store) == nil } -func (db *DB) IsIndexed(path string) bool { - return db.store.Get(idPrefix+path, nil) == nil -} - -func (db *DB) GetPath(id string) (path string, err error) { - err = db.store.Get(pathPrefix+id, &path) +func (db *DB) GetPath(tx store.KVStore, id string) (path string, err error) { + f := func(tx store.KVStore) error { + return tx.Get(pathPrefix+id, &path) + } + if tx != nil { + err = f(tx) + return + } + err = f(db.store) return } -func (db *DB) GetID(path string) (id string, err error) { - err = db.store.Get(idPrefix+path, &id) +func (db *DB) GetID(tx store.KVStore, path string) (id string, err error) { + f := func(tx store.KVStore) error { + return tx.Get(idPrefix+path, &id) + } + if tx != nil { + err = f(tx) + return + } + err = f(db.store) return } -func (db *DB) DeletePath(id string) error { - return db.store.Delete(pathPrefix + id) +func (db *DB) DeletePath(tx store.KVStore, id string) error { + f := func(tx store.KVStore) error { + return tx.Delete(pathPrefix + id) + } + if tx != nil { + return f(tx) + } + return f(db.store) } -func (db *DB) DeleteID(path string) error { - return db.store.Delete(idPrefix + path) +func (db *DB) DeleteID(tx store.KVStore, path string) error { + f := func(tx store.KVStore) error { + return tx.Delete(idPrefix + path) + } + if tx != nil { + return f(tx) + } + return f(db.store) } -func (db *DB) NewFile(id, path string, tag []string) (err error) { - err = db.store.Set(idPrefix+path, id) - if err != nil { - return +func (db *DB) NewFile(tx store.KVStore, id, path string, tag []string) (err error) { + f := func(tx store.KVStore) error { + err = tx.Set(idPrefix+path, id) + if err != nil { + return + } + err = tx.Set(pathPrefix+id, path) + if err != nil { + return + } + return tx.Set(tagPrefix+path, tag) } - err = db.store.Set(pathPrefix+id, path) - if err != nil { - return + if tx != nil { + return f(tx) } - return db.SetTag(path, tag) + return db.store.Tx(true, f) } func (db *DB) MoveFile(op, np string) (err error) { - if !db.IsIndexed(op) { - return nil - } - id, tag, err := db.DeleteFile(op) - if err != nil { - return - } - return db.NewFile(id, np, tag) + return db.store.Tx(true, func(tx store.KVStore) error { + id, tag, err := db.DeleteFile(tx, op) + if err != nil { + return + } + return db.NewFile(tx, id, np, tag) + }) } -func (db *DB) DeleteFile(path string) (id string, tag []string, err error) { - if !db.IsIndexed(path) { +func (db *DB) DeleteFile(tx store.KVStore, path string) (id string, tags []string, err error) { + f := func(tx store.KVStore) error { + err = tx.Get(idPrefix+path, &id) + if err != nil { + return + } + err = tx.Delete(idPrefix + path) + if err != nil { + return + } + err = tx.Get(tagPrefix+path, &tags) + if err != nil { + return + } + err = tx.Delete(tagPrefix + path) + if err != nil { + return + } + return tx.Delete(pathPrefix + id) + } + if tx != nil { + err = f(tx) return } - id, err = db.GetID(path) - if err != nil { - return - } - err = db.DeleteID(path) - if err != nil { - return - } - tag, err = db.UnsetTag(path) - if err != nil { - return - } - err = db.DeletePath(id) + err = db.store.Tx(true, f) return } diff --git a/pkg/store/boltstore.go b/pkg/store/boltstore.go index 4d9c006..0c36c51 100644 --- a/pkg/store/boltstore.go +++ b/pkg/store/boltstore.go @@ -84,6 +84,19 @@ func (bs *BoltStore) Restore(r io.Reader) (err error) { }) } +func (bs *BoltStore) Tx(writeable bool, f func(tx KVStore) error) error { + tx, err := db.db.Begin(writeable) + if err != nil { + return err + } + err = f(&BoltStoreTx{tx, bs}) + if err != nil { + tx.Rollback() + return err + } + return tx.Commit() +} + func (bs *BoltStore) Get(key string, v interface{}) (err error) { err = bs.db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(defaultBoltBucket)).Get([]byte(key)) @@ -120,12 +133,23 @@ func (bs *BoltStore) ForEach(f func(string, []byte) error) error { } func (bs *BoltStore) ForEachPrefix(prefix string, f func(string, []byte) error) error { - return bs.ForEach(func(k string, v []byte) error { - if trim, ok := hasTrimPrefix(k, prefix); ok { - return f(trim, v) + return bs.db.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(defaultBoltBucket)).Cursor() + p := []byte(prefix) + for k, v := c.Seek(p); k != nil && bytes.HasPrefix(k, p); k, v = c.Next() { + err := f(trimPrefix(k, p), v) + if err != nil { + return err + } } return nil }) + //return bs.ForEach(func(k string, v []byte) error { + // if trim, ok := hasTrimPrefix(k, prefix); ok { + // return f(trim, v) + // } + // return nil + //}) } func (bs *BoltStore) Delete(key string) error { @@ -137,3 +161,52 @@ func (bs *BoltStore) Delete(key string) error { func (bs *BoltStore) Close() error { return bs.db.Close() } + +type BoltStoreTx struct { + tx *bolt.Tx + bs *BoltStore +} + +func (tx *BoltStoreTx) Get(key string, v interface{}) (err error) { + b := tx.tx.Bucket([]byte(defaultBoltBucket)).Get([]byte(key)) + if b == nil { + return ErrKeyNotFound + } + if v != nil { + return tx.bs.Unmarshal(b, v) + } + return nil +} + +func (tx *BoltStoreTx) Set(key string, v interface{}) error { + var b []byte + if v != nil { + b, err = tx.bs.Marshal(v) + if err != nil { + return + } + } + return tx.tx.Bucket([]byte(defaultBoltBucket)).Put([]byte(key), b) +} + +func (tx *BoltStoreTx) ForEach(f func(string, []byte) error) error { + return tx.tx.Bucket([]byte(defaultBoltBucket)).ForEach(func(k, v []byte) error { + return f(string(k), v) + }) +} + +func (tx *BoltStoreTx) ForEachPrefix(prefix string, f func(string, []byte) error) error { + c := tx.tx.Bucket([]byte(defaultBoltBucket)).Cursor() + p := []byte(prefix) + for k, v := c.Seek(p); k != nil && bytes.HasPrefix(k, p); k, v = c.Next() { + err := f(trimPrefix(k, p), v) + if err != nil { + return err + } + } + return nil +} + +func (tx *BoltStoreTx) Delete(key string) error { + return tx.tx.Bucket([]byte(defaultBoltBucket)).Delete([]byte(key)) +} diff --git a/pkg/store/helper.go b/pkg/store/helper.go index db17a72..5d956b0 100644 --- a/pkg/store/helper.go +++ b/pkg/store/helper.go @@ -9,3 +9,8 @@ func hasTrimPrefix(s, prefix string) (string, bool) { } return s, false } + +// Parts taken from bytes.TrimPrefix +func trimPrefix(b, prefix []byte) string { + return string(b[len(prefix):]) +} diff --git a/pkg/store/kvstore.go b/pkg/store/kvstore.go new file mode 100644 index 0000000..c17a4fd --- /dev/null +++ b/pkg/store/kvstore.go @@ -0,0 +1,11 @@ +// Copyright (C) 2019 Marius Schellenberger + +package store + +type KVStore interface { + Get(string, interface{}) error + Set(string, interface{}) error + ForEach(func(string, []byte) error) error + ForEachPrefix(string, func(string, []byte) error) error + Delete(string) error +} diff --git a/pkg/store/store.go b/pkg/store/store.go index 4a5f90b..9421c86 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -14,10 +14,7 @@ type Store interface { Marshaler Dumper Restorer - Get(string, interface{}) error - Set(string, interface{}) error - ForEach(func(string, []byte) error) error - ForEachPrefix(string, func(string, []byte) error) error - Delete(string) error + KVStore + Tx(writeable bool, f func(tx KVStore) error) error Close() error }