bugfixes more search results and dump restore options

vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go

@@ -18,22 +18,28 @@
 package boltdb
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/blevesearch/bleve/index/store"
 	"github.com/blevesearch/bleve/registry"
 	"github.com/boltdb/bolt"
 )
 
-const Name = "boltdb"
+const (
+	Name                    = "boltdb"
+	defaultCompactBatchSize = 100
+)
 
 type Store struct {
-	path   string
-	bucket string
-	db     *bolt.DB
-	noSync bool
-	mo     store.MergeOperator
+	path        string
+	bucket      string
+	db          *bolt.DB
+	noSync      bool
+	fillPercent float64
+	mo          store.MergeOperator
 }
 
 func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
@@ -41,6 +47,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 	if !ok {
 		return nil, fmt.Errorf("must specify path")
 	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
 
 	bucket, ok := config["bucket"].(string)
 	if !ok {
@@ -49,27 +58,41 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 
 	noSync, _ := config["nosync"].(bool)
 
-	db, err := bolt.Open(path, 0600, nil)
+	fillPercent, ok := config["fillPercent"].(float64)
+	if !ok {
+		fillPercent = bolt.DefaultFillPercent
+	}
+
+	bo := &bolt.Options{}
+	ro, ok := config["read_only"].(bool)
+	if ok {
+		bo.ReadOnly = ro
+	}
+
+	db, err := bolt.Open(path, 0600, bo)
 	if err != nil {
 		return nil, err
 	}
 	db.NoSync = noSync
 
-	err = db.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+	if !bo.ReadOnly {
+		err = db.Update(func(tx *bolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists([]byte(bucket))
 
-		return err
-	})
-	if err != nil {
-		return nil, err
+			return err
+		})
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	rv := Store{
-		path:   path,
-		bucket: bucket,
-		db:     db,
-		mo:     mo,
-		noSync: noSync,
+		path:        path,
+		bucket:      bucket,
+		db:          db,
+		mo:          mo,
+		noSync:      noSync,
+		fillPercent: fillPercent,
 	}
 	return &rv, nil
 }
@@ -102,6 +125,46 @@ func (bs *Store) Stats() json.Marshaler {
 	}
 }
 
+// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
+// Removing entries is a workaround for github issue #374.
+func (bs *Store) CompactWithBatchSize(batchSize int) error {
+	for {
+		cnt := 0
+		err := bs.db.Batch(func(tx *bolt.Tx) error {
+			c := tx.Bucket([]byte(bs.bucket)).Cursor()
+			prefix := []byte("d")
+
+			for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+				if bytes.Equal(v, []byte{0}) {
+					cnt++
+					if err := c.Delete(); err != nil {
+						return err
+					}
+					if cnt == batchSize {
+						break
+					}
+				}
+
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if cnt == 0 {
+			break
+		}
+	}
+	return nil
+}
+
+// Compact calls CompactWithBatchSize with a default batch size of 100.  This is a workaround
+// for github issue #374.
+func (bs *Store) Compact() error {
+	return bs.CompactWithBatchSize(defaultCompactBatchSize)
+}
+
 func init() {
 	registry.RegisterKVStore(Name, New)
 }

vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go

@@ -27,7 +27,7 @@ func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch
 	return make([]byte, options.TotalBytes), w.NewBatch(), nil
 }
 
-func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
+func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {
 
 	emulatedBatch, ok := batch.(*store.EmulatedBatch)
 	if !ok {
@@ -36,39 +36,53 @@ func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
 
 	tx, err := w.store.db.Begin(true)
 	if err != nil {
-		return err
+		return
 	}
+	// defer function to ensure that once started,
+	// we either Commit tx or Rollback
+	defer func() {
+		// if nothing went wrong, commit
+		if err == nil {
+			// careful to catch error here too
+			err = tx.Commit()
+		} else {
+			// caller should see error that caused abort,
+			// not success or failure of Rollback itself
+			_ = tx.Rollback()
+		}
+	}()
 
 	bucket := tx.Bucket([]byte(w.store.bucket))
+	bucket.FillPercent = w.store.fillPercent
 
 	for k, mergeOps := range emulatedBatch.Merger.Merges {
 		kb := []byte(k)
 		existingVal := bucket.Get(kb)
 		mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
 		if !fullMergeOk {
-			return fmt.Errorf("merge operator returned failure")
+			err = fmt.Errorf("merge operator returned failure")
+			return
 		}
 		err = bucket.Put(kb, mergedVal)
 		if err != nil {
-			return err
+			return
 		}
 	}
 
 	for _, op := range emulatedBatch.Ops {
 		if op.V != nil {
-			err := bucket.Put(op.K, op.V)
+			err = bucket.Put(op.K, op.V)
 			if err != nil {
-				return err
+				return
 			}
 		} else {
-			err := bucket.Delete(op.K)
+			err = bucket.Delete(op.K)
 			if err != nil {
-				return err
+				return
 			}
 		}
 	}
-
-	return tx.Commit()
+	return
 }
 
 func (w *Writer) Close() error {

vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go

@@ -17,6 +17,8 @@ package gtreap
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"sync"
 
 	"github.com/blevesearch/bleve/index/store"
@@ -42,6 +44,14 @@ func itemCompare(a, b interface{}) int {
 }
 
 func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path != "" {
+		return nil, os.ErrInvalid
+	}
+
 	rv := Store{
 		t:  gtreap.NewTreap(itemCompare),
 		mo: mo,