bugfixes more search results and dump restore options

vendor/github.com/blevesearch/bleve/index/index.go

@@ -10,9 +10,9 @@
 package index
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
-	"time"
 
 	"github.com/blevesearch/bleve/document"
 	"github.com/blevesearch/bleve/index/store"
@@ -24,8 +24,6 @@ type Index interface {
 	Open() error
 	Close() error
 
-	DocCount() (uint64, error)
-
 	Update(doc *document.Document) error
 	Delete(id string) error
 	Batch(batch *Batch) error
@@ -33,10 +31,6 @@ type Index interface {
 	SetInternal(key, val []byte) error
 	DeleteInternal(key []byte) error
 
-	DumpAll() chan interface{}
-	DumpDoc(id string) chan interface{}
-	DumpFields() chan interface{}
-
 	// Reader returns a low-level accessor on the index data. Close it to
 	// release associated resources.
 	Reader() (IndexReader, error)
@@ -49,25 +43,14 @@ type Index interface {
 	Advanced() (store.KVStore, error)
 }
 
-// AsyncIndex is an interface for indexes which perform
-// some important operations asynchronously.
-type AsyncIndex interface {
-	// Wait will block until asynchronous operations started
-	// before this call have finished or until the specified
-	// timeout has been reached.  If the timeout is reached
-	// an error is returned.
-	Wait(timeout time.Duration) error
-}
-
 type IndexReader interface {
-	TermFieldReader(term []byte, field string) (TermFieldReader, error)
+	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)
 
-	// DocIDReader returns an iterator over documents which identifiers are
-	// greater than or equal to start and smaller than end. Set start to the
-	// empty string to iterate from the first document, end to the empty string
-	// to iterate to the last one.
+	// DocIDReader returns an iterator over all doc ids
 	// The caller must close returned instance to release associated resources.
-	DocIDReader(start, end string) (DocIDReader, error)
+	DocIDReaderAll() (DocIDReader, error)
+
+	DocIDReaderOnly(ids []string) (DocIDReader, error)
 
 	FieldDict(field string) (FieldDict, error)
 
@@ -76,19 +59,47 @@ type IndexReader interface {
 	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
 
 	Document(id string) (*document.Document, error)
-	DocumentFieldTerms(id string) (FieldTerms, error)
+	DocumentFieldTerms(id IndexInternalID, fields []string) (FieldTerms, error)
 
 	Fields() ([]string, error)
 
 	GetInternal(key []byte) ([]byte, error)
 
-	DocCount() uint64
+	DocCount() (uint64, error)
+
+	ExternalID(id IndexInternalID) (string, error)
+	InternalID(id string) (IndexInternalID, error)
+
+	DumpAll() chan interface{}
+	DumpDoc(id string) chan interface{}
+	DumpFields() chan interface{}
 
 	Close() error
 }
 
+// FieldTerms contains the terms used by a document, keyed by field
 type FieldTerms map[string][]string
 
+// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields
+func (f FieldTerms) FieldsNotYetCached(fields []string) []string {
+	var rv []string
+	for _, field := range fields {
+		if _, ok := f[field]; !ok {
+			rv = append(rv, field)
+		}
+	}
+	return rv
+}
+
+// Merge will combine two FieldTerms
+// it assumes that the terms lists are complete (thus do not need to be merged)
+// field terms from the other list always replace the ones in the receiver
+func (f FieldTerms) Merge(other FieldTerms) {
+	for field, terms := range other {
+		f[field] = terms
+	}
+}
+
 type TermFieldVector struct {
 	Field          string
 	ArrayPositions []uint64
@@ -97,25 +108,48 @@ type TermFieldVector struct {
 	End            uint64
 }
 
+// IndexInternalID is an opaque document identifier interal to the index impl
+type IndexInternalID []byte
+
+func (id IndexInternalID) Equals(other IndexInternalID) bool {
+	return id.Compare(other) == 0
+}
+
+func (id IndexInternalID) Compare(other IndexInternalID) int {
+	return bytes.Compare(id, other)
+}
+
 type TermFieldDoc struct {
 	Term    string
-	ID      string
+	ID      IndexInternalID
 	Freq    uint64
 	Norm    float64
 	Vectors []*TermFieldVector
 }
 
+// Reset allows an already allocated TermFieldDoc to be reused
+func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
+	// remember the []byte used for the ID
+	id := tfd.ID
+	// idiom to copy over from empty TermFieldDoc (0 allocations)
+	*tfd = TermFieldDoc{}
+	// reuse the []byte already allocated (and reset len to 0)
+	tfd.ID = id[:0]
+	return tfd
+}
+
 // TermFieldReader is the interface exposing the enumeration of documents
 // containing a given term in a given field. Documents are returned in byte
 // lexicographic order over their identifiers.
 type TermFieldReader interface {
 	// Next returns the next document containing the term in this field, or nil
-	// when it reaches the end of the enumeration.
-	Next() (*TermFieldDoc, error)
+	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
+	// is optional, and when non-nil, will be used instead of allocating memory.
+	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)
 
 	// Advance resets the enumeration at specified document or its immediate
 	// follower.
-	Advance(ID string) (*TermFieldDoc, error)
+	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)
 
 	// Count returns the number of documents contains the term in this field.
 	Count() uint64
@@ -135,15 +169,15 @@ type FieldDict interface {
 // DocIDReader is the interface exposing enumeration of documents identifiers.
 // Close the reader to release associated resources.
 type DocIDReader interface {
-	// Next returns the next document identifier in ascending lexicographic
-	// byte order, or io.EOF when the end of the sequence is reached.
-	Next() (string, error)
+	// Next returns the next document internal identifier in the natural
+	// index order, or io.EOF when the end of the sequence is reached.
+	Next() (IndexInternalID, error)
 
-	// Advance resets the iteration to the first identifier greater than or
-	// equal to ID. If ID is smaller than the start of the range, the iteration
+	// Advance resets the iteration to the first internal identifier greater than
+	// or equal to ID. If ID is smaller than the start of the range, the iteration
 	// will start there instead. If ID is greater than or equal to the end of
 	// the range, Next() call will return io.EOF.
-	Advance(ID string) (string, error)
+	Advance(ID IndexInternalID) (IndexInternalID, error)
 	Close() error
 }