bugfixes more search results and dump restore options
This commit is contained in:
parent
f4fcf0d9a8
commit
3cd460318a
124 changed files with 3915 additions and 9579 deletions
229
vendor/github.com/blevesearch/bleve/index_impl.go
generated
vendored
229
vendor/github.com/blevesearch/bleve/index_impl.go
generated
vendored
|
|
@ -22,12 +22,12 @@ import (
|
|||
"github.com/blevesearch/bleve/document"
|
||||
"github.com/blevesearch/bleve/index"
|
||||
"github.com/blevesearch/bleve/index/store"
|
||||
"github.com/blevesearch/bleve/index/store/gtreap"
|
||||
"github.com/blevesearch/bleve/index/upside_down"
|
||||
"github.com/blevesearch/bleve/registry"
|
||||
"github.com/blevesearch/bleve/search"
|
||||
"github.com/blevesearch/bleve/search/collectors"
|
||||
"github.com/blevesearch/bleve/search/facets"
|
||||
"github.com/blevesearch/bleve/search/highlight"
|
||||
)
|
||||
|
||||
type indexImpl struct {
|
||||
|
|
@ -49,49 +49,6 @@ func indexStorePath(path string) string {
|
|||
return path + string(os.PathSeparator) + storePath
|
||||
}
|
||||
|
||||
func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) {
|
||||
rv := indexImpl{
|
||||
path: "",
|
||||
name: "mem",
|
||||
m: mapping,
|
||||
meta: newIndexMeta(indexType, gtreap.Name, nil),
|
||||
stats: &IndexStat{},
|
||||
}
|
||||
|
||||
// open the index
|
||||
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
||||
if indexTypeConstructor == nil {
|
||||
return nil, ErrorUnknownIndexType
|
||||
}
|
||||
|
||||
var err error
|
||||
rv.i, err = indexTypeConstructor(rv.meta.Storage, nil, Config.analysisQueue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = rv.i.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// now persist the mapping
|
||||
mappingBytes, err := json.Marshal(mapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// mark the index as open
|
||||
rv.mutex.Lock()
|
||||
defer rv.mutex.Unlock()
|
||||
rv.open = true
|
||||
indexStats.Register(&rv)
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
|
||||
// first validate the mapping
|
||||
err := mapping.Validate()
|
||||
|
|
@ -99,14 +56,14 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
return newMemIndex(indexType, mapping)
|
||||
}
|
||||
|
||||
if kvconfig == nil {
|
||||
kvconfig = map[string]interface{}{}
|
||||
}
|
||||
|
||||
if kvstore == "" {
|
||||
return nil, fmt.Errorf("bleve not configured for file based indexing")
|
||||
}
|
||||
|
||||
rv := indexImpl{
|
||||
path: path,
|
||||
name: path,
|
||||
|
|
@ -115,13 +72,17 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
|
|||
}
|
||||
rv.stats = &IndexStat{i: &rv}
|
||||
// at this point there is hope that we can be successful, so save index meta
|
||||
err = rv.meta.Save(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if path != "" {
|
||||
err = rv.meta.Save(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kvconfig["create_if_missing"] = true
|
||||
kvconfig["error_if_exists"] = true
|
||||
kvconfig["path"] = indexStorePath(path)
|
||||
} else {
|
||||
kvconfig["path"] = ""
|
||||
}
|
||||
kvconfig["create_if_missing"] = true
|
||||
kvconfig["error_if_exists"] = true
|
||||
kvconfig["path"] = indexStorePath(path)
|
||||
|
||||
// open the index
|
||||
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
||||
|
|
@ -349,7 +310,7 @@ func (i *indexImpl) Document(id string) (doc *document.Document, err error) {
|
|||
|
||||
// DocCount returns the number of documents in the
|
||||
// index.
|
||||
func (i *indexImpl) DocCount() (uint64, error) {
|
||||
func (i *indexImpl) DocCount() (count uint64, err error) {
|
||||
i.mutex.RLock()
|
||||
defer i.mutex.RUnlock()
|
||||
|
||||
|
|
@ -357,7 +318,19 @@ func (i *indexImpl) DocCount() (uint64, error) {
|
|||
return 0, ErrorIndexClosed
|
||||
}
|
||||
|
||||
return i.i.DocCount()
|
||||
// open a reader for this search
|
||||
indexReader, err := i.i.Reader()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error opening index reader %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := indexReader.Close(); err == nil && cerr != nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
count, err = indexReader.DocCount()
|
||||
return
|
||||
}
|
||||
|
||||
// Search executes a search request operation.
|
||||
|
|
@ -378,7 +351,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
|
|||
return nil, ErrorIndexClosed
|
||||
}
|
||||
|
||||
collector := collectors.NewTopScorerSkipCollector(req.Size, req.From)
|
||||
collector := collectors.NewTopNCollector(req.Size, req.From, req.Sort)
|
||||
|
||||
// open a reader for this search
|
||||
indexReader, err := i.i.Reader()
|
||||
|
|
@ -429,16 +402,18 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
|
|||
collector.SetFacetsBuilder(facetsBuilder)
|
||||
}
|
||||
|
||||
err = collector.Collect(ctx, searcher)
|
||||
err = collector.Collect(ctx, searcher, indexReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hits := collector.Results()
|
||||
|
||||
var highlighter highlight.Highlighter
|
||||
|
||||
if req.Highlight != nil {
|
||||
// get the right highlighter
|
||||
highlighter, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
|
||||
highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -451,74 +426,62 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
|
|||
if highlighter == nil {
|
||||
return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
|
||||
}
|
||||
|
||||
for _, hit := range hits {
|
||||
doc, err := indexReader.Document(hit.ID)
|
||||
if err == nil && doc != nil {
|
||||
highlightFields := req.Highlight.Fields
|
||||
if highlightFields == nil {
|
||||
// add all fields with matches
|
||||
highlightFields = make([]string, 0, len(hit.Locations))
|
||||
for k := range hit.Locations {
|
||||
highlightFields = append(highlightFields, k)
|
||||
}
|
||||
}
|
||||
|
||||
for _, hf := range highlightFields {
|
||||
highlighter.BestFragmentsInField(hit, doc, hf, 1)
|
||||
}
|
||||
} else if err == nil {
|
||||
// unexpected case, a doc ID that was found as a search hit
|
||||
// was unable to be found during document lookup
|
||||
return nil, ErrorIndexReadInconsistency
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(req.Fields) > 0 {
|
||||
for _, hit := range hits {
|
||||
// FIXME avoid loading doc second time
|
||||
// if we already loaded it for highlighting
|
||||
for _, hit := range hits {
|
||||
if len(req.Fields) > 0 || highlighter != nil {
|
||||
doc, err := indexReader.Document(hit.ID)
|
||||
if err == nil && doc != nil {
|
||||
for _, f := range req.Fields {
|
||||
for _, docF := range doc.Fields {
|
||||
if f == "*" || docF.Name() == f {
|
||||
var value interface{}
|
||||
switch docF := docF.(type) {
|
||||
case *document.TextField:
|
||||
value = string(docF.Value())
|
||||
case *document.NumericField:
|
||||
num, err := docF.Number()
|
||||
if err == nil {
|
||||
value = num
|
||||
if len(req.Fields) > 0 {
|
||||
for _, f := range req.Fields {
|
||||
for _, docF := range doc.Fields {
|
||||
if f == "*" || docF.Name() == f {
|
||||
var value interface{}
|
||||
switch docF := docF.(type) {
|
||||
case *document.TextField:
|
||||
value = string(docF.Value())
|
||||
case *document.NumericField:
|
||||
num, err := docF.Number()
|
||||
if err == nil {
|
||||
value = num
|
||||
}
|
||||
case *document.DateTimeField:
|
||||
datetime, err := docF.DateTime()
|
||||
if err == nil {
|
||||
value = datetime.Format(time.RFC3339)
|
||||
}
|
||||
case *document.BooleanField:
|
||||
boolean, err := docF.Boolean()
|
||||
if err == nil {
|
||||
value = boolean
|
||||
}
|
||||
}
|
||||
case *document.DateTimeField:
|
||||
datetime, err := docF.DateTime()
|
||||
if err == nil {
|
||||
value = datetime.Format(time.RFC3339)
|
||||
if value != nil {
|
||||
hit.AddFieldValue(docF.Name(), value)
|
||||
}
|
||||
case *document.BooleanField:
|
||||
boolean, err := docF.Boolean()
|
||||
if err == nil {
|
||||
value = boolean
|
||||
}
|
||||
}
|
||||
if value != nil {
|
||||
hit.AddFieldValue(docF.Name(), value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if highlighter != nil {
|
||||
highlightFields := req.Highlight.Fields
|
||||
if highlightFields == nil {
|
||||
// add all fields with matches
|
||||
highlightFields = make([]string, 0, len(hit.Locations))
|
||||
for k := range hit.Locations {
|
||||
highlightFields = append(highlightFields, k)
|
||||
}
|
||||
}
|
||||
for _, hf := range highlightFields {
|
||||
highlighter.BestFragmentsInField(hit, doc, hf, 1)
|
||||
}
|
||||
}
|
||||
} else if doc == nil {
|
||||
// unexpected case, a doc ID that was found as a search hit
|
||||
// was unable to be found during document lookup
|
||||
return nil, ErrorIndexReadInconsistency
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, hit := range hits {
|
||||
if i.name != "" {
|
||||
hit.Index = i.name
|
||||
}
|
||||
|
|
@ -656,52 +619,12 @@ func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.Fiel
|
|||
}, nil
|
||||
}
|
||||
|
||||
// DumpAll writes all index rows to a channel.
|
||||
// INTERNAL: do not rely on this function, it is
|
||||
// only intended to be used by the debug utilities
|
||||
func (i *indexImpl) DumpAll() chan interface{} {
|
||||
i.mutex.RLock()
|
||||
defer i.mutex.RUnlock()
|
||||
|
||||
if !i.open {
|
||||
return nil
|
||||
}
|
||||
|
||||
return i.i.DumpAll()
|
||||
}
|
||||
|
||||
// DumpFields writes all field rows in the index
|
||||
// to a channel.
|
||||
// INTERNAL: do not rely on this function, it is
|
||||
// only intended to be used by the debug utilities
|
||||
func (i *indexImpl) DumpFields() chan interface{} {
|
||||
i.mutex.RLock()
|
||||
defer i.mutex.RUnlock()
|
||||
|
||||
if !i.open {
|
||||
return nil
|
||||
}
|
||||
return i.i.DumpFields()
|
||||
}
|
||||
|
||||
// DumpDoc writes all rows in the index associated
|
||||
// with the specified identifier to a channel.
|
||||
// INTERNAL: do not rely on this function, it is
|
||||
// only intended to be used by the debug utilities
|
||||
func (i *indexImpl) DumpDoc(id string) chan interface{} {
|
||||
i.mutex.RLock()
|
||||
defer i.mutex.RUnlock()
|
||||
|
||||
if !i.open {
|
||||
return nil
|
||||
}
|
||||
return i.i.DumpDoc(id)
|
||||
}
|
||||
|
||||
func (i *indexImpl) Close() error {
|
||||
i.mutex.Lock()
|
||||
defer i.mutex.Unlock()
|
||||
|
||||
indexStats.UnRegister(i)
|
||||
|
||||
i.open = false
|
||||
return i.i.Close()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue