bugfixes and updated go.mod

vendor/github.com/blevesearch/bleve/analysis/analyzer/custom/custom.go

@@ -0,0 +1,145 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package custom
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "custom"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+
+	var err error
+	var charFilters []analysis.CharFilter
+	charFiltersValue, ok := config["char_filters"]
+	if ok {
+		switch charFiltersValue := charFiltersValue.(type) {
+		case []string:
+			charFilters, err = getCharFilters(charFiltersValue, cache)
+			if err != nil {
+				return nil, err
+			}
+		case []interface{}:
+			charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersValue, "char filter")
+			if err != nil {
+				return nil, err
+			}
+			charFilters, err = getCharFilters(charFiltersNames, cache)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unsupported type for char_filters, must be slice")
+		}
+	}
+
+	var tokenizerName string
+	tokenizerValue, ok := config["tokenizer"]
+	if ok {
+		tokenizerName, ok = tokenizerValue.(string)
+		if !ok {
+			return nil, fmt.Errorf("must specify tokenizer as string")
+		}
+	} else {
+		return nil, fmt.Errorf("must specify tokenizer")
+	}
+
+	tokenizer, err := cache.TokenizerNamed(tokenizerName)
+	if err != nil {
+		return nil, err
+	}
+
+	var tokenFilters []analysis.TokenFilter
+	tokenFiltersValue, ok := config["token_filters"]
+	if ok {
+		switch tokenFiltersValue := tokenFiltersValue.(type) {
+		case []string:
+			tokenFilters, err = getTokenFilters(tokenFiltersValue, cache)
+			if err != nil {
+				return nil, err
+			}
+		case []interface{}:
+			tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersValue, "token filter")
+			if err != nil {
+				return nil, err
+			}
+			tokenFilters, err = getTokenFilters(tokenFiltersNames, cache)
+			if err != nil {
+				return nil, err
+			}
+		default:
+			return nil, fmt.Errorf("unsupported type for token_filters, must be slice")
+		}
+	}
+
+	rv := analysis.Analyzer{
+		Tokenizer: tokenizer,
+	}
+	if charFilters != nil {
+		rv.CharFilters = charFilters
+	}
+	if tokenFilters != nil {
+		rv.TokenFilters = tokenFilters
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(Name, AnalyzerConstructor)
+}
+
+func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) {
+	charFilters := make([]analysis.CharFilter, len(charFilterNames))
+	for i, charFilterName := range charFilterNames {
+		charFilter, err := cache.CharFilterNamed(charFilterName)
+		if err != nil {
+			return nil, err
+		}
+		charFilters[i] = charFilter
+	}
+
+	return charFilters, nil
+}
+
+func getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) {
+	tokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames))
+	for i, tokenFilterName := range tokenFilterNames {
+		tokenFilter, err := cache.TokenFilterNamed(tokenFilterName)
+		if err != nil {
+			return nil, err
+		}
+		tokenFilters[i] = tokenFilter
+	}
+
+	return tokenFilters, nil
+}
+
+func convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) {
+	stringSlice := make([]string, len(interfaceSlice))
+	for i, interfaceObj := range interfaceSlice {
+		stringObj, ok := interfaceObj.(string)
+		if ok {
+			stringSlice[i] = stringObj
+		} else {
+			return nil, fmt.Errorf(objType + " name must be a string")
+		}
+	}
+
+	return stringSlice, nil
+}

vendor/github.com/blevesearch/bleve/analysis/token/edgengram/edgengram.go

@@ -0,0 +1,115 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package edgengram
+
+import (
+	"bytes"
+	"fmt"
+	"unicode/utf8"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const Name = "edge_ngram"
+
+type Side bool
+
+const BACK Side = true
+const FRONT Side = false
+
+type EdgeNgramFilter struct {
+	back      Side
+	minLength int
+	maxLength int
+}
+
+func NewEdgeNgramFilter(side Side, minLength, maxLength int) *EdgeNgramFilter {
+	return &EdgeNgramFilter{
+		back:      side,
+		minLength: minLength,
+		maxLength: maxLength,
+	}
+}
+
+func (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	rv := make(analysis.TokenStream, 0, len(input))
+
+	for _, token := range input {
+		runeCount := utf8.RuneCount(token.Term)
+		runes := bytes.Runes(token.Term)
+		if s.back {
+			i := runeCount
+			// index of the starting rune for this token
+			for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {
+				// build an ngram of this size starting at i
+				if i-ngramSize >= 0 {
+					ngramTerm := analysis.BuildTermFromRunes(runes[i-ngramSize : i])
+					token := analysis.Token{
+						Position: token.Position,
+						Start:    token.Start,
+						End:      token.End,
+						Type:     token.Type,
+						Term:     ngramTerm,
+					}
+					rv = append(rv, &token)
+				}
+			}
+		} else {
+			i := 0
+			// index of the starting rune for this token
+			for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {
+				// build an ngram of this size starting at i
+				if i+ngramSize <= runeCount {
+					ngramTerm := analysis.BuildTermFromRunes(runes[i : i+ngramSize])
+					token := analysis.Token{
+						Position: token.Position,
+						Start:    token.Start,
+						End:      token.End,
+						Type:     token.Type,
+						Term:     ngramTerm,
+					}
+					rv = append(rv, &token)
+				}
+			}
+		}
+	}
+
+	return rv
+}
+
+func EdgeNgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	side := FRONT
+	back, ok := config["back"].(bool)
+	if ok && back {
+		side = BACK
+	}
+	minVal, ok := config["min"].(float64)
+	if !ok {
+		return nil, fmt.Errorf("must specify min")
+	}
+	min := int(minVal)
+	maxVal, ok := config["max"].(float64)
+	if !ok {
+		return nil, fmt.Errorf("must specify max")
+	}
+	max := int(maxVal)
+
+	return NewEdgeNgramFilter(side, min, max), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(Name, EdgeNgramFilterConstructor)
+}