mirror of https://github.com/go-gitea/gitea
Upgrade blevesearch dependency to v2.0.1 (#14346)
* Upgrade blevesearch dependency to v2.0.1 * Update rupture to v1.0.0 * Fix testpull/14368/head^2
Lauris BH
4 years ago
committed by
GitHub
parent
3aa53dc6bc
commit
f5abe2f563
@ -1,110 +0,0 @@ |
||||
// Copyright (c) 2015 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package index |
||||
|
||||
import ( |
||||
"reflect" |
||||
|
||||
"github.com/blevesearch/bleve/analysis" |
||||
"github.com/blevesearch/bleve/document" |
||||
"github.com/blevesearch/bleve/size" |
||||
) |
||||
|
||||
var reflectStaticSizeAnalysisResult int |
||||
|
||||
func init() { |
||||
var ar AnalysisResult |
||||
reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size()) |
||||
} |
||||
|
||||
type IndexRow interface { |
||||
KeySize() int |
||||
KeyTo([]byte) (int, error) |
||||
Key() []byte |
||||
|
||||
ValueSize() int |
||||
ValueTo([]byte) (int, error) |
||||
Value() []byte |
||||
} |
||||
|
||||
type AnalysisResult struct { |
||||
DocID string |
||||
Rows []IndexRow |
||||
|
||||
// scorch
|
||||
Document *document.Document |
||||
Analyzed []analysis.TokenFrequencies |
||||
Length []int |
||||
} |
||||
|
||||
func (a *AnalysisResult) Size() int { |
||||
rv := reflectStaticSizeAnalysisResult |
||||
for _, analyzedI := range a.Analyzed { |
||||
rv += analyzedI.Size() |
||||
} |
||||
rv += len(a.Length) * size.SizeOfInt |
||||
return rv |
||||
} |
||||
|
||||
type AnalysisWork struct { |
||||
i Index |
||||
d *document.Document |
||||
rc chan *AnalysisResult |
||||
} |
||||
|
||||
func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork { |
||||
return &AnalysisWork{ |
||||
i: i, |
||||
d: d, |
||||
rc: rc, |
||||
} |
||||
} |
||||
|
||||
type AnalysisQueue struct { |
||||
queue chan *AnalysisWork |
||||
done chan struct{} |
||||
} |
||||
|
||||
func (q *AnalysisQueue) Queue(work *AnalysisWork) { |
||||
q.queue <- work |
||||
} |
||||
|
||||
func (q *AnalysisQueue) Close() { |
||||
close(q.done) |
||||
} |
||||
|
||||
func NewAnalysisQueue(numWorkers int) *AnalysisQueue { |
||||
rv := AnalysisQueue{ |
||||
queue: make(chan *AnalysisWork), |
||||
done: make(chan struct{}), |
||||
} |
||||
for i := 0; i < numWorkers; i++ { |
||||
go AnalysisWorker(rv) |
||||
} |
||||
return &rv |
||||
} |
||||
|
||||
func AnalysisWorker(q AnalysisQueue) { |
||||
// read work off the queue
|
||||
for { |
||||
select { |
||||
case <-q.done: |
||||
return |
||||
case w := <-q.queue: |
||||
r := w.i.Analyze(w.d) |
||||
w.rc <- r |
||||
} |
||||
} |
||||
} |
||||
@ -1,137 +0,0 @@ |
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package segment |
||||
|
||||
import ( |
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/index" |
||||
"github.com/couchbase/vellum" |
||||
) |
||||
|
||||
type EmptySegment struct{} |
||||
|
||||
func (e *EmptySegment) Dictionary(field string) (TermDictionary, error) { |
||||
return &EmptyDictionary{}, nil |
||||
} |
||||
|
||||
func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error { |
||||
return nil |
||||
} |
||||
|
||||
func (e *EmptySegment) DocID(num uint64) ([]byte, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *EmptySegment) Count() uint64 { |
||||
return 0 |
||||
} |
||||
|
||||
func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error) { |
||||
r := roaring.NewBitmap() |
||||
return r, nil |
||||
} |
||||
|
||||
func (e *EmptySegment) Fields() []string { |
||||
return []string{} |
||||
} |
||||
|
||||
func (e *EmptySegment) Close() error { |
||||
return nil |
||||
} |
||||
|
||||
func (e *EmptySegment) Size() uint64 { |
||||
return 0 |
||||
} |
||||
|
||||
func (e *EmptySegment) AddRef() { |
||||
} |
||||
|
||||
func (e *EmptySegment) DecRef() error { |
||||
return nil |
||||
} |
||||
|
||||
type EmptyDictionary struct{} |
||||
|
||||
func (e *EmptyDictionary) PostingsList(term []byte, |
||||
except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) { |
||||
return &EmptyPostingsList{}, nil |
||||
} |
||||
|
||||
func (e *EmptyDictionary) Iterator() DictionaryIterator { |
||||
return &EmptyDictionaryIterator{} |
||||
} |
||||
|
||||
func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator { |
||||
return &EmptyDictionaryIterator{} |
||||
} |
||||
|
||||
func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator { |
||||
return &EmptyDictionaryIterator{} |
||||
} |
||||
|
||||
func (e *EmptyDictionary) AutomatonIterator(a vellum.Automaton, |
||||
startKeyInclusive, endKeyExclusive []byte) DictionaryIterator { |
||||
return &EmptyDictionaryIterator{} |
||||
} |
||||
|
||||
func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte, |
||||
includeCount bool) DictionaryIterator { |
||||
return &EmptyDictionaryIterator{} |
||||
} |
||||
|
||||
func (e *EmptyDictionary) Contains(key []byte) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
type EmptyDictionaryIterator struct{} |
||||
|
||||
func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *EmptyDictionaryIterator) Contains(key []byte) (bool, error) { |
||||
return false, nil |
||||
} |
||||
|
||||
type EmptyPostingsList struct{} |
||||
|
||||
func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool, |
||||
prealloc PostingsIterator) PostingsIterator { |
||||
return &EmptyPostingsIterator{} |
||||
} |
||||
|
||||
func (e *EmptyPostingsList) Size() int { |
||||
return 0 |
||||
} |
||||
|
||||
func (e *EmptyPostingsList) Count() uint64 { |
||||
return 0 |
||||
} |
||||
|
||||
type EmptyPostingsIterator struct{} |
||||
|
||||
func (e *EmptyPostingsIterator) Next() (Posting, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *EmptyPostingsIterator) Size() int { |
||||
return 0 |
||||
} |
||||
|
||||
var AnEmptyPostingsIterator = &EmptyPostingsIterator{} |
||||
@ -1,176 +0,0 @@ |
||||
// Copyright 2014 The Cockroach Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
// implied. See the License for the specific language governing
|
||||
// permissions and limitations under the License.
|
||||
|
||||
// This code originated from:
|
||||
// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go
|
||||
|
||||
// Modified to not use pkg/errors
|
||||
|
||||
package segment |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
) |
||||
|
||||
const ( |
||||
MaxVarintSize = 9 |
||||
|
||||
// IntMin is chosen such that the range of int tags does not overlap the
|
||||
// ascii character set that is frequently used in testing.
|
||||
IntMin = 0x80 // 128
|
||||
intMaxWidth = 8 |
||||
intZero = IntMin + intMaxWidth // 136
|
||||
intSmall = IntMax - intZero - intMaxWidth // 109
|
||||
// IntMax is the maximum int tag value.
|
||||
IntMax = 0xfd // 253
|
||||
) |
||||
|
||||
// EncodeUvarintAscending encodes the uint64 value using a variable length
|
||||
// (length-prefixed) representation. The length is encoded as a single
|
||||
// byte indicating the number of encoded bytes (-8) to follow. See
|
||||
// EncodeVarintAscending for rationale. The encoded bytes are appended to the
|
||||
// supplied buffer and the final buffer is returned.
|
||||
func EncodeUvarintAscending(b []byte, v uint64) []byte { |
||||
switch { |
||||
case v <= intSmall: |
||||
return append(b, intZero+byte(v)) |
||||
case v <= 0xff: |
||||
return append(b, IntMax-7, byte(v)) |
||||
case v <= 0xffff: |
||||
return append(b, IntMax-6, byte(v>>8), byte(v)) |
||||
case v <= 0xffffff: |
||||
return append(b, IntMax-5, byte(v>>16), byte(v>>8), byte(v)) |
||||
case v <= 0xffffffff: |
||||
return append(b, IntMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||
case v <= 0xffffffffff: |
||||
return append(b, IntMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), |
||||
byte(v)) |
||||
case v <= 0xffffffffffff: |
||||
return append(b, IntMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), |
||||
byte(v>>8), byte(v)) |
||||
case v <= 0xffffffffffffff: |
||||
return append(b, IntMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), |
||||
byte(v>>16), byte(v>>8), byte(v)) |
||||
default: |
||||
return append(b, IntMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), |
||||
byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||
} |
||||
} |
||||
|
||||
// DecodeUvarintAscending decodes a varint encoded uint64 from the input
|
||||
// buffer. The remainder of the input buffer and the decoded uint64
|
||||
// are returned.
|
||||
func DecodeUvarintAscending(b []byte) ([]byte, uint64, error) { |
||||
if len(b) == 0 { |
||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") |
||||
} |
||||
length := int(b[0]) - intZero |
||||
b = b[1:] // skip length byte
|
||||
if length <= intSmall { |
||||
return b, uint64(length), nil |
||||
} |
||||
length -= intSmall |
||||
if length < 0 || length > 8 { |
||||
return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) |
||||
} else if len(b) < length { |
||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) |
||||
} |
||||
var v uint64 |
||||
// It is faster to range over the elements in a slice than to index
|
||||
// into the slice on each loop iteration.
|
||||
for _, t := range b[:length] { |
||||
v = (v << 8) | uint64(t) |
||||
} |
||||
return b[length:], v, nil |
||||
} |
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
type MemUvarintReader struct { |
||||
C int // index of next byte to read from S
|
||||
S []byte |
||||
} |
||||
|
||||
func NewMemUvarintReader(s []byte) *MemUvarintReader { |
||||
return &MemUvarintReader{S: s} |
||||
} |
||||
|
||||
// Len returns the number of unread bytes.
|
||||
func (r *MemUvarintReader) Len() int { |
||||
n := len(r.S) - r.C |
||||
if n < 0 { |
||||
return 0 |
||||
} |
||||
return n |
||||
} |
||||
|
||||
var ErrMemUvarintReaderOverflow = errors.New("MemUvarintReader overflow") |
||||
|
||||
// ReadUvarint reads an encoded uint64. The original code this was
|
||||
// based on is at encoding/binary/ReadUvarint().
|
||||
func (r *MemUvarintReader) ReadUvarint() (uint64, error) { |
||||
var x uint64 |
||||
var s uint |
||||
var C = r.C |
||||
var S = r.S |
||||
|
||||
for { |
||||
b := S[C] |
||||
C++ |
||||
|
||||
if b < 0x80 { |
||||
r.C = C |
||||
|
||||
// why 63? The original code had an 'i += 1' loop var and
|
||||
// checked for i > 9 || i == 9 ...; but, we no longer
|
||||
// check for the i var, but instead check here for s,
|
||||
// which is incremented by 7. So, 7*9 == 63.
|
||||
//
|
||||
// why the "extra" >= check? The normal case is that s <
|
||||
// 63, so we check this single >= guard first so that we
|
||||
// hit the normal, nil-error return pathway sooner.
|
||||
if s >= 63 && (s > 63 || s == 63 && b > 1) { |
||||
return 0, ErrMemUvarintReaderOverflow |
||||
} |
||||
|
||||
return x | uint64(b)<<s, nil |
||||
} |
||||
|
||||
x |= uint64(b&0x7f) << s |
||||
s += 7 |
||||
} |
||||
} |
||||
|
||||
// SkipUvarint skips ahead one encoded uint64.
|
||||
func (r *MemUvarintReader) SkipUvarint() { |
||||
for { |
||||
b := r.S[r.C] |
||||
r.C++ |
||||
|
||||
if b < 0x80 { |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
// SkipBytes skips a count number of bytes.
|
||||
func (r *MemUvarintReader) SkipBytes(count int) { |
||||
r.C = r.C + count |
||||
} |
||||
|
||||
func (r *MemUvarintReader) Reset(s []byte) { |
||||
r.C = 0 |
||||
r.S = s |
||||
} |
||||
@ -1,58 +0,0 @@ |
||||
// Copyright (c) 2020 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package segment |
||||
|
||||
import ( |
||||
"github.com/RoaringBitmap/roaring" |
||||
"github.com/blevesearch/bleve/index" |
||||
) |
||||
|
||||
// Plugin represents the essential functions required by a package to plug in
|
||||
// it's segment implementation
|
||||
type Plugin interface { |
||||
|
||||
// Type is the name for this segment plugin
|
||||
Type() string |
||||
|
||||
// Version is a numeric value identifying a specific version of this type.
|
||||
// When incompatible changes are made to a particular type of plugin, the
|
||||
// version must be incremented.
|
||||
Version() uint32 |
||||
|
||||
// New takes a set of AnalysisResults and turns them into a new Segment
|
||||
New(results []*index.AnalysisResult) (Segment, uint64, error) |
||||
|
||||
// Open attempts to open the file at the specified path and
|
||||
// return the corresponding Segment
|
||||
Open(path string) (Segment, error) |
||||
|
||||
// Merge takes a set of Segments, and creates a new segment on disk at
|
||||
// the specified path.
|
||||
// Drops is a set of bitmaps (one for each segment) indicating which
|
||||
// documents can be dropped from the segments during the merge.
|
||||
// If the closeCh channel is closed, Merge will cease doing work at
|
||||
// the next opportunity, and return an error (closed).
|
||||
// StatsReporter can optionally be provided, in which case progress
|
||||
// made during the merge is reported while operation continues.
|
||||
// Returns:
|
||||
// A slice of new document numbers (one for each input segment),
|
||||
// this allows the caller to know a particular document's new
|
||||
// document number in the newly merged segment.
|
||||
// The number of bytes written to the new segment file.
|
||||
// An error, if any occurred.
|
||||
Merge(segments []Segment, drops []*roaring.Bitmap, path string, |
||||
closeCh chan struct{}, s StatsReporter) ( |
||||
[][]uint64, uint64, error) |
||||
} |
||||
@ -1,93 +0,0 @@ |
||||
// Copyright (c) 2019 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scorch |
||||
|
||||
import ( |
||||
"fmt" |
||||
|
||||
"github.com/blevesearch/bleve/index/scorch/segment" |
||||
|
||||
zapv11 "github.com/blevesearch/zap/v11" |
||||
zapv12 "github.com/blevesearch/zap/v12" |
||||
zapv13 "github.com/blevesearch/zap/v13" |
||||
zapv14 "github.com/blevesearch/zap/v14" |
||||
zapv15 "github.com/blevesearch/zap/v15" |
||||
) |
||||
|
||||
var supportedSegmentPlugins map[string]map[uint32]segment.Plugin |
||||
var defaultSegmentPlugin segment.Plugin |
||||
|
||||
func init() { |
||||
ResetPlugins() |
||||
RegisterPlugin(zapv15.Plugin(), false) |
||||
RegisterPlugin(zapv14.Plugin(), false) |
||||
RegisterPlugin(zapv13.Plugin(), false) |
||||
RegisterPlugin(zapv12.Plugin(), false) |
||||
RegisterPlugin(zapv11.Plugin(), true) |
||||
} |
||||
|
||||
func ResetPlugins() { |
||||
supportedSegmentPlugins = map[string]map[uint32]segment.Plugin{} |
||||
} |
||||
|
||||
func RegisterPlugin(plugin segment.Plugin, makeDefault bool) { |
||||
if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { |
||||
supportedSegmentPlugins[plugin.Type()] = map[uint32]segment.Plugin{} |
||||
} |
||||
supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin |
||||
if makeDefault { |
||||
defaultSegmentPlugin = plugin |
||||
} |
||||
} |
||||
|
||||
func SupportedSegmentTypes() (rv []string) { |
||||
for k := range supportedSegmentPlugins { |
||||
rv = append(rv, k) |
||||
} |
||||
return |
||||
} |
||||
|
||||
func SupportedSegmentTypeVersions(typ string) (rv []uint32) { |
||||
for k := range supportedSegmentPlugins[typ] { |
||||
rv = append(rv, k) |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func chooseSegmentPlugin(forcedSegmentType string, |
||||
forcedSegmentVersion uint32) (segment.Plugin, error) { |
||||
if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { |
||||
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { |
||||
return segPlugin, nil |
||||
} |
||||
return nil, fmt.Errorf( |
||||
"unsupported version %d for segment type: %s, supported: %v", |
||||
forcedSegmentVersion, forcedSegmentType, |
||||
SupportedSegmentTypeVersions(forcedSegmentType)) |
||||
} |
||||
return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", |
||||
forcedSegmentType, SupportedSegmentTypes()) |
||||
} |
||||
|
||||
func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, |
||||
forcedSegmentVersion uint32) error { |
||||
segPlugin, err := chooseSegmentPlugin(forcedSegmentType, |
||||
forcedSegmentVersion) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
s.segPlugin = segPlugin |
||||
return nil |
||||
} |
||||
@ -0,0 +1,70 @@ |
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package analysis |
||||
|
||||
import ( |
||||
index "github.com/blevesearch/bleve_index_api" |
||||
) |
||||
|
||||
func TokenFrequency(tokens TokenStream, arrayPositions []uint64, options index.FieldIndexingOptions) index.TokenFrequencies { |
||||
rv := make(map[string]*index.TokenFreq, len(tokens)) |
||||
|
||||
if options.IncludeTermVectors() { |
||||
tls := make([]index.TokenLocation, len(tokens)) |
||||
tlNext := 0 |
||||
|
||||
for _, token := range tokens { |
||||
tls[tlNext] = index.TokenLocation{ |
||||
ArrayPositions: arrayPositions, |
||||
Start: token.Start, |
||||
End: token.End, |
||||
Position: token.Position, |
||||
} |
||||
|
||||
curr, ok := rv[string(token.Term)] |
||||
if ok { |
||||
curr.Locations = append(curr.Locations, &tls[tlNext]) |
||||
} else { |
||||
curr = &index.TokenFreq{ |
||||
Term: token.Term, |
||||
Locations: []*index.TokenLocation{&tls[tlNext]}, |
||||
} |
||||
rv[string(token.Term)] = curr |
||||
} |
||||
|
||||
if !options.SkipFreqNorm() { |
||||
curr.SetFrequency(curr.Frequency() + 1) |
||||
} |
||||
|
||||
tlNext++ |
||||
} |
||||
} else { |
||||
for _, token := range tokens { |
||||
curr, exists := rv[string(token.Term)] |
||||
if !exists { |
||||
curr = &index.TokenFreq{ |
||||
Term: token.Term, |
||||
} |
||||
rv[string(token.Term)] = curr |
||||
} |
||||
|
||||
if !options.SkipFreqNorm() { |
||||
curr.SetFrequency(curr.Frequency() + 1) |
||||
} |
||||
} |
||||
} |
||||
|
||||
return rv |
||||
} |
||||
@ -1,8 +1,8 @@ |
||||
package en |
||||
|
||||
import ( |
||||
"github.com/blevesearch/bleve/analysis" |
||||
"github.com/blevesearch/bleve/registry" |
||||
"github.com/blevesearch/bleve/v2/analysis" |
||||
"github.com/blevesearch/bleve/v2/registry" |
||||
) |
||||
|
||||
const StopName = "stop_en" |
||||
@ -1,18 +1,20 @@ |
||||
module github.com/blevesearch/bleve |
||||
module github.com/blevesearch/bleve/v2 |
||||
|
||||
go 1.13 |
||||
|
||||
require ( |
||||
github.com/RoaringBitmap/roaring v0.4.23 |
||||
github.com/blevesearch/blevex v1.0.0 |
||||
github.com/blevesearch/bleve_index_api v1.0.0 |
||||
github.com/blevesearch/go-porterstemmer v1.0.3 |
||||
github.com/blevesearch/scorch_segment_api v1.0.0 |
||||
github.com/blevesearch/segment v0.9.0 |
||||
github.com/blevesearch/snowballstem v0.9.0 |
||||
github.com/blevesearch/zap/v11 v11.0.14 |
||||
github.com/blevesearch/zap/v12 v12.0.14 |
||||
github.com/blevesearch/zap/v13 v13.0.6 |
||||
github.com/blevesearch/zap/v14 v14.0.5 |
||||
github.com/blevesearch/zap/v15 v15.0.3 |
||||
github.com/blevesearch/upsidedown_store_api v1.0.1 |
||||
github.com/blevesearch/zapx/v11 v11.1.10 |
||||
github.com/blevesearch/zapx/v12 v12.1.10 |
||||
github.com/blevesearch/zapx/v13 v13.1.10 |
||||
github.com/blevesearch/zapx/v14 v14.1.10 |
||||
github.com/blevesearch/zapx/v15 v15.1.10 |
||||
github.com/couchbase/moss v0.1.0 |
||||
github.com/couchbase/vellum v1.0.2 |
||||
github.com/golang/protobuf v1.3.2 |
||||
@ -0,0 +1,128 @@ |
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= |
||||
github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= |
||||
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= |
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= |
||||
github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= |
||||
github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= |
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= |
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= |
||||
github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= |
||||
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= |
||||
github.com/blevesearch/scorch_segment_api v1.0.0 h1:BUkCPWDg2gimTEyVDXf85I2buqqt4lh28uaVMiJsIYk= |
||||
github.com/blevesearch/scorch_segment_api v1.0.0/go.mod h1:KgRYmlfYC27NeM6cXOHx8LBgq7jn0atpV8mVWoBKBng= |
||||
github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= |
||||
github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= |
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= |
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= |
||||
github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= |
||||
github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= |
||||
github.com/blevesearch/zapx/v11 v11.1.10 h1:8Eo3rXiHsVSP9Sk+4StrrwLrj9vyulhMVPmxTf8ZuDg= |
||||
github.com/blevesearch/zapx/v11 v11.1.10/go.mod h1:DTjbcBqrr/Uo82UBilDC8lEew42gN/OcIyiTNFtSijc= |
||||
github.com/blevesearch/zapx/v12 v12.1.10 h1:sqR+/0Z4dSTovApRqLA1HnilMtQer7a4UvPrNmPzlTM= |
||||
github.com/blevesearch/zapx/v12 v12.1.10/go.mod h1:14NmKnPrnKAIyiEJM566k/Jk+FQpuiflT5d3uaaK3MI= |
||||
github.com/blevesearch/zapx/v13 v13.1.10 h1:zCneEVRJDXwtDfSwh+33Dxguliv192vCK283zdGH4Sw= |
||||
github.com/blevesearch/zapx/v13 v13.1.10/go.mod h1:YsVY6YGpTEAlJOMjdL7EsdBLvjWd8kPa2gwJDNpqLJo= |
||||
github.com/blevesearch/zapx/v14 v14.1.10 h1:nD0vw2jxKogJFfA5WyoS4wNwZlVby3Aq8aW7CZi6YIw= |
||||
github.com/blevesearch/zapx/v14 v14.1.10/go.mod h1:hsULl5eJSxs5NEfBsmeT9qrqdCP+/ecpVZKt60M4V64= |
||||
github.com/blevesearch/zapx/v15 v15.1.10 h1:kZR3b9jO9l6s2B5UHI+1N1llLzJ4nYikkXQTMrDl1vQ= |
||||
github.com/blevesearch/zapx/v15 v15.1.10/go.mod h1:4ypq25bwtSQKzwEF1UERyIhmGTbMT3brY/n4NC5gRnM= |
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= |
||||
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= |
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= |
||||
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= |
||||
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= |
||||
github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI= |
||||
github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= |
||||
github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= |
||||
github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= |
||||
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= |
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= |
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= |
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= |
||||
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= |
||||
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= |
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= |
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= |
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
||||
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= |
||||
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= |
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= |
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= |
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= |
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= |
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= |
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= |
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= |
||||
github.com/kljensen/snowball v0.6.0 h1:6DZLCcZeL0cLfodx+Md4/OLC6b/bfurWUOUGs1ydfOU= |
||||
github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= |
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= |
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= |
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= |
||||
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= |
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= |
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= |
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= |
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= |
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= |
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= |
||||
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= |
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= |
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= |
||||
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= |
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= |
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= |
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= |
||||
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= |
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= |
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= |
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= |
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= |
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= |
||||
github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= |
||||
github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= |
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= |
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= |
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= |
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= |
||||
github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= |
||||
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= |
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= |
||||
github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= |
||||
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= |
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= |
||||
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= |
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= |
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= |
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= |
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= |
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= |
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= |
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= |
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= |
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= |
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= |
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
||||
@ -0,0 +1,33 @@ |
||||
// Copyright (c) 2020 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scorch |
||||
|
||||
import segment "github.com/blevesearch/scorch_segment_api" |
||||
|
||||
type emptyPostingsIterator struct{} |
||||
|
||||
func (e *emptyPostingsIterator) Next() (segment.Posting, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *emptyPostingsIterator) Advance(uint64) (segment.Posting, error) { |
||||
return nil, nil |
||||
} |
||||
|
||||
func (e *emptyPostingsIterator) Size() int { |
||||
return 0 |
||||
} |
||||
|
||||
var anEmptyPostingsIterator = &emptyPostingsIterator{} |
||||
@ -0,0 +1,92 @@ |
||||
// Copyright 2014 The Cockroach Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
// implied. See the License for the specific language governing
|
||||
// permissions and limitations under the License.
|
||||
|
||||
// This code originated from:
|
||||
// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go
|
||||
|
||||
// Modified to not use pkg/errors
|
||||
|
||||
package scorch |
||||
|
||||
import "fmt" |
||||
|
||||
const ( |
||||
// intMin is chosen such that the range of int tags does not overlap the
|
||||
// ascii character set that is frequently used in testing.
|
||||
intMin = 0x80 // 128
|
||||
intMaxWidth = 8 |
||||
intZero = intMin + intMaxWidth // 136
|
||||
intSmall = intMax - intZero - intMaxWidth // 109
|
||||
// intMax is the maximum int tag value.
|
||||
intMax = 0xfd // 253
|
||||
) |
||||
|
||||
// encodeUvarintAscending encodes the uint64 value using a variable length
|
||||
// (length-prefixed) representation. The length is encoded as a single
|
||||
// byte indicating the number of encoded bytes (-8) to follow. See
|
||||
// EncodeVarintAscending for rationale. The encoded bytes are appended to the
|
||||
// supplied buffer and the final buffer is returned.
|
||||
func encodeUvarintAscending(b []byte, v uint64) []byte { |
||||
switch { |
||||
case v <= intSmall: |
||||
return append(b, intZero+byte(v)) |
||||
case v <= 0xff: |
||||
return append(b, intMax-7, byte(v)) |
||||
case v <= 0xffff: |
||||
return append(b, intMax-6, byte(v>>8), byte(v)) |
||||
case v <= 0xffffff: |
||||
return append(b, intMax-5, byte(v>>16), byte(v>>8), byte(v)) |
||||
case v <= 0xffffffff: |
||||
return append(b, intMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||
case v <= 0xffffffffff: |
||||
return append(b, intMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), |
||||
byte(v)) |
||||
case v <= 0xffffffffffff: |
||||
return append(b, intMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), |
||||
byte(v>>8), byte(v)) |
||||
case v <= 0xffffffffffffff: |
||||
return append(b, intMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), |
||||
byte(v>>16), byte(v>>8), byte(v)) |
||||
default: |
||||
return append(b, intMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), |
||||
byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) |
||||
} |
||||
} |
||||
|
||||
// decodeUvarintAscending decodes a varint encoded uint64 from the input
|
||||
// buffer. The remainder of the input buffer and the decoded uint64
|
||||
// are returned.
|
||||
func decodeUvarintAscending(b []byte) ([]byte, uint64, error) { |
||||
if len(b) == 0 { |
||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") |
||||
} |
||||
length := int(b[0]) - intZero |
||||
b = b[1:] // skip length byte
|
||||
if length <= intSmall { |
||||
return b, uint64(length), nil |
||||
} |
||||
length -= intSmall |
||||
if length < 0 || length > 8 { |
||||
return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) |
||||
} else if len(b) < length { |
||||
return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) |
||||
} |
||||
var v uint64 |
||||
// It is faster to range over the elements in a slice than to index
|
||||
// into the slice on each loop iteration.
|
||||
for _, t := range b[:length] { |
||||
v = (v << 8) | uint64(t) |
||||
} |
||||
return b[length:], v, nil |
||||
} |
||||
@ -0,0 +1,133 @@ |
||||
// Copyright (c) 2019 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package scorch |
||||
|
||||
import ( |
||||
"fmt" |
||||
"github.com/RoaringBitmap/roaring" |
||||
index "github.com/blevesearch/bleve_index_api" |
||||
|
||||
segment "github.com/blevesearch/scorch_segment_api" |
||||
|
||||
zapv11 "github.com/blevesearch/zapx/v11" |
||||
zapv12 "github.com/blevesearch/zapx/v12" |
||||
zapv13 "github.com/blevesearch/zapx/v13" |
||||
zapv14 "github.com/blevesearch/zapx/v14" |
||||
zapv15 "github.com/blevesearch/zapx/v15" |
||||
) |
||||
|
||||
// SegmentPlugin represents the essential functions required by a package to plug in
|
||||
// it's segment implementation
|
||||
type SegmentPlugin interface { |
||||
|
||||
// Type is the name for this segment plugin
|
||||
Type() string |
||||
|
||||
// Version is a numeric value identifying a specific version of this type.
|
||||
// When incompatible changes are made to a particular type of plugin, the
|
||||
// version must be incremented.
|
||||
Version() uint32 |
||||
|
||||
// New takes a set of Documents and turns them into a new Segment
|
||||
New(results []index.Document) (segment.Segment, uint64, error) |
||||
|
||||
// Open attempts to open the file at the specified path and
|
||||
// return the corresponding Segment
|
||||
Open(path string) (segment.Segment, error) |
||||
|
||||
// Merge takes a set of Segments, and creates a new segment on disk at
|
||||
// the specified path.
|
||||
// Drops is a set of bitmaps (one for each segment) indicating which
|
||||
// documents can be dropped from the segments during the merge.
|
||||
// If the closeCh channel is closed, Merge will cease doing work at
|
||||
// the next opportunity, and return an error (closed).
|
||||
// StatsReporter can optionally be provided, in which case progress
|
||||
// made during the merge is reported while operation continues.
|
||||
// Returns:
|
||||
// A slice of new document numbers (one for each input segment),
|
||||
// this allows the caller to know a particular document's new
|
||||
// document number in the newly merged segment.
|
||||
// The number of bytes written to the new segment file.
|
||||
// An error, if any occurred.
|
||||
Merge(segments []segment.Segment, drops []*roaring.Bitmap, path string, |
||||
closeCh chan struct{}, s segment.StatsReporter) ( |
||||
[][]uint64, uint64, error) |
||||
} |
||||
|
||||
var supportedSegmentPlugins map[string]map[uint32]SegmentPlugin |
||||
var defaultSegmentPlugin SegmentPlugin |
||||
|
||||
func init() { |
||||
ResetSegmentPlugins() |
||||
RegisterSegmentPlugin(&zapv15.ZapPlugin{}, true) |
||||
RegisterSegmentPlugin(&zapv14.ZapPlugin{}, false) |
||||
RegisterSegmentPlugin(&zapv13.ZapPlugin{}, false) |
||||
RegisterSegmentPlugin(&zapv12.ZapPlugin{}, false) |
||||
RegisterSegmentPlugin(&zapv11.ZapPlugin{}, false) |
||||
} |
||||
|
||||
func ResetSegmentPlugins() { |
||||
supportedSegmentPlugins = map[string]map[uint32]SegmentPlugin{} |
||||
} |
||||
|
||||
func RegisterSegmentPlugin(plugin SegmentPlugin, makeDefault bool) { |
||||
if _, ok := supportedSegmentPlugins[plugin.Type()]; !ok { |
||||
supportedSegmentPlugins[plugin.Type()] = map[uint32]SegmentPlugin{} |
||||
} |
||||
supportedSegmentPlugins[plugin.Type()][plugin.Version()] = plugin |
||||
if makeDefault { |
||||
defaultSegmentPlugin = plugin |
||||
} |
||||
} |
||||
|
||||
func SupportedSegmentTypes() (rv []string) { |
||||
for k := range supportedSegmentPlugins { |
||||
rv = append(rv, k) |
||||
} |
||||
return |
||||
} |
||||
|
||||
func SupportedSegmentTypeVersions(typ string) (rv []uint32) { |
||||
for k := range supportedSegmentPlugins[typ] { |
||||
rv = append(rv, k) |
||||
} |
||||
return rv |
||||
} |
||||
|
||||
func chooseSegmentPlugin(forcedSegmentType string, |
||||
forcedSegmentVersion uint32) (SegmentPlugin, error) { |
||||
if versions, ok := supportedSegmentPlugins[forcedSegmentType]; ok { |
||||
if segPlugin, ok := versions[uint32(forcedSegmentVersion)]; ok { |
||||
return segPlugin, nil |
||||
} |
||||
return nil, fmt.Errorf( |
||||
"unsupported version %d for segment type: %s, supported: %v", |
||||
forcedSegmentVersion, forcedSegmentType, |
||||
SupportedSegmentTypeVersions(forcedSegmentType)) |
||||
} |
||||
return nil, fmt.Errorf("unsupported segment type: %s, supported: %v", |
||||
forcedSegmentType, SupportedSegmentTypes()) |
||||
} |
||||
|
||||
func (s *Scorch) loadSegmentPlugin(forcedSegmentType string, |
||||
forcedSegmentVersion uint32) error { |
||||
segPlugin, err := chooseSegmentPlugin(forcedSegmentType, |
||||
forcedSegmentVersion) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
s.segPlugin = segPlugin |
||||
return nil |
||||
} |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue