Adding upstream version 2.5.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

index/upsidedown/store/boltdb/iterator.go

@@ -0,0 +1,85 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"bytes"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+type Iterator struct {
+	store  *Store
+	tx     *bolt.Tx
+	cursor *bolt.Cursor
+	prefix []byte
+	start  []byte
+	end    []byte
+	valid  bool
+	key    []byte
+	val    []byte
+}
+
+func (i *Iterator) updateValid() {
+	i.valid = (i.key != nil)
+	if i.valid {
+		if i.prefix != nil {
+			i.valid = bytes.HasPrefix(i.key, i.prefix)
+		} else if i.end != nil {
+			i.valid = bytes.Compare(i.key, i.end) < 0
+		}
+	}
+}
+
+func (i *Iterator) Seek(k []byte) {
+	if i.start != nil && bytes.Compare(k, i.start) < 0 {
+		k = i.start
+	}
+	if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {
+		if bytes.Compare(k, i.prefix) < 0 {
+			k = i.prefix
+		} else {
+			i.valid = false
+			return
+		}
+	}
+	i.key, i.val = i.cursor.Seek(k)
+	i.updateValid()
+}
+
+func (i *Iterator) Next() {
+	i.key, i.val = i.cursor.Next()
+	i.updateValid()
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	return i.key, i.val, i.valid
+}
+
+func (i *Iterator) Key() []byte {
+	return i.key
+}
+
+func (i *Iterator) Value() []byte {
+	return i.val
+}
+
+func (i *Iterator) Valid() bool {
+	return i.valid
+}
+
+func (i *Iterator) Close() error {
+	return nil
+}

index/upsidedown/store/boltdb/reader.go

@@ -0,0 +1,73 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	store "github.com/blevesearch/upsidedown_store_api"
+	bolt "go.etcd.io/bbolt"
+)
+
+type Reader struct {
+	store  *Store
+	tx     *bolt.Tx
+	bucket *bolt.Bucket
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	var rv []byte
+	v := r.bucket.Get(key)
+	if v != nil {
+		rv = make([]byte, len(v))
+		copy(rv, v)
+	}
+	return rv, nil
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
+	cursor := r.bucket.Cursor()
+
+	rv := &Iterator{
+		store:  r.store,
+		tx:     r.tx,
+		cursor: cursor,
+		prefix: prefix,
+	}
+
+	rv.Seek(prefix)
+	return rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	cursor := r.bucket.Cursor()
+
+	rv := &Iterator{
+		store:  r.store,
+		tx:     r.tx,
+		cursor: cursor,
+		start:  start,
+		end:    end,
+	}
+
+	rv.Seek(start)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.tx.Rollback()
+}

index/upsidedown/store/boltdb/stats.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"github.com/blevesearch/bleve/v2/util"
+)
+
+type stats struct {
+	s *Store
+}
+
+func (s *stats) MarshalJSON() ([]byte, error) {
+	bs := s.s.db.Stats()
+	return util.MarshalJSON(bs)
+}

index/upsidedown/store/boltdb/store.go

@@ -0,0 +1,184 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
+// following options:
+//
+// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
+//
+// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
+// operations in exchange of losing integrity guarantees if indexation aborts
+// without closing the index. Use it when rebuilding indexes from zero.
+package boltdb
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	store "github.com/blevesearch/upsidedown_store_api"
+	bolt "go.etcd.io/bbolt"
+)
+
+const (
+	Name                    = "boltdb"
+	defaultCompactBatchSize = 100
+)
+
+type Store struct {
+	path        string
+	bucket      string
+	db          *bolt.DB
+	noSync      bool
+	fillPercent float64
+	mo          store.MergeOperator
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
+
+	bucket, ok := config["bucket"].(string)
+	if !ok {
+		bucket = "bleve"
+	}
+
+	noSync, _ := config["nosync"].(bool)
+
+	fillPercent, ok := config["fillPercent"].(float64)
+	if !ok {
+		fillPercent = bolt.DefaultFillPercent
+	}
+
+	bo := &bolt.Options{}
+	ro, ok := config["read_only"].(bool)
+	if ok {
+		bo.ReadOnly = ro
+	}
+
+	if initialMmapSize, ok := config["initialMmapSize"].(int); ok {
+		bo.InitialMmapSize = initialMmapSize
+	} else if initialMmapSize, ok := config["initialMmapSize"].(float64); ok {
+		bo.InitialMmapSize = int(initialMmapSize)
+	}
+
+	db, err := bolt.Open(path, 0600, bo)
+	if err != nil {
+		return nil, err
+	}
+	db.NoSync = noSync
+
+	if !bo.ReadOnly {
+		err = db.Update(func(tx *bolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+
+			return err
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	rv := Store{
+		path:        path,
+		bucket:      bucket,
+		db:          db,
+		mo:          mo,
+		noSync:      noSync,
+		fillPercent: fillPercent,
+	}
+	return &rv, nil
+}
+
+func (bs *Store) Close() error {
+	return bs.db.Close()
+}
+
+func (bs *Store) Reader() (store.KVReader, error) {
+	tx, err := bs.db.Begin(false)
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{
+		store:  bs,
+		tx:     tx,
+		bucket: tx.Bucket([]byte(bs.bucket)),
+	}, nil
+}
+
+func (bs *Store) Writer() (store.KVWriter, error) {
+	return &Writer{
+		store: bs,
+	}, nil
+}
+
+func (bs *Store) Stats() json.Marshaler {
+	return &stats{
+		s: bs,
+	}
+}
+
+// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
+// Removing entries is a workaround for github issue #374.
+func (bs *Store) CompactWithBatchSize(batchSize int) error {
+	for {
+		cnt := 0
+		err := bs.db.Batch(func(tx *bolt.Tx) error {
+			c := tx.Bucket([]byte(bs.bucket)).Cursor()
+			prefix := []byte("d")
+
+			for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+				if bytes.Equal(v, []byte{0}) {
+					cnt++
+					if err := c.Delete(); err != nil {
+						return err
+					}
+					if cnt == batchSize {
+						break
+					}
+				}
+
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if cnt == 0 {
+			break
+		}
+	}
+	return nil
+}
+
+// Compact calls CompactWithBatchSize with a default batch size of 100.  This is a workaround
+// for github issue #374.
+func (bs *Store) Compact() error {
+	return bs.CompactWithBatchSize(defaultCompactBatchSize)
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/boltdb/store_test.go

@@ -0,0 +1,148 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !darwin || !arm64
+
+package boltdb
+
+import (
+	"os"
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+	bolt "go.etcd.io/bbolt"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, map[string]interface{}{"path": "test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.RemoveAll("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestBoltDBKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestBoltDBReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestBoltDBReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestBoltDBWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestBoltDBPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestBoltDBPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestBoltDBRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestBoltDBRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestBoltDBMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}
+
+func TestBoltDBConfig(t *testing.T) {
+	var tests = []struct {
+		in          map[string]interface{}
+		path        string
+		bucket      string
+		noSync      bool
+		fillPercent float64
+	}{
+		{
+			map[string]interface{}{"path": "test", "bucket": "mybucket", "nosync": true, "fillPercent": 0.75},
+			"test",
+			"mybucket",
+			true,
+			0.75,
+		},
+		{
+			map[string]interface{}{"path": "test"},
+			"test",
+			"bleve",
+			false,
+			bolt.DefaultFillPercent,
+		},
+	}
+
+	for _, test := range tests {
+		kv, err := New(nil, test.in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bs, ok := kv.(*Store)
+		if !ok {
+			t.Fatal("failed type assertion to *boltdb.Store")
+		}
+		if bs.path != test.path {
+			t.Fatalf("path: expected %q, got %q", test.path, bs.path)
+		}
+		if bs.bucket != test.bucket {
+			t.Fatalf("bucket: expected %q, got %q", test.bucket, bs.bucket)
+		}
+		if bs.noSync != test.noSync {
+			t.Fatalf("noSync: expected %t, got %t", test.noSync, bs.noSync)
+		}
+		if bs.fillPercent != test.fillPercent {
+			t.Fatalf("fillPercent: expected %f, got %f", test.fillPercent, bs.fillPercent)
+		}
+		cleanup(t, kv)
+	}
+}

index/upsidedown/store/boltdb/writer.go

@@ -0,0 +1,95 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"fmt"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w.store.mo)
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	return make([]byte, options.TotalBytes), w.NewBatch(), nil
+}
+
+func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {
+
+	emulatedBatch, ok := batch.(*store.EmulatedBatch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	tx, err := w.store.db.Begin(true)
+	if err != nil {
+		return
+	}
+	// defer function to ensure that once started,
+	// we either Commit tx or Rollback
+	defer func() {
+		// if nothing went wrong, commit
+		if err == nil {
+			// careful to catch error here too
+			err = tx.Commit()
+		} else {
+			// caller should see error that caused abort,
+			// not success or failure of Rollback itself
+			_ = tx.Rollback()
+		}
+	}()
+
+	bucket := tx.Bucket([]byte(w.store.bucket))
+	bucket.FillPercent = w.store.fillPercent
+
+	for k, mergeOps := range emulatedBatch.Merger.Merges {
+		kb := []byte(k)
+		existingVal := bucket.Get(kb)
+		mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			err = fmt.Errorf("merge operator returned failure")
+			return
+		}
+		err = bucket.Put(kb, mergedVal)
+		if err != nil {
+			return
+		}
+	}
+
+	for _, op := range emulatedBatch.Ops {
+		if op.V != nil {
+			err = bucket.Put(op.K, op.V)
+			if err != nil {
+				return
+			}
+		} else {
+			err = bucket.Delete(op.K)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func (w *Writer) Close() error {
+	return nil
+}