Adding upstream version 2.5.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

index/upsidedown/store/moss/batch.go

@@ -0,0 +1,87 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Batch struct {
+	store   *Store
+	merge   *store.EmulatedMerge
+	batch   moss.Batch
+	buf     []byte // Non-nil when using pre-alloc'ed / NewBatchEx().
+	bufUsed int
+}
+
+func (b *Batch) Set(key, val []byte) {
+	var err error
+	if b.buf != nil {
+		b.bufUsed += len(key) + len(val)
+		err = b.batch.AllocSet(key, val)
+	} else {
+		err = b.batch.Set(key, val)
+	}
+
+	if err != nil {
+		b.store.Logf("bleve moss batch.Set err: %v", err)
+	}
+}
+
+func (b *Batch) Delete(key []byte) {
+	var err error
+	if b.buf != nil {
+		b.bufUsed += len(key)
+		err = b.batch.AllocDel(key)
+	} else {
+		err = b.batch.Del(key)
+	}
+
+	if err != nil {
+		b.store.Logf("bleve moss batch.Delete err: %v", err)
+	}
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	if b.buf != nil {
+		b.bufUsed += len(key) + len(val)
+	}
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Reset() {
+	err := b.Close()
+	if err != nil {
+		b.store.Logf("bleve moss batch.Close err: %v", err)
+		return
+	}
+
+	batch, err := b.store.ms.NewBatch(0, 0)
+	if err == nil {
+		b.batch = batch
+		b.merge = store.NewEmulatedMerge(b.store.mo)
+		b.buf = nil
+		b.bufUsed = 0
+	}
+}
+
+func (b *Batch) Close() error {
+	b.merge = nil
+	err := b.batch.Close()
+	b.batch = nil
+	return err
+}

index/upsidedown/store/moss/iterator.go

@@ -0,0 +1,87 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+)
+
+type Iterator struct {
+	store *Store
+	ss    moss.Snapshot
+	iter  moss.Iterator
+	start []byte
+	end   []byte
+	k     []byte
+	v     []byte
+	err   error
+}
+
+func (x *Iterator) Seek(seekToKey []byte) {
+	_ = x.iter.SeekTo(seekToKey)
+
+	x.k, x.v, x.err = x.iter.Current()
+}
+
+func (x *Iterator) Next() {
+	_ = x.iter.Next()
+
+	x.k, x.v, x.err = x.iter.Current()
+}
+
+func (x *Iterator) Current() ([]byte, []byte, bool) {
+	return x.k, x.v, x.err == nil
+}
+
+func (x *Iterator) Key() []byte {
+	if x.err != nil {
+		return nil
+	}
+
+	return x.k
+}
+
+func (x *Iterator) Value() []byte {
+	if x.err != nil {
+		return nil
+	}
+
+	return x.v
+}
+
+func (x *Iterator) Valid() bool {
+	return x.err == nil
+}
+
+func (x *Iterator) Close() error {
+	var err error
+
+	x.ss = nil
+
+	if x.iter != nil {
+		err = x.iter.Close()
+		x.iter = nil
+	}
+
+	x.k = nil
+	x.v = nil
+	x.err = moss.ErrIteratorDone
+
+	return err
+}
+
+func (x *Iterator) current() {
+	x.k, x.v, x.err = x.iter.Current()
+}

index/upsidedown/store/moss/lower.go

@@ -0,0 +1,571 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package moss provides a KVStore implementation based on the
+// github.com/couchbase/moss library.
+
+package moss
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/couchbase/moss"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+func initLowerLevelStore(
+	config map[string]interface{},
+	lowerLevelStoreName string,
+	lowerLevelStoreConfig map[string]interface{},
+	lowerLevelMaxBatchSize uint64,
+	options moss.CollectionOptions,
+) (moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error) {
+	if lowerLevelStoreConfig == nil {
+		lowerLevelStoreConfig = map[string]interface{}{}
+	}
+
+	for k, v := range config {
+		_, exists := lowerLevelStoreConfig[k]
+		if !exists {
+			lowerLevelStoreConfig[k] = v
+		}
+	}
+
+	if lowerLevelStoreName == "mossStore" {
+		return InitMossStore(lowerLevelStoreConfig, options)
+	}
+
+	constructor := registry.KVStoreConstructorByName(lowerLevelStoreName)
+	if constructor == nil {
+		return nil, nil, nil, nil, fmt.Errorf("moss store, initLowerLevelStore,"+
+			" could not find lower level store: %s", lowerLevelStoreName)
+	}
+
+	kvStore, err := constructor(options.MergeOperator, lowerLevelStoreConfig)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	llStore := &llStore{
+		refs:     0,
+		config:   config,
+		llConfig: lowerLevelStoreConfig,
+		kvStore:  kvStore,
+		logf:     options.Log,
+	}
+
+	llUpdate := func(ssHigher moss.Snapshot) (ssLower moss.Snapshot, err error) {
+		return llStore.update(ssHigher, lowerLevelMaxBatchSize)
+	}
+
+	llSnapshot, err := llUpdate(nil)
+	if err != nil {
+		_ = kvStore.Close()
+		return nil, nil, nil, nil, err
+	}
+
+	return llSnapshot, llUpdate, kvStore, nil, nil // llStore.refs is now 1.
+}
+
+// ------------------------------------------------
+
+// llStore is a lower level store and provides ref-counting around a
+// bleve store.KVStore.
+type llStore struct {
+	kvStore store.KVStore
+
+	config   map[string]interface{}
+	llConfig map[string]interface{}
+
+	logf func(format string, a ...interface{})
+
+	m    sync.Mutex // Protects fields that follow.
+	refs int
+}
+
+// llSnapshot represents a lower-level snapshot, wrapping a bleve
+// store.KVReader, and implements the moss.Snapshot interface.
+type llSnapshot struct {
+	llStore        *llStore // Holds 1 refs on the llStore.
+	kvReader       store.KVReader
+	childSnapshots map[string]*llSnapshot
+
+	m    sync.Mutex // Protects fields that follow.
+	refs int
+}
+
+// llIterator represents a lower-level iterator, wrapping a bleve
+// store.KVIterator, and implements the moss.Iterator interface.
+type llIterator struct {
+	llSnapshot *llSnapshot // Holds 1 refs on the llSnapshot.
+
+	// Some lower-level KVReader implementations need a separate
+	// KVReader clone, due to KVReader single-threaded'ness.
+	kvReader store.KVReader
+
+	kvIterator store.KVIterator
+}
+
+type readerSource interface {
+	Reader() (store.KVReader, error)
+}
+
+// ------------------------------------------------
+
+func (s *llStore) addRef() *llStore {
+	s.m.Lock()
+	s.refs += 1
+	s.m.Unlock()
+
+	return s
+}
+
+func (s *llStore) decRef() {
+	s.m.Lock()
+	s.refs -= 1
+	if s.refs <= 0 {
+		err := s.kvStore.Close()
+		if err != nil {
+			s.logf("llStore kvStore.Close err: %v", err)
+		}
+	}
+	s.m.Unlock()
+}
+
+// update() mutates this lower level store with latest data from the
+// given higher level moss.Snapshot and returns a new moss.Snapshot
+// that the higher level can use which represents this lower level
+// store.
+func (s *llStore) update(ssHigher moss.Snapshot, maxBatchSize uint64) (
+	ssLower moss.Snapshot, err error,
+) {
+	if ssHigher != nil {
+		iter, err := ssHigher.StartIterator(nil, nil, moss.IteratorOptions{
+			IncludeDeletions: true,
+			SkipLowerLevel:   true,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			err = iter.Close()
+			if err != nil {
+				s.logf("llStore iter.Close err: %v", err)
+			}
+		}()
+
+		kvWriter, err := s.kvStore.Writer()
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			err = kvWriter.Close()
+			if err != nil {
+				s.logf("llStore kvWriter.Close err: %v", err)
+			}
+		}()
+
+		batch := kvWriter.NewBatch()
+
+		defer func() {
+			if batch != nil {
+				err = batch.Close()
+				if err != nil {
+					s.logf("llStore batch.Close err: %v", err)
+				}
+			}
+		}()
+
+		var readOptions moss.ReadOptions
+
+		i := uint64(0)
+		for {
+			if i%1000000 == 0 {
+				s.logf("llStore.update, i: %d", i)
+			}
+
+			ex, key, val, err := iter.CurrentEx()
+			if err == moss.ErrIteratorDone {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+
+			switch ex.Operation {
+			case moss.OperationSet:
+				batch.Set(key, val)
+
+			case moss.OperationDel:
+				batch.Delete(key)
+
+			case moss.OperationMerge:
+				val, err = ssHigher.Get(key, readOptions)
+				if err != nil {
+					return nil, err
+				}
+
+				if val != nil {
+					batch.Set(key, val)
+				} else {
+					batch.Delete(key)
+				}
+
+			default:
+				return nil, fmt.Errorf("moss store, update,"+
+					" unexpected operation, ex: %v", ex)
+			}
+
+			i++
+
+			err = iter.Next()
+			if err == moss.ErrIteratorDone {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+
+			if maxBatchSize > 0 && i%maxBatchSize == 0 {
+				err = kvWriter.ExecuteBatch(batch)
+				if err != nil {
+					return nil, err
+				}
+
+				err = batch.Close()
+				if err != nil {
+					return nil, err
+				}
+
+				batch = kvWriter.NewBatch()
+			}
+		}
+
+		if i > 0 {
+			s.logf("llStore.update, ExecuteBatch,"+
+				" path: %s, total: %d, start", s.llConfig["path"], i)
+
+			err = kvWriter.ExecuteBatch(batch)
+			if err != nil {
+				return nil, err
+			}
+
+			s.logf("llStore.update, ExecuteBatch,"+
+				" path: %s: total: %d, done", s.llConfig["path"], i)
+		}
+	}
+
+	kvReader, err := s.kvStore.Reader()
+	if err != nil {
+		return nil, err
+	}
+
+	s.logf("llStore.update, new reader")
+
+	return &llSnapshot{
+		llStore:  s.addRef(),
+		kvReader: kvReader,
+		refs:     1,
+	}, nil
+}
+
+// ------------------------------------------------
+
+func (llss *llSnapshot) addRef() *llSnapshot {
+	llss.m.Lock()
+	llss.refs += 1
+	llss.m.Unlock()
+
+	return llss
+}
+
+func (llss *llSnapshot) decRef() {
+	llss.m.Lock()
+	llss.refs -= 1
+	if llss.refs <= 0 {
+		if llss.kvReader != nil {
+			err := llss.kvReader.Close()
+			if err != nil {
+				llss.llStore.logf("llSnapshot kvReader.Close err: %v", err)
+			}
+
+			llss.kvReader = nil
+		}
+
+		if llss.llStore != nil {
+			llss.llStore.decRef()
+			llss.llStore = nil
+		}
+	}
+	llss.m.Unlock()
+}
+
+// ChildCollectionNames returns an array of child collection name strings.
+func (llss *llSnapshot) ChildCollectionNames() ([]string, error) {
+	childCollections := make([]string, len(llss.childSnapshots))
+	idx := 0
+	for name := range llss.childSnapshots {
+		childCollections[idx] = name
+		idx++
+	}
+	return childCollections, nil
+}
+
+// ChildCollectionSnapshot returns a Snapshot on a given child
+// collection by its name.
+func (llss *llSnapshot) ChildCollectionSnapshot(childCollectionName string) (
+	moss.Snapshot, error,
+) {
+	childSnapshot, exists := llss.childSnapshots[childCollectionName]
+	if !exists {
+		return nil, nil
+	}
+	childSnapshot.addRef()
+	return childSnapshot, nil
+}
+
+func (llss *llSnapshot) Close() error {
+	llss.decRef()
+
+	return nil
+}
+
+func (llss *llSnapshot) Get(key []byte,
+	readOptions moss.ReadOptions,
+) ([]byte, error) {
+	rs, ok := llss.kvReader.(readerSource)
+	if ok {
+		r2, err := rs.Reader()
+		if err != nil {
+			return nil, err
+		}
+
+		val, err := r2.Get(key)
+
+		_ = r2.Close()
+
+		return val, err
+	}
+
+	return llss.kvReader.Get(key)
+}
+
+func (llss *llSnapshot) StartIterator(
+	startKeyInclusive, endKeyExclusive []byte,
+	iteratorOptions moss.IteratorOptions,
+) (moss.Iterator, error) {
+	rs, ok := llss.kvReader.(readerSource)
+	if ok {
+		r2, err := rs.Reader()
+		if err != nil {
+			return nil, err
+		}
+
+		i2 := r2.RangeIterator(startKeyInclusive, endKeyExclusive)
+
+		return &llIterator{llSnapshot: llss.addRef(), kvReader: r2, kvIterator: i2}, nil
+	}
+
+	i := llss.kvReader.RangeIterator(startKeyInclusive, endKeyExclusive)
+
+	return &llIterator{llSnapshot: llss.addRef(), kvReader: nil, kvIterator: i}, nil
+}
+
+// ------------------------------------------------
+
+func (lli *llIterator) Close() error {
+	var err0 error
+	if lli.kvIterator != nil {
+		err0 = lli.kvIterator.Close()
+		lli.kvIterator = nil
+	}
+
+	var err1 error
+	if lli.kvReader != nil {
+		err1 = lli.kvReader.Close()
+		lli.kvReader = nil
+	}
+
+	lli.llSnapshot.decRef()
+	lli.llSnapshot = nil
+
+	if err0 != nil {
+		return err0
+	}
+
+	if err1 != nil {
+		return err1
+	}
+
+	return nil
+}
+
+func (lli *llIterator) Next() error {
+	lli.kvIterator.Next()
+
+	return nil
+}
+
+func (lli *llIterator) SeekTo(k []byte) error {
+	lli.kvIterator.Seek(k)
+
+	return nil
+}
+
+func (lli *llIterator) Current() (key, val []byte, err error) {
+	key, val, ok := lli.kvIterator.Current()
+	if !ok {
+		return nil, nil, moss.ErrIteratorDone
+	}
+
+	return key, val, nil
+}
+
+func (lli *llIterator) CurrentEx() (
+	entryEx moss.EntryEx, key, val []byte, err error,
+) {
+	return moss.EntryEx{}, nil, nil, moss.ErrUnimplemented
+}
+
+// ------------------------------------------------
+
+func InitMossStore(config map[string]interface{}, options moss.CollectionOptions) (
+	moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error,
+) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
+	}
+	if path == "" {
+		return nil, nil, nil, nil, os.ErrInvalid
+	}
+
+	err := os.MkdirAll(path, 0o700)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("lower: InitMossStore mkdir,"+
+			" path: %s, err: %v", path, err)
+	}
+
+	storeOptions := moss.StoreOptions{
+		CollectionOptions: options,
+	}
+	v, ok := config["mossStoreOptions"]
+	if ok {
+		b, err := util.MarshalJSON(v) // Convert from map[string]interface{}.
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+
+		err = util.UnmarshalJSON(b, &storeOptions)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
+
+	s, err := moss.OpenStore(path, storeOptions)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("lower: moss.OpenStore,"+
+			" path: %s, err: %v", path, err)
+	}
+
+	sw := &mossStoreWrapper{s: s}
+
+	llUpdate := func(ssHigher moss.Snapshot) (moss.Snapshot, error) {
+		ss, err := sw.s.Persist(ssHigher, moss.StorePersistOptions{
+			CompactionConcern: moss.CompactionAllow,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sw.AddRef() // Ref-count to be owned by snapshot wrapper.
+
+		return moss.NewSnapshotWrapper(ss, sw), nil
+	}
+
+	llSnapshot, err := llUpdate(nil)
+	if err != nil {
+		_ = s.Close()
+		return nil, nil, nil, nil, err
+	}
+
+	llStats := func() map[string]interface{} {
+		stats, err := s.Stats()
+		if err != nil {
+			return nil
+		}
+		return stats
+	}
+
+	return llSnapshot, llUpdate, sw, llStats, nil
+}
+
+// mossStoreWrapper implements the bleve.index.store.KVStore
+// interface, but only barely enough to allow it to be passed around
+// as a lower-level store.  Advanced apps will likely cast the
+// mossStoreWrapper to access the Actual() method.
+type mossStoreWrapper struct {
+	m    sync.Mutex
+	refs int
+	s    *moss.Store
+}
+
+func (w *mossStoreWrapper) AddRef() {
+	w.m.Lock()
+	w.refs++
+	w.m.Unlock()
+}
+
+func (w *mossStoreWrapper) Close() (err error) {
+	w.m.Lock()
+	w.refs--
+	if w.refs <= 0 {
+		err = w.s.Close()
+		w.s = nil
+	}
+	w.m.Unlock()
+	return err
+}
+
+func (w *mossStoreWrapper) Reader() (store.KVReader, error) {
+	return nil, fmt.Errorf("unexpected")
+}
+
+func (w *mossStoreWrapper) Writer() (store.KVWriter, error) {
+	return nil, fmt.Errorf("unexpected")
+}
+
+func (w *mossStoreWrapper) Actual() *moss.Store {
+	w.m.Lock()
+	rv := w.s
+	w.m.Unlock()
+	return rv
+}
+
+func (w *mossStoreWrapper) histograms() string {
+	var rv string
+	w.m.Lock()
+	if w.s != nil {
+		rv = w.s.Histograms().String()
+	}
+	w.m.Unlock()
+	return rv
+}

index/upsidedown/store/moss/lower_test.go

@@ -0,0 +1,103 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"os"
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func openWithLower(t *testing.T, mo store.MergeOperator) (string, store.KVStore) {
+	tmpDir, _ := os.MkdirTemp("", "mossStore")
+
+	config := map[string]interface{}{
+		"path":                    tmpDir,
+		"mossLowerLevelStoreName": "mossStore",
+	}
+
+	rv, err := New(mo, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tmpDir, rv
+}
+
+func cleanupWithLower(t *testing.T, s store.KVStore, tmpDir string) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.RemoveAll(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMossWithLowerKVCrud(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestMossWithLowerReaderIsolation(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestMossWithLowerReaderOwnsGetBytes(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestMossWithLowerWriterOwnsBytes(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestMossWithLowerPrefixIterator(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestMossWithLowerPrefixIteratorSeek(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestMossWithLowerRangeIterator(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestMossWithLowerRangeIteratorSeek(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestMossWithLowerMerge(t *testing.T) {
+	tmpDir, s := openWithLower(t, &test.TestMergeCounter{})
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/moss/reader.go

@@ -0,0 +1,97 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Reader struct {
+	store *Store
+	ss    moss.Snapshot
+}
+
+func (r *Reader) Get(k []byte) (v []byte, err error) {
+	v, err = r.ss.Get(k, moss.ReadOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if v != nil {
+		return append(make([]byte, 0, len(v)), v...), nil
+	}
+	return nil, nil
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
+	kEnd := incrementBytes(k)
+
+	iter, err := r.ss.StartIterator(k, kEnd, moss.IteratorOptions{})
+	if err != nil {
+		return nil
+	}
+
+	rv := &Iterator{
+		store: r.store,
+		ss:    r.ss,
+		iter:  iter,
+		start: k,
+		end:   kEnd,
+	}
+
+	rv.current()
+
+	return rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	iter, err := r.ss.StartIterator(start, end, moss.IteratorOptions{})
+	if err != nil {
+		return nil
+	}
+
+	rv := &Iterator{
+		store: r.store,
+		ss:    r.ss,
+		iter:  iter,
+		start: start,
+		end:   end,
+	}
+
+	rv.current()
+
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.ss.Close()
+}
+
+func incrementBytes(in []byte) []byte {
+	rv := make([]byte, len(in))
+	copy(rv, in)
+	for i := len(rv) - 1; i >= 0; i-- {
+		rv[i] = rv[i] + 1
+		if rv[i] != 0 {
+			return rv // didn't overflow, so stop
+		}
+	}
+	return nil // overflowed
+}

index/upsidedown/store/moss/stats.go

@@ -0,0 +1,58 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type stats struct {
+	s *Store
+}
+
+func (s *stats) statsMap() map[string]interface{} {
+	ms := map[string]interface{}{}
+
+	var err error
+	ms["moss"], err = s.s.ms.Stats()
+	if err != nil {
+		return ms
+	}
+
+	if s.s.llstore != nil {
+		if o, ok := s.s.llstore.(store.KVStoreStats); ok {
+			ms["kv"] = o.StatsMap()
+		}
+	}
+
+	_, exists := ms["kv"]
+	if !exists && s.s.llstats != nil {
+		ms["kv"] = s.s.llstats()
+	}
+
+	if msw, ok := s.s.llstore.(*mossStoreWrapper); ok {
+		ms["store_histograms"] = msw.histograms()
+	}
+
+	ms["coll_histograms"] = s.s.ms.Histograms().String()
+
+	return ms
+}
+
+func (s *stats) MarshalJSON() ([]byte, error) {
+	m := s.statsMap()
+	return util.MarshalJSON(m)
+}

index/upsidedown/store/moss/store.go

@@ -0,0 +1,231 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package moss provides a KVStore implementation based on the
+// github.com/couchbase/moss library.
+
+package moss
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/couchbase/moss"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+// RegistryCollectionOptions should be treated as read-only after
+// process init()'ialization.
+var RegistryCollectionOptions = map[string]moss.CollectionOptions{}
+
+const Name = "moss"
+
+type Store struct {
+	m       sync.Mutex
+	ms      moss.Collection
+	mo      store.MergeOperator
+	llstore store.KVStore // May be nil.
+	llstats statsFunc     // May be nil.
+
+	s      *stats
+	config map[string]interface{}
+}
+
+type statsFunc func() map[string]interface{}
+
+// New initializes a moss storage with values from the optional
+// config["mossCollectionOptions"] (a JSON moss.CollectionOptions).
+// Next, values from the RegistryCollectionOptions, named by the
+// optional config["mossCollectionOptionsName"], take precedence.
+// Finally, base case defaults are taken from
+// moss.DefaultCollectionOptions.
+func New(mo store.MergeOperator, config map[string]interface{}) (
+	store.KVStore, error) {
+	options := moss.DefaultCollectionOptions // Copy.
+
+	v, ok := config["mossCollectionOptionsName"]
+	if ok {
+		name, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not parse config[mossCollectionOptionsName]: %v", v)
+		}
+
+		options, ok = RegistryCollectionOptions[name] // Copy.
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not find RegistryCollectionOptions, name: %s", name)
+		}
+	}
+
+	options.MergeOperator = mo
+	options.DeferredSort = true
+
+	v, ok = config["mossCollectionOptions"]
+	if ok {
+		b, err := util.MarshalJSON(v) // Convert from map[string]interface{}.
+		if err != nil {
+			return nil, fmt.Errorf("moss store,"+
+				" could not marshal config[mossCollectionOptions]: %v, err: %v", v, err)
+		}
+
+		err = util.UnmarshalJSON(b, &options)
+		if err != nil {
+			return nil, fmt.Errorf("moss store,"+
+				" could not unmarshal config[mossCollectionOptions]: %v, err: %v", v, err)
+		}
+	}
+
+	// --------------------------------------------------
+
+	if options.Log == nil || options.Debug <= 0 {
+		options.Log = func(format string, a ...interface{}) {}
+	}
+
+	// --------------------------------------------------
+
+	mossLowerLevelStoreName := ""
+	v, ok = config["mossLowerLevelStoreName"]
+	if ok {
+		mossLowerLevelStoreName, ok = v.(string)
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not parse config[mossLowerLevelStoreName]: %v", v)
+		}
+	}
+
+	var llStore store.KVStore
+	var llStats statsFunc
+
+	if options.LowerLevelInit == nil &&
+		options.LowerLevelUpdate == nil &&
+		mossLowerLevelStoreName != "" {
+		mossLowerLevelStoreConfig := map[string]interface{}{}
+		v, ok := config["mossLowerLevelStoreConfig"]
+		if ok {
+			mossLowerLevelStoreConfig, ok = v.(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("moss store, initLowerLevelStore,"+
+					" could parse mossLowerLevelStoreConfig: %v", v)
+			}
+		}
+
+		mossLowerLevelMaxBatchSize := uint64(0)
+		v, ok = config["mossLowerLevelMaxBatchSize"]
+		if ok {
+			mossLowerLevelMaxBatchSizeF, ok := v.(float64)
+			if !ok {
+				return nil, fmt.Errorf("moss store,"+
+					" could not parse config[mossLowerLevelMaxBatchSize]: %v", v)
+			}
+
+			mossLowerLevelMaxBatchSize = uint64(mossLowerLevelMaxBatchSizeF)
+		}
+
+		lowerLevelInit, lowerLevelUpdate, lowerLevelStore, lowerLevelStats, err :=
+			initLowerLevelStore(config,
+				mossLowerLevelStoreName,
+				mossLowerLevelStoreConfig,
+				mossLowerLevelMaxBatchSize,
+				options)
+		if err != nil {
+			return nil, err
+		}
+
+		options.LowerLevelInit = lowerLevelInit
+		options.LowerLevelUpdate = lowerLevelUpdate
+
+		llStore = lowerLevelStore
+		llStats = lowerLevelStats
+	}
+
+	// --------------------------------------------------
+
+	ms, err := moss.NewCollection(options)
+	if err != nil {
+		return nil, err
+	}
+	err = ms.Start()
+	if err != nil {
+		return nil, err
+	}
+	rv := Store{
+		ms:      ms,
+		mo:      mo,
+		llstore: llStore,
+		llstats: llStats,
+		config:  config,
+	}
+	rv.s = &stats{s: &rv}
+	return &rv, nil
+}
+
+func (s *Store) Close() error {
+	if val, ok := s.config["mossAbortCloseEnabled"]; ok {
+		if v, ok := val.(bool); ok && v {
+			if msw, ok := s.llstore.(*mossStoreWrapper); ok {
+				if s := msw.Actual(); s != nil {
+					_ = s.CloseEx(moss.StoreCloseExOptions{Abort: true})
+				}
+			}
+		}
+	}
+	return s.ms.Close()
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	ss, err := s.ms.Snapshot()
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{ss: ss}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	return &Writer{s: s}, nil
+}
+
+func (s *Store) Logf(fmt string, args ...interface{}) {
+	options := s.ms.Options()
+	if options.Log != nil {
+		options.Log(fmt, args...)
+	}
+}
+
+func (s *Store) Stats() json.Marshaler {
+	return s.s
+}
+
+func (s *Store) StatsMap() map[string]interface{} {
+	return s.s.statsMap()
+}
+
+func (s *Store) LowerLevelStore() store.KVStore {
+	return s.llstore
+}
+
+func (s *Store) Collection() moss.Collection {
+	return s.ms
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/moss/store_test.go

@@ -0,0 +1,91 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMossKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestMossReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestMossReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestMossWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestMossPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestMossPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestMossRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestMossRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestMossMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/moss/writer.go

@@ -0,0 +1,97 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"fmt"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+
+	"github.com/couchbase/moss"
+)
+
+type Writer struct {
+	s *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	b, err := w.s.ms.NewBatch(0, 0)
+	if err != nil {
+		return nil
+	}
+
+	return &Batch{
+		store: w.s,
+		merge: store.NewEmulatedMerge(w.s.mo),
+		batch: b,
+	}
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) (
+	[]byte, store.KVBatch, error) {
+	numOps := options.NumSets + options.NumDeletes + options.NumMerges
+
+	b, err := w.s.ms.NewBatch(numOps, options.TotalBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	buf, err := b.Alloc(options.TotalBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return buf, &Batch{
+		store:   w.s,
+		merge:   store.NewEmulatedMerge(w.s.mo),
+		batch:   b,
+		buf:     buf,
+		bufUsed: 0,
+	}, nil
+}
+
+func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {
+	batch, ok := b.(*Batch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	for kStr, mergeOps := range batch.merge.Merges {
+		for _, v := range mergeOps {
+			if batch.buf != nil {
+				kLen := len(kStr)
+				vLen := len(v)
+				kBuf := batch.buf[batch.bufUsed : batch.bufUsed+kLen]
+				vBuf := batch.buf[batch.bufUsed+kLen : batch.bufUsed+kLen+vLen]
+				copy(kBuf, kStr)
+				copy(vBuf, v)
+				batch.bufUsed += kLen + vLen
+				err = batch.batch.AllocMerge(kBuf, vBuf)
+			} else {
+				err = batch.batch.Merge([]byte(kStr), v)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return w.s.ms.ExecuteBatch(batch.batch, moss.WriteOptions{})
+}
+
+func (w *Writer) Close() error {
+	w.s = nil
+	return nil
+}