Adding upstream version 2.5.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

index/upsidedown/analysis.go

@@ -0,0 +1,129 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type IndexRow interface {
+	KeySize() int
+	KeyTo([]byte) (int, error)
+	Key() []byte
+
+	ValueSize() int
+	ValueTo([]byte) (int, error)
+	Value() []byte
+}
+
+type AnalysisResult struct {
+	DocID string
+	Rows  []IndexRow
+}
+
+func (udc *UpsideDownCouch) Analyze(d index.Document) *AnalysisResult {
+	return udc.analyze(d)
+}
+
+func (udc *UpsideDownCouch) analyze(d index.Document) *AnalysisResult {
+	rv := &AnalysisResult{
+		DocID: d.ID(),
+		Rows:  make([]IndexRow, 0, 100),
+	}
+
+	docIDBytes := []byte(d.ID())
+
+	// track our back index entries
+	backIndexStoredEntries := make([]*BackIndexStoreEntry, 0)
+
+	// information we collate as we merge fields with same name
+	fieldTermFreqs := make(map[uint16]index.TokenFrequencies)
+	fieldLengths := make(map[uint16]int)
+	fieldIncludeTermVectors := make(map[uint16]bool)
+	fieldNames := make(map[uint16]string)
+
+	analyzeField := func(field index.Field, storable bool) {
+		fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name())
+		if newFieldRow != nil {
+			rv.Rows = append(rv.Rows, newFieldRow)
+		}
+		fieldNames[fieldIndex] = field.Name()
+
+		if field.Options().IsIndexed() {
+			field.Analyze()
+			fieldLength := field.AnalyzedLength()
+			tokenFreqs := field.AnalyzedTokenFrequencies()
+			existingFreqs := fieldTermFreqs[fieldIndex]
+			if existingFreqs == nil {
+				fieldTermFreqs[fieldIndex] = tokenFreqs
+			} else {
+				existingFreqs.MergeAll(field.Name(), tokenFreqs)
+				fieldTermFreqs[fieldIndex] = existingFreqs
+			}
+			fieldLengths[fieldIndex] += fieldLength
+			fieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors()
+		}
+
+		if storable && field.Options().IsStored() {
+			rv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries)
+		}
+	}
+
+	// walk all the fields, record stored fields now
+	// place information about indexed fields into map
+	// this collates information across fields with
+	// same names (arrays)
+	d.VisitFields(func(field index.Field) {
+		analyzeField(field, true)
+	})
+
+	if d.HasComposite() {
+		for fieldIndex, tokenFreqs := range fieldTermFreqs {
+			// see if any of the composite fields need this
+			d.VisitComposite(func(field index.CompositeField) {
+				field.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs)
+			})
+		}
+
+		d.VisitComposite(func(field index.CompositeField) {
+			analyzeField(field, false)
+		})
+	}
+
+	rowsCapNeeded := len(rv.Rows) + 1
+	for _, tokenFreqs := range fieldTermFreqs {
+		rowsCapNeeded += len(tokenFreqs)
+	}
+
+	rv.Rows = append(make([]IndexRow, 0, rowsCapNeeded), rv.Rows...)
+
+	backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs))
+
+	// walk through the collated information and process
+	// once for each indexed field (unique name)
+	for fieldIndex, tokenFreqs := range fieldTermFreqs {
+		fieldLength := fieldLengths[fieldIndex]
+		includeTermVectors := fieldIncludeTermVectors[fieldIndex]
+
+		// encode this field
+		rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries)
+	}
+
+	// build the back index row
+	backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries)
+	rv.Rows = append(rv.Rows, backIndexRow)
+
+	return rv
+}

index/upsidedown/analysis_test.go

@@ -0,0 +1,115 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/null"
+	"github.com/blevesearch/bleve/v2/registry"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+func TestAnalysisBug328(t *testing.T) {
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed(standard.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := document.NewDocument("1")
+	f := document.NewTextFieldCustom("title", nil, []byte("bleve"), index.IndexField|index.IncludeTermVectors, analyzer)
+	d.AddField(f)
+	f = document.NewTextFieldCustom("body", nil, []byte("bleve"), index.IndexField|index.IncludeTermVectors, analyzer)
+	d.AddField(f)
+	cf := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, []string{}, index.IndexField|index.IncludeTermVectors)
+	d.AddField(cf)
+
+	rv := idx.(*UpsideDownCouch).analyze(d)
+	fieldIndexes := make(map[uint16]string)
+	for _, row := range rv.Rows {
+		if row, ok := row.(*FieldRow); ok {
+			fieldIndexes[row.index] = row.name
+		}
+		if row, ok := row.(*TermFrequencyRow); ok && string(row.term) == "bleve" {
+			for _, vec := range row.vectors {
+				if vec.field != row.field {
+					if fieldIndexes[row.field] != "_all" {
+						t.Errorf("row named %s field %d - vector field %d", fieldIndexes[row.field], row.field, vec.field)
+					}
+				}
+			}
+		}
+	}
+}
+
+func BenchmarkAnalyze(b *testing.B) {
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed(standard.Name)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	d := document.NewDocument("1")
+	f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
+	d.AddField(f)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		rv := idx.(*UpsideDownCouch).analyze(d)
+		if len(rv.Rows) < 92 || len(rv.Rows) > 93 {
+			b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows))
+		}
+	}
+}
+
+var bleveWikiArticle1K = []byte(`Boiling liquid expanding vapor explosion
+From Wikipedia, the free encyclopedia
+See also: Boiler explosion and Steam explosion
+
+Flames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire.
+
+This article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013)
+A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1]
+Contents  [hide]
+1 Mechanism
+1.1 Water example
+1.2 BLEVEs without chemical reactions
+2 Fires
+3 Incidents
+4 Safety measures
+5 See also
+6 References
+7 External links
+Mechanism[edit]
+
+This section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013)
+There are three characteristics of liquids which are relevant to the discussion of a BLEVE:`)

index/upsidedown/benchmark_all.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+BENCHMARKS=`grep "func Benchmark" *_test.go  | sed 's/.*func //' | sed s/\(.*{//`
+
+for BENCHMARK in $BENCHMARKS
+do
+	go test -v -run=xxx -bench=^$BENCHMARK$ -benchtime=10s -tags 'forestdb leveldb'  | grep -v ok | grep -v PASS
+done

index/upsidedown/benchmark_boltdb_test.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb"
+)
+
+var boltTestConfig = map[string]interface{}{
+	"path": "test",
+}
+
+func BenchmarkBoltDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 1)
+}
+
+func BenchmarkBoltDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 2)
+}
+
+func BenchmarkBoltDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, boltdb.Name, boltTestConfig, DestroyTest, 4)
+}
+
+// batches
+
+func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 10)
+}
+
+func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 10)
+}
+
+func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 10)
+}
+
+func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 100)
+}
+
+func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 100)
+}
+
+func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 100)
+}
+
+func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 1, 1000)
+}
+
+func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 2, 1000)
+}
+
+func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, boltdb.Name, boltTestConfig, DestroyTest, 4, 1000)
+}

index/upsidedown/benchmark_common_test.go

@@ -0,0 +1,149 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	_ "github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/registry"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+var benchmarkDocBodies = []string{
+	"A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.",
+	"A boiler explosion is a catastrophic failure of a boiler. As seen today, boiler explosions are of two kinds. One kind is a failure of the pressure parts of the steam and water sides. There can be many different causes, such as failure of the safety valve, corrosion of critical parts of the boiler, or low water level. Corrosion along the edges of lap joints was a common cause of early boiler explosions.",
+	"A boiler is a closed vessel in which water or other fluid is heated. The fluid does not necessarily boil. (In North America the term \"furnace\" is normally used if the purpose is not actually to boil the fluid.) The heated or vaporized fluid exits the boiler for use in various processes or heating applications,[1][2] including central heating, boiler-based power generation, cooking, and sanitation.",
+	"A pressure vessel is a closed container designed to hold gases or liquids at a pressure substantially different from the ambient pressure.",
+	"Pressure (symbol: p or P) is the ratio of force to the area over which that force is distributed.",
+	"Liquid is one of the four fundamental states of matter (the others being solid, gas, and plasma), and is the only state with a definite volume but no fixed shape.",
+	"The boiling point of a substance is the temperature at which the vapor pressure of the liquid equals the pressure surrounding the liquid[1][2] and the liquid changes into a vapor.",
+	"Vapor pressure or equilibrium vapor pressure is defined as the pressure exerted by a vapor in thermodynamic equilibrium with its condensed phases (solid or liquid) at a given temperature in a closed system.",
+	"Industrial gases are a group of gases that are specifically manufactured for use in a wide range of industries, which include oil and gas, petrochemicals, chemicals, power, mining, steelmaking, metals, environmental protection, medicine, pharmaceuticals, biotechnology, food, water, fertilizers, nuclear power, electronics and aerospace.",
+	"The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.",
+}
+
+type KVStoreDestroy func() error
+
+func DestroyTest() error {
+	return os.RemoveAll("test")
+}
+
+func CommonBenchmarkIndex(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers int) {
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed("standard")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	indexDocument := document.NewDocument("").
+		AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))
+
+	b.ResetTimer()
+	b.StopTimer()
+	for i := 0; i < b.N; i++ {
+		analysisQueue := index.NewAnalysisQueue(analysisWorkers)
+		idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		err = idx.Open()
+		if err != nil {
+			b.Fatal(err)
+		}
+		indexDocument.SetID(strconv.Itoa(i))
+		// just time the indexing portion
+		b.StartTimer()
+		err = idx.Update(indexDocument)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.StopTimer()
+		err = idx.Close()
+		if err != nil {
+			b.Fatal(err)
+		}
+		err = destroy()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue.Close()
+	}
+}
+
+func CommonBenchmarkIndexBatch(b *testing.B, storeName string, storeConfig map[string]interface{}, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed("standard")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	b.StopTimer()
+	for i := 0; i < b.N; i++ {
+
+		analysisQueue := index.NewAnalysisQueue(analysisWorkers)
+		idx, err := NewUpsideDownCouch(storeName, storeConfig, analysisQueue)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		err = idx.Open()
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		b.StartTimer()
+		batch := index.NewBatch()
+		for j := 0; j < 1000; j++ {
+			if j%batchSize == 0 {
+				if len(batch.IndexOps) > 0 {
+					err := idx.Batch(batch)
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+				batch = index.NewBatch()
+			}
+			indexDocument := document.NewDocument("").
+				AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
+			indexDocument.SetID(strconv.Itoa(i) + "-" + strconv.Itoa(j))
+			batch.Update(indexDocument)
+		}
+		// close last batch
+		if len(batch.IndexOps) > 0 {
+			err := idx.Batch(batch)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StopTimer()
+		err = idx.Close()
+		if err != nil {
+			b.Fatal(err)
+		}
+		err = destroy()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue.Close()
+	}
+}

index/upsidedown/benchmark_gtreap_test.go

@@ -0,0 +1,71 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap"
+)
+
+func BenchmarkGTreapIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 1)
+}
+
+func BenchmarkGTreapIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 2)
+}
+
+func BenchmarkGTreapIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, gtreap.Name, nil, DestroyTest, 4)
+}
+
+// batches
+
+func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 10)
+}
+
+func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 10)
+}
+
+func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 10)
+}
+
+func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 100)
+}
+
+func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 100)
+}
+
+func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 100)
+}
+
+func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 1, 1000)
+}
+
+func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 2, 1000)
+}
+
+func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, gtreap.Name, nil, DestroyTest, 4, 1000)
+}

index/upsidedown/benchmark_null_test.go

@@ -0,0 +1,71 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/null"
+)
+
+func BenchmarkNullIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 1)
+}
+
+func BenchmarkNullIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 2)
+}
+
+func BenchmarkNullIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, null.Name, nil, DestroyTest, 4)
+}
+
+// batches
+
+func BenchmarkNullIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 10)
+}
+
+func BenchmarkNullIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 10)
+}
+
+func BenchmarkNullIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 10)
+}
+
+func BenchmarkNullIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 100)
+}
+
+func BenchmarkNullIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 100)
+}
+
+func BenchmarkNullIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 100)
+}
+
+func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 1, 1000)
+}
+
+func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 2, 1000)
+}
+
+func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, null.Name, nil, DestroyTest, 4, 1000)
+}

index/upsidedown/dump.go

@@ -0,0 +1,174 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"bytes"
+	"sort"
+
+	"github.com/blevesearch/upsidedown_store_api"
+)
+
+// the functions in this file are only intended to be used by
+// the bleve_dump utility and the debug http handlers
+// if your application relies on them, you're doing something wrong
+// they may change or be removed at any time
+
+func dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) {
+	start := prefix
+	if start == nil {
+		start = []byte{0}
+	}
+	it := kvreader.PrefixIterator(start)
+	defer func() {
+		cerr := it.Close()
+		if cerr != nil {
+			rv <- cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		ck := make([]byte, len(key))
+		copy(ck, key)
+		cv := make([]byte, len(val))
+		copy(cv, val)
+		row, err := ParseFromKeyValue(ck, cv)
+		if err != nil {
+			rv <- err
+			return
+		}
+		rv <- row
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+}
+
+func dumpRange(kvreader store.KVReader, rv chan interface{}, start, end []byte) {
+	it := kvreader.RangeIterator(start, end)
+	defer func() {
+		cerr := it.Close()
+		if cerr != nil {
+			rv <- cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		ck := make([]byte, len(key))
+		copy(ck, key)
+		cv := make([]byte, len(val))
+		copy(cv, val)
+		row, err := ParseFromKeyValue(ck, cv)
+		if err != nil {
+			rv <- err
+			return
+		}
+		rv <- row
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+}
+
+func (i *IndexReader) DumpAll() chan interface{} {
+	rv := make(chan interface{})
+	go func() {
+		defer close(rv)
+		dumpRange(i.kvreader, rv, nil, nil)
+	}()
+	return rv
+}
+
+func (i *IndexReader) DumpFields() chan interface{} {
+	rv := make(chan interface{})
+	go func() {
+		defer close(rv)
+		dumpPrefix(i.kvreader, rv, []byte{'f'})
+	}()
+	return rv
+}
+
+type keyset [][]byte
+
+func (k keyset) Len() int           { return len(k) }
+func (k keyset) Swap(i, j int)      { k[i], k[j] = k[j], k[i] }
+func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 }
+
+// DumpDoc returns all rows in the index related to this doc id
+func (i *IndexReader) DumpDoc(id string) chan interface{} {
+	idBytes := []byte(id)
+
+	rv := make(chan interface{})
+
+	go func() {
+		defer close(rv)
+
+		back, err := backIndexRowForDoc(i.kvreader, []byte(id))
+		if err != nil {
+			rv <- err
+			return
+		}
+
+		// no such doc
+		if back == nil {
+			return
+		}
+		// build sorted list of term keys
+		keys := make(keyset, 0)
+		for _, entry := range back.termsEntries {
+			for i := range entry.Terms {
+				tfr := NewTermFrequencyRow([]byte(entry.Terms[i]), uint16(*entry.Field), idBytes, 0, 0)
+				key := tfr.Key()
+				keys = append(keys, key)
+			}
+		}
+		sort.Sort(keys)
+
+		// first add all the stored rows
+		storedRowPrefix := NewStoredRow(idBytes, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc()
+		dumpPrefix(i.kvreader, rv, storedRowPrefix)
+
+		// now walk term keys in order and add them as well
+		if len(keys) > 0 {
+			it := i.kvreader.RangeIterator(keys[0], nil)
+			defer func() {
+				cerr := it.Close()
+				if cerr != nil {
+					rv <- cerr
+				}
+			}()
+
+			for _, key := range keys {
+				it.Seek(key)
+				rkey, rval, valid := it.Current()
+				if !valid {
+					break
+				}
+				rck := make([]byte, len(rkey))
+				copy(rck, key)
+				rcv := make([]byte, len(rval))
+				copy(rcv, rval)
+				row, err := ParseFromKeyValue(rck, rcv)
+				if err != nil {
+					rv <- err
+					return
+				}
+				rv <- row
+			}
+		}
+	}()
+
+	return rv
+}

index/upsidedown/dump_test.go

@@ -0,0 +1,155 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"testing"
+	"time"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb"
+	index "github.com/blevesearch/bleve_index_api"
+
+	"github.com/blevesearch/bleve/v2/document"
+)
+
+func TestDump(t *testing.T) {
+	defer func() {
+		err := DestroyTest()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	reader, err := idx.Reader()
+	if err != nil {
+		t.Fatal(err)
+	}
+	docCount, err := reader.DocCount()
+	if err != nil {
+		t.Error(err)
+	}
+	if docCount != expectedCount {
+		t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), index.IndexField|index.StoreField))
+	doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField))
+	dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)
+	if err != nil {
+		t.Error(err)
+	}
+	doc.AddField(dateField)
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), index.IndexField|index.StoreField))
+	doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, index.IndexField|index.StoreField))
+	dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), time.RFC3339, index.IndexField|index.StoreField)
+	if err != nil {
+		t.Error(err)
+	}
+	doc.AddField(dateField)
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	fieldsCount := 0
+	reader, err = idx.Reader()
+	if err != nil {
+		t.Fatal(err)
+	}
+	upsideDownReader, ok := reader.(*IndexReader)
+	if !ok {
+		t.Fatal("dump is only supported by index type upsidedown")
+	}
+	fieldsRows := upsideDownReader.DumpFields()
+	for range fieldsRows {
+		fieldsCount++
+	}
+	if fieldsCount != 3 {
+		t.Errorf("expected 3 fields, got %d", fieldsCount)
+	}
+
+	// 1 text term
+	// 16 numeric terms
+	// 16 date terms
+	// 3 stored fields
+	expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3)
+	docRowCount := 0
+	docRows := upsideDownReader.DumpDoc("1")
+	for range docRows {
+		docRowCount++
+	}
+	if docRowCount != expectedDocRowCount {
+		t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
+	}
+
+	docRowCount = 0
+	docRows = upsideDownReader.DumpDoc("2")
+	for range docRows {
+		docRowCount++
+	}
+	if docRowCount != expectedDocRowCount {
+		t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
+	}
+
+	// 1 version
+	// fieldsCount field rows
+	// 2 docs * expectedDocRowCount
+	// 2 back index rows
+	// 2 text term row count (2 different text terms)
+	// 16 numeric term row counts (shared for both docs, same numeric value)
+	// 16 date term row counts (shared for both docs, same date value)
+	expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep))))
+	allRowCount := 0
+	allRows := upsideDownReader.DumpAll()
+	for range allRows {
+		allRowCount++
+	}
+	if allRowCount != expectedAllRowCount {
+		t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount)
+	}
+
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

index/upsidedown/field_cache.go

@@ -0,0 +1,88 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"sync"
+)
+
+type FieldCache struct {
+	fieldIndexes   map[string]uint16
+	indexFields    []string
+	lastFieldIndex int
+	mutex          sync.RWMutex
+}
+
+func NewFieldCache() *FieldCache {
+	return &FieldCache{
+		fieldIndexes:   make(map[string]uint16),
+		lastFieldIndex: -1,
+	}
+}
+
+func (f *FieldCache) AddExisting(field string, index uint16) {
+	f.mutex.Lock()
+	f.addLOCKED(field, index)
+	f.mutex.Unlock()
+}
+
+func (f *FieldCache) addLOCKED(field string, index uint16) uint16 {
+	f.fieldIndexes[field] = index
+	if len(f.indexFields) < int(index)+1 {
+		prevIndexFields := f.indexFields
+		f.indexFields = make([]string, int(index)+16)
+		copy(f.indexFields, prevIndexFields)
+	}
+	f.indexFields[int(index)] = field
+	if int(index) > f.lastFieldIndex {
+		f.lastFieldIndex = int(index)
+	}
+	return index
+}
+
+// FieldNamed returns the index of the field, and whether or not it existed
+// before this call.  if createIfMissing is true, and new field index is assigned
+// but the second return value will still be false
+func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool) {
+	f.mutex.RLock()
+	if index, ok := f.fieldIndexes[field]; ok {
+		f.mutex.RUnlock()
+		return index, true
+	} else if !createIfMissing {
+		f.mutex.RUnlock()
+		return 0, false
+	}
+	// trade read lock for write lock
+	f.mutex.RUnlock()
+	f.mutex.Lock()
+	// need to check again with write lock
+	if index, ok := f.fieldIndexes[field]; ok {
+		f.mutex.Unlock()
+		return index, true
+	}
+	// assign next field id
+	index := f.addLOCKED(field, uint16(f.lastFieldIndex+1))
+	f.mutex.Unlock()
+	return index, false
+}
+
+func (f *FieldCache) FieldIndexed(index uint16) (field string) {
+	f.mutex.RLock()
+	if int(index) < len(f.indexFields) {
+		field = f.indexFields[int(index)]
+	}
+	f.mutex.RUnlock()
+	return field
+}

index/upsidedown/field_dict.go

@@ -0,0 +1,86 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"fmt"
+
+	index "github.com/blevesearch/bleve_index_api"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type UpsideDownCouchFieldDict struct {
+	indexReader *IndexReader
+	iterator    store.KVIterator
+	dictRow     *DictionaryRow
+	dictEntry   *index.DictEntry
+	field       uint16
+}
+
+func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) {
+
+	startKey := NewDictionaryRow(startTerm, field, 0).Key()
+	if endTerm == nil {
+		endTerm = []byte{ByteSeparator}
+	} else {
+		endTerm = incrementBytes(endTerm)
+	}
+	endKey := NewDictionaryRow(endTerm, field, 0).Key()
+
+	it := indexReader.kvreader.RangeIterator(startKey, endKey)
+
+	return &UpsideDownCouchFieldDict{
+		indexReader: indexReader,
+		iterator:    it,
+		dictRow:     &DictionaryRow{},   // Pre-alloced, reused row.
+		dictEntry:   &index.DictEntry{}, // Pre-alloced, reused entry.
+		field:       field,
+	}, nil
+
+}
+
+func (r *UpsideDownCouchFieldDict) BytesRead() uint64 {
+	return 0
+}
+
+func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) {
+	key, val, valid := r.iterator.Current()
+	if !valid {
+		return nil, nil
+	}
+
+	err := r.dictRow.parseDictionaryK(key)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error parsing dictionary row key: %v", err)
+	}
+	err = r.dictRow.parseDictionaryV(val)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error parsing dictionary row val: %v", err)
+	}
+	r.dictEntry.Term = string(r.dictRow.term)
+	r.dictEntry.Count = r.dictRow.count
+	// advance the iterator to the next term
+	r.iterator.Next()
+	return r.dictEntry, nil
+
+}
+
+func (r *UpsideDownCouchFieldDict) Cardinality() int {
+	return 0
+}
+
+func (r *UpsideDownCouchFieldDict) Close() error {
+	return r.iterator.Close()
+}

index/upsidedown/field_dict_test.go

@@ -0,0 +1,183 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+func TestIndexFieldDict(t *testing.T) {
+	defer func() {
+		err := DestroyTest()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), index.IndexField|index.IncludeTermVectors, testAnalyzer))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	dict, err := indexReader.FieldDict("name")
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount := 0
+	curr, err := dict.Next()
+	for err == nil && curr != nil {
+		termCount++
+		if curr.Term != "test" {
+			t.Errorf("expected term to be 'test', got '%s'", curr.Term)
+		}
+		curr, err = dict.Next()
+	}
+	if termCount != 1 {
+		t.Errorf("expected 1 term for this field, got %d", termCount)
+	}
+
+	dict2, err := indexReader.FieldDict("desc")
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms := make([]string, 0)
+	curr, err = dict2.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict2.Next()
+	}
+	if termCount != 3 {
+		t.Errorf("expected 3 term for this field, got %d", termCount)
+	}
+	expectedTerms := []string{"eat", "more", "rice"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+
+	// test start and end range
+	dict3, err := indexReader.FieldDictRange("desc", []byte("fun"), []byte("nice"))
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms = make([]string, 0)
+	curr, err = dict3.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict3.Next()
+	}
+	if termCount != 1 {
+		t.Errorf("expected 1 term for this field, got %d", termCount)
+	}
+	expectedTerms = []string{"more"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+
+	// test use case for prefix
+	dict4, err := indexReader.FieldDictPrefix("prefix", []byte("cat"))
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict4.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms = make([]string, 0)
+	curr, err = dict4.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict4.Next()
+	}
+	if termCount != 3 {
+		t.Errorf("expected 3 term for this field, got %d", termCount)
+	}
+	expectedTerms = []string{"cat", "cats", "catting"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+}

index/upsidedown/index_reader.go

@@ -0,0 +1,228 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/document"
+	index "github.com/blevesearch/bleve_index_api"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+var reflectStaticSizeIndexReader int
+
+func init() {
+	var ir IndexReader
+	reflectStaticSizeIndexReader = int(reflect.TypeOf(ir).Size())
+}
+
+type IndexReader struct {
+	index    *UpsideDownCouch
+	kvreader store.KVReader
+	docCount uint64
+}
+
+func (i *IndexReader) TermFieldReader(ctx context.Context, term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) {
+	fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false)
+	if fieldExists {
+		return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors)
+	}
+	return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors)
+}
+
+func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) {
+	return i.FieldDictRange(fieldName, nil, nil)
+}
+
+func (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
+	fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false)
+	if fieldExists {
+		return newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm)
+	}
+	return newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{})
+}
+
+func (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) {
+	return i.FieldDictRange(fieldName, termPrefix, termPrefix)
+}
+
+func (i *IndexReader) DocIDReaderAll() (index.DocIDReader, error) {
+	return newUpsideDownCouchDocIDReader(i)
+}
+
+func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) {
+	return newUpsideDownCouchDocIDReaderOnly(i, ids)
+}
+
+func (i *IndexReader) Document(id string) (doc index.Document, err error) {
+	// first hit the back index to confirm doc exists
+	var backIndexRow *BackIndexRow
+	backIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id))
+	if err != nil {
+		return
+	}
+	if backIndexRow == nil {
+		return
+	}
+	rvd := document.NewDocument(id)
+	storedRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil)
+	storedRowScanPrefix := storedRow.ScanPrefixForDoc()
+	it := i.kvreader.PrefixIterator(storedRowScanPrefix)
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		safeVal := make([]byte, len(val))
+		copy(safeVal, val)
+		var row *StoredRow
+		row, err = NewStoredRowKV(key, safeVal)
+		if err != nil {
+			return nil, err
+		}
+		if row != nil {
+			fieldName := i.index.fieldCache.FieldIndexed(row.field)
+			field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value)
+			if field != nil {
+				rvd.AddField(field)
+			}
+		}
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+	return rvd, nil
+}
+
+func (i *IndexReader) documentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocValueVisitor) error {
+	fieldsMap := make(map[uint16]string, len(fields))
+	for _, f := range fields {
+		id, ok := i.index.fieldCache.FieldNamed(f, false)
+		if ok {
+			fieldsMap[id] = f
+		}
+	}
+
+	tempRow := BackIndexRow{
+		doc: id,
+	}
+
+	keyBuf := GetRowBuffer()
+	if tempRow.KeySize() > len(keyBuf.buf) {
+		keyBuf.buf = make([]byte, 2*tempRow.KeySize())
+	}
+	defer PutRowBuffer(keyBuf)
+	keySize, err := tempRow.KeyTo(keyBuf.buf)
+	if err != nil {
+		return err
+	}
+
+	value, err := i.kvreader.Get(keyBuf.buf[:keySize])
+	if err != nil {
+		return err
+	}
+	if value == nil {
+		return nil
+	}
+
+	return visitBackIndexRow(value, func(field uint32, term []byte) {
+		if field, ok := fieldsMap[uint16(field)]; ok {
+			visitor(field, term)
+		}
+	})
+}
+
+func (i *IndexReader) Fields() (fields []string, err error) {
+	fields = make([]string, 0)
+	it := i.kvreader.PrefixIterator([]byte{'f'})
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		var row UpsideDownCouchRow
+		row, err = ParseFromKeyValue(key, val)
+		if err != nil {
+			fields = nil
+			return
+		}
+		if row != nil {
+			fieldRow, ok := row.(*FieldRow)
+			if ok {
+				fields = append(fields, fieldRow.name)
+			}
+		}
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+	return
+}
+
+func (i *IndexReader) GetInternal(key []byte) ([]byte, error) {
+	internalRow := NewInternalRow(key, nil)
+	return i.kvreader.Get(internalRow.Key())
+}
+
+func (i *IndexReader) DocCount() (uint64, error) {
+	return i.docCount, nil
+}
+
+func (i *IndexReader) Close() error {
+	return i.kvreader.Close()
+}
+
+func (i *IndexReader) ExternalID(id index.IndexInternalID) (string, error) {
+	return string(id), nil
+}
+
+func (i *IndexReader) InternalID(id string) (index.IndexInternalID, error) {
+	return index.IndexInternalID(id), nil
+}
+
+func incrementBytes(in []byte) []byte {
+	rv := make([]byte, len(in))
+	copy(rv, in)
+	for i := len(rv) - 1; i >= 0; i-- {
+		rv[i] = rv[i] + 1
+		if rv[i] != 0 {
+			// didn't overflow, so stop
+			break
+		}
+	}
+	return rv
+}
+
+func (i *IndexReader) DocValueReader(fields []string) (index.DocValueReader, error) {
+	return &DocValueReader{i: i, fields: fields}, nil
+}
+
+type DocValueReader struct {
+	i      *IndexReader
+	fields []string
+}
+
+func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID,
+	visitor index.DocValueVisitor) error {
+	return dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor)
+}
+
+func (dvr *DocValueReader) BytesRead() uint64 { return 0 }

index/upsidedown/reader.go

@@ -0,0 +1,376 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"bytes"
+	"reflect"
+	"sort"
+	"sync/atomic"
+
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+	"github.com/blevesearch/upsidedown_store_api"
+)
+
+var reflectStaticSizeUpsideDownCouchTermFieldReader int
+var reflectStaticSizeUpsideDownCouchDocIDReader int
+
+func init() {
+	var tfr UpsideDownCouchTermFieldReader
+	reflectStaticSizeUpsideDownCouchTermFieldReader =
+		int(reflect.TypeOf(tfr).Size())
+	var cdr UpsideDownCouchDocIDReader
+	reflectStaticSizeUpsideDownCouchDocIDReader =
+		int(reflect.TypeOf(cdr).Size())
+}
+
+type UpsideDownCouchTermFieldReader struct {
+	count              uint64
+	indexReader        *IndexReader
+	iterator           store.KVIterator
+	term               []byte
+	tfrNext            *TermFrequencyRow
+	tfrPrealloc        TermFrequencyRow
+	keyBuf             []byte
+	field              uint16
+	includeTermVectors bool
+}
+
+func (r *UpsideDownCouchTermFieldReader) Size() int {
+	sizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr +
+		len(r.term) +
+		r.tfrPrealloc.Size() +
+		len(r.keyBuf)
+
+	if r.tfrNext != nil {
+		sizeInBytes += r.tfrNext.Size()
+	}
+
+	return sizeInBytes
+}
+
+func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) {
+	bufNeeded := termFrequencyRowKeySize(term, nil)
+	if bufNeeded < dictionaryRowKeySize(term) {
+		bufNeeded = dictionaryRowKeySize(term)
+	}
+	buf := make([]byte, bufNeeded)
+
+	bufUsed := dictionaryRowKeyTo(buf, field, term)
+	val, err := indexReader.kvreader.Get(buf[:bufUsed])
+	if err != nil {
+		return nil, err
+	}
+	if val == nil {
+		atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))
+		rv := &UpsideDownCouchTermFieldReader{
+			count:              0,
+			term:               term,
+			field:              field,
+			includeTermVectors: includeTermVectors,
+		}
+		rv.tfrNext = &rv.tfrPrealloc
+		return rv, nil
+	}
+
+	count, err := dictionaryRowParseV(val)
+	if err != nil {
+		return nil, err
+	}
+
+	bufUsed = termFrequencyRowKeyTo(buf, field, term, nil)
+	it := indexReader.kvreader.PrefixIterator(buf[:bufUsed])
+
+	atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1))
+	return &UpsideDownCouchTermFieldReader{
+		indexReader:        indexReader,
+		iterator:           it,
+		count:              count,
+		term:               term,
+		field:              field,
+		includeTermVectors: includeTermVectors,
+	}, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Count() uint64 {
+	return r.count
+}
+
+func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) {
+	if r.iterator != nil {
+		// We treat tfrNext also like an initialization flag, which
+		// tells us whether we need to invoke the underlying
+		// iterator.Next().  The first time, don't call iterator.Next().
+		if r.tfrNext != nil {
+			r.iterator.Next()
+		} else {
+			r.tfrNext = &r.tfrPrealloc
+		}
+		key, val, valid := r.iterator.Current()
+		if valid {
+			tfr := r.tfrNext
+			err := tfr.parseKDoc(key, r.term)
+			if err != nil {
+				return nil, err
+			}
+			err = tfr.parseV(val, r.includeTermVectors)
+			if err != nil {
+				return nil, err
+			}
+			rv := preAlloced
+			if rv == nil {
+				rv = &index.TermFieldDoc{}
+			}
+			rv.ID = append(rv.ID, tfr.doc...)
+			rv.Freq = tfr.freq
+			rv.Norm = float64(tfr.norm)
+			if tfr.vectors != nil {
+				rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors)
+			}
+			return rv, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) {
+	if r.iterator != nil {
+		if r.tfrNext == nil {
+			r.tfrNext = &TermFrequencyRow{}
+		}
+		tfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0)
+		r.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0])
+		if err != nil {
+			return nil, err
+		}
+		r.iterator.Seek(r.keyBuf)
+		key, val, valid := r.iterator.Current()
+		if valid {
+			err := tfr.parseKDoc(key, r.term)
+			if err != nil {
+				return nil, err
+			}
+			err = tfr.parseV(val, r.includeTermVectors)
+			if err != nil {
+				return nil, err
+			}
+			rv = preAlloced
+			if rv == nil {
+				rv = &index.TermFieldDoc{}
+			}
+			rv.ID = append(rv.ID, tfr.doc...)
+			rv.Freq = tfr.freq
+			rv.Norm = float64(tfr.norm)
+			if tfr.vectors != nil {
+				rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors)
+			}
+			return rv, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Close() error {
+	if r.indexReader != nil {
+		atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1))
+	}
+	if r.iterator != nil {
+		return r.iterator.Close()
+	}
+	return nil
+}
+
+type UpsideDownCouchDocIDReader struct {
+	indexReader *IndexReader
+	iterator    store.KVIterator
+	only        []string
+	onlyPos     int
+	onlyMode    bool
+}
+
+func (r *UpsideDownCouchDocIDReader) Size() int {
+	sizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader +
+		reflectStaticSizeIndexReader + size.SizeOfPtr
+
+	for _, entry := range r.only {
+		sizeInBytes += size.SizeOfString + len(entry)
+	}
+
+	return sizeInBytes
+}
+
+func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) {
+	startBytes := []byte{0x0}
+	endBytes := []byte{0xff}
+
+	bisr := NewBackIndexRow(startBytes, nil, nil)
+	bier := NewBackIndexRow(endBytes, nil, nil)
+	it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key())
+
+	return &UpsideDownCouchDocIDReader{
+		indexReader: indexReader,
+		iterator:    it,
+	}, nil
+}
+
+func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) {
+	// we don't actually own the list of ids, so if before we sort we must copy
+	idsCopy := make([]string, len(ids))
+	copy(idsCopy, ids)
+	// ensure ids are sorted
+	sort.Strings(idsCopy)
+	startBytes := []byte{0x0}
+	if len(idsCopy) > 0 {
+		startBytes = []byte(idsCopy[0])
+	}
+	endBytes := []byte{0xff}
+	if len(idsCopy) > 0 {
+		endBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1]))
+	}
+	bisr := NewBackIndexRow(startBytes, nil, nil)
+	bier := NewBackIndexRow(endBytes, nil, nil)
+	it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key())
+
+	return &UpsideDownCouchDocIDReader{
+		indexReader: indexReader,
+		iterator:    it,
+		only:        idsCopy,
+		onlyMode:    true,
+	}, nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) {
+	key, val, valid := r.iterator.Current()
+
+	if r.onlyMode {
+		var rv index.IndexInternalID
+		for valid && r.onlyPos < len(r.only) {
+			br, err := NewBackIndexRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {
+				ok := r.nextOnly()
+				if !ok {
+					return nil, nil
+				}
+				r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
+				key, val, valid = r.iterator.Current()
+				continue
+			} else {
+				rv = append([]byte(nil), br.doc...)
+				break
+			}
+		}
+		if valid && r.onlyPos < len(r.only) {
+			ok := r.nextOnly()
+			if ok {
+				r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
+			}
+			return rv, nil
+		}
+
+	} else {
+		if valid {
+			br, err := NewBackIndexRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			rv := append([]byte(nil), br.doc...)
+			r.iterator.Next()
+			return rv, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) {
+
+	if r.onlyMode {
+		r.onlyPos = sort.SearchStrings(r.only, string(docID))
+		if r.onlyPos >= len(r.only) {
+			// advanced to key after our last only key
+			return nil, nil
+		}
+		r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
+		key, val, valid := r.iterator.Current()
+
+		var rv index.IndexInternalID
+		for valid && r.onlyPos < len(r.only) {
+			br, err := NewBackIndexRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) {
+				// the only key we seek'd to didn't exist
+				// now look for the closest key that did exist in only
+				r.onlyPos = sort.SearchStrings(r.only, string(br.doc))
+				if r.onlyPos >= len(r.only) {
+					// advanced to key after our last only key
+					return nil, nil
+				}
+				// now seek to this new only key
+				r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
+				key, val, valid = r.iterator.Current()
+				continue
+			} else {
+				rv = append([]byte(nil), br.doc...)
+				break
+			}
+		}
+		if valid && r.onlyPos < len(r.only) {
+			ok := r.nextOnly()
+			if ok {
+				r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key())
+			}
+			return rv, nil
+		}
+	} else {
+		bir := NewBackIndexRow(docID, nil, nil)
+		r.iterator.Seek(bir.Key())
+		key, val, valid := r.iterator.Current()
+		if valid {
+			br, err := NewBackIndexRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			rv := append([]byte(nil), br.doc...)
+			r.iterator.Next()
+			return rv, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Close() error {
+	return r.iterator.Close()
+}
+
+// move the r.only pos forward one, skipping duplicates
+// return true if there is more data, or false if we got to the end of the list
+func (r *UpsideDownCouchDocIDReader) nextOnly() bool {
+
+	// advance 1 position, until we see a different key
+	//   it's already sorted, so this skips duplicates
+	start := r.onlyPos
+	r.onlyPos++
+	for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] {
+		start = r.onlyPos
+		r.onlyPos++
+	}
+	// inidicate if we got to the end of the list
+	return r.onlyPos < len(r.only)
+}

index/upsidedown/reader_test.go

@@ -0,0 +1,548 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/boltdb"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+func TestIndexReader(t *testing.T) {
+	defer func() {
+		err := DestroyTest()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors, testAnalyzer))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// first look for a term that doesn't exist
+	reader, err := indexReader.TermFieldReader(context.TODO(), []byte("nope"), "name", true, true, true)
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+	count := reader.Count()
+	if count != 0 {
+		t.Errorf("Expected doc count to be: %d got: %d", 0, count)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true)
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+
+	count = reader.Count()
+	if count != expectedCount {
+		t.Errorf("Expected doc count to be: %d got: %d", expectedCount, count)
+	}
+
+	var match *index.TermFieldDoc
+	var actualCount uint64
+	match, err = reader.Next(nil)
+	for err == nil && match != nil {
+		match, err = reader.Next(nil)
+		if err != nil {
+			t.Errorf("unexpected error reading next")
+		}
+		actualCount++
+	}
+	if actualCount != count {
+		t.Errorf("count was 2, but only saw %d", actualCount)
+	}
+
+	expectedMatch := &index.TermFieldDoc{
+		ID:   index.IndexInternalID("2"),
+		Freq: 1,
+		Norm: 0.5773502588272095,
+		Vectors: []*index.TermFieldVector{
+			{
+				Field: "desc",
+				Pos:   3,
+				Start: 9,
+				End:   13,
+			},
+		},
+	}
+	tfr, err := indexReader.TermFieldReader(context.TODO(), []byte("rice"), "desc", true, true, true)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	match, err = tfr.Next(nil)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(expectedMatch, match) {
+		t.Errorf("got %#v, expected %#v", match, expectedMatch)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now test usage of advance
+	reader, err = indexReader.TermFieldReader(context.TODO(), []byte("test"), "name", true, true, true)
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+
+	match, err = reader.Advance(index.IndexInternalID("2"), nil)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match == nil {
+		t.Fatalf("Expected match, got nil")
+	}
+	if !match.ID.Equals(index.IndexInternalID("2")) {
+		t.Errorf("Expected ID '2', got '%s'", match.ID)
+	}
+	match, err = reader.Advance(index.IndexInternalID("3"), nil)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now test creating a reader for a field that doesn't exist
+	reader, err = indexReader.TermFieldReader(context.TODO(), []byte("water"), "doesnotexist", true, true, true)
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+	count = reader.Count()
+	if count != 0 {
+		t.Errorf("expected count 0 for reader of non-existent field")
+	}
+	match, err = reader.Next(nil)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+	match, err = reader.Advance(index.IndexInternalID("anywhere"), nil)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+}
+
+func TestIndexDocIdReader(t *testing.T) {
+	defer func() {
+		err := DestroyTest()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test")))
+	doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), index.IndexField|index.IncludeTermVectors))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	// first get all doc ids
+	reader, err := indexReader.DocIDReaderAll()
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	id, err := reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	count := uint64(0)
+	for id != nil {
+		count++
+		id, err = reader.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if count != expectedCount {
+		t.Errorf("expected %d, got %d", expectedCount, count)
+	}
+
+	// try it again, but jump to the second doc this time
+	reader2, err := indexReader.DocIDReaderAll()
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader2.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	id, err = reader2.Advance(index.IndexInternalID("2"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("2")) {
+		t.Errorf("expected to find id '2', got '%s'", id)
+	}
+
+	id, err = reader2.Advance(index.IndexInternalID("3"))
+	if err != nil {
+		t.Error(err)
+	}
+	if id != nil {
+		t.Errorf("expected to find id '', got '%s'", id)
+	}
+}
+
+func TestCrashBadBackIndexRow(t *testing.T) {
+	br, err := NewBackIndexRowKV([]byte{byte('b'), byte('a'), ByteSeparator}, []byte{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(br.doc) != "a" {
+		t.Fatal(err)
+	}
+}
+
+func TestIndexDocIdOnlyReader(t *testing.T) {
+	defer func() {
+		err := DestroyTest()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	analysisQueue := index.NewAnalysisQueue(1)
+	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	doc := document.NewDocument("1")
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("3")
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("5")
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("7")
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("9")
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	onlyIds := []string{"1", "5", "9"}
+	reader, err := indexReader.DocIDReaderOnly(onlyIds)
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	id, err := reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	count := uint64(0)
+	for id != nil {
+		count++
+		id, err = reader.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if count != 3 {
+		t.Errorf("expected 3, got %d", count)
+	}
+
+	// try it again, but jump
+	reader2, err := indexReader.DocIDReaderOnly(onlyIds)
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader2.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	id, err = reader2.Advance(index.IndexInternalID("5"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("5")) {
+		t.Errorf("expected to find id '5', got '%s'", id)
+	}
+
+	id, err = reader2.Advance(index.IndexInternalID("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if id != nil {
+		t.Errorf("expected to find id '', got '%s'", id)
+	}
+
+	// some keys aren't actually there
+	onlyIds = []string{"0", "2", "4", "5", "6", "8", "a"}
+	reader3, err := indexReader.DocIDReaderOnly(onlyIds)
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader3.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	id, err = reader3.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	count = uint64(0)
+	for id != nil {
+		count++
+		id, err = reader3.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if count != 1 {
+		t.Errorf("expected 1, got %d", count)
+	}
+
+	// mix advance and next
+	onlyIds = []string{"0", "1", "3", "5", "6", "9"}
+	reader4, err := indexReader.DocIDReaderOnly(onlyIds)
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader4.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	// first key is "1"
+	id, err = reader4.Next()
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("1")) {
+		t.Errorf("expected to find id '1', got '%s'", id)
+	}
+
+	// advancing to key we dont have gives next
+	id, err = reader4.Advance(index.IndexInternalID("2"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("3")) {
+		t.Errorf("expected to find id '3', got '%s'", id)
+	}
+
+	// next after advance works
+	id, err = reader4.Next()
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("5")) {
+		t.Errorf("expected to find id '5', got '%s'", id)
+	}
+
+	// advancing to key we do have works
+	id, err = reader4.Advance(index.IndexInternalID("9"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("9")) {
+		t.Errorf("expected to find id '9', got '%s'", id)
+	}
+
+	// advance backwards at end
+	id, err = reader4.Advance(index.IndexInternalID("4"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("5")) {
+		t.Errorf("expected to find id '5', got '%s'", id)
+	}
+
+	// next after advance works
+	id, err = reader4.Next()
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("9")) {
+		t.Errorf("expected to find id '9', got '%s'", id)
+	}
+
+	// advance backwards to key that exists, but not in only set
+	id, err = reader4.Advance(index.IndexInternalID("7"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !id.Equals(index.IndexInternalID("9")) {
+		t.Errorf("expected to find id '9', got '%s'", id)
+	}
+}

index/upsidedown/row_merge.go

@@ -0,0 +1,76 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"encoding/binary"
+)
+
+var mergeOperator upsideDownMerge
+
+var dictionaryTermIncr []byte
+var dictionaryTermDecr []byte
+
+func init() {
+	dictionaryTermIncr = make([]byte, 8)
+	binary.LittleEndian.PutUint64(dictionaryTermIncr, uint64(1))
+	dictionaryTermDecr = make([]byte, 8)
+	var negOne = int64(-1)
+	binary.LittleEndian.PutUint64(dictionaryTermDecr, uint64(negOne))
+}
+
+type upsideDownMerge struct{}
+
+func (m *upsideDownMerge) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) {
+	// set up record based on key
+	dr, err := NewDictionaryRowK(key)
+	if err != nil {
+		return nil, false
+	}
+	if len(existingValue) > 0 {
+		// if existing value, parse it
+		err = dr.parseDictionaryV(existingValue)
+		if err != nil {
+			return nil, false
+		}
+	}
+
+	// now process operands
+	for _, operand := range operands {
+		next := int64(binary.LittleEndian.Uint64(operand))
+		if next < 0 && uint64(-next) > dr.count {
+			// subtracting next from existing would overflow
+			dr.count = 0
+		} else if next < 0 {
+			dr.count -= uint64(-next)
+		} else {
+			dr.count += uint64(next)
+		}
+	}
+
+	return dr.Value(), true
+}
+
+func (m *upsideDownMerge) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) {
+	left := int64(binary.LittleEndian.Uint64(leftOperand))
+	right := int64(binary.LittleEndian.Uint64(rightOperand))
+	rv := make([]byte, 8)
+	binary.LittleEndian.PutUint64(rv, uint64(left+right))
+	return rv, true
+}
+
+func (m *upsideDownMerge) Name() string {
+	return "upsideDownMerge"
+}

index/upsidedown/row_merge_test.go

@@ -0,0 +1,57 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+)
+
+func TestPartialMerge(t *testing.T) {
+
+	tests := []struct {
+		in  [][]byte
+		out uint64
+	}{
+		{
+			in:  [][]byte{dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr, dictionaryTermIncr},
+			out: 5,
+		},
+	}
+
+	mo := &upsideDownMerge{}
+	for _, test := range tests {
+		curr := test.in[0]
+		for _, next := range test.in[1:] {
+			var ok bool
+			curr, ok = mo.PartialMerge([]byte("key"), curr, next)
+			if !ok {
+				t.Errorf("expected partial merge ok")
+			}
+		}
+		actual := decodeCount(curr)
+		if actual != test.out {
+			t.Errorf("expected %d, got %d", test.out, actual)
+		}
+	}
+
+}
+
+func decodeCount(in []byte) uint64 {
+	buf := bytes.NewBuffer(in)
+	count, _ := binary.ReadUvarint(buf)
+	return count
+}

index/upsidedown/row_test.go

@@ -0,0 +1,382 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"math"
+	"reflect"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func TestRows(t *testing.T) {
+	tests := []struct {
+		input  UpsideDownCouchRow
+		outKey []byte
+		outVal []byte
+	}{
+		{
+			NewVersionRow(1),
+			[]byte{'v'},
+			[]byte{0x1},
+		},
+		{
+			NewFieldRow(0, "name"),
+			[]byte{'f', 0, 0},
+			[]byte{'n', 'a', 'm', 'e', ByteSeparator},
+		},
+		{
+			NewFieldRow(1, "desc"),
+			[]byte{'f', 1, 0},
+			[]byte{'d', 'e', 's', 'c', ByteSeparator},
+		},
+		{
+			NewFieldRow(513, "style"),
+			[]byte{'f', 1, 2},
+			[]byte{'s', 't', 'y', 'l', 'e', ByteSeparator},
+		},
+		{
+			NewDictionaryRow([]byte{'b', 'e', 'e', 'r'}, 0, 27),
+			[]byte{'d', 0, 0, 'b', 'e', 'e', 'r'},
+			[]byte{27},
+		},
+		{
+			NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte("catz"), 3, 3.14),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'c', 'a', 't', 'z'},
+			[]byte{3, 195, 235, 163, 130, 4},
+		},
+		{
+			NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 3, 3.14),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 195, 235, 163, 130, 4},
+		},
+		{
+			NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 3, 3.14, []*TermVector{{field: 0, pos: 1, start: 3, end: 11}, {field: 0, pos: 2, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0},
+		},
+		// test larger varints
+		{
+			NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11}, {field: 0, pos: 2198, start: 23, end: 31}, {field: 0, pos: 3, start: 43, end: 51}}),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 0, 0, 150, 17, 23, 31, 0, 0, 3, 43, 51, 0},
+		},
+		// test vectors with arrayPositions
+		{
+			NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, []byte("budweiser"), 25896, 3.14, []*TermVector{{field: 255, pos: 1, start: 3, end: 11, arrayPositions: []uint64{0}}, {field: 0, pos: 2198, start: 23, end: 31, arrayPositions: []uint64{1, 2}}, {field: 0, pos: 3, start: 43, end: 51, arrayPositions: []uint64{3, 4, 5}}}),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 1, 0, 0, 150, 17, 23, 31, 2, 1, 2, 0, 3, 43, 51, 3, 3, 4, 5},
+		},
+		{
+			NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}}, nil),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r'},
+		},
+		{
+			NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}, {Field: proto.Uint32(1), Terms: []string{"beat"}}}, nil),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't'},
+		},
+		{
+			NewBackIndexRow([]byte("budweiser"), []*BackIndexTermsEntry{{Field: proto.Uint32(0), Terms: []string{"beer"}}, {Field: proto.Uint32(1), Terms: []string{"beat"}}}, []*BackIndexStoreEntry{{Field: proto.Uint32(3)}, {Field: proto.Uint32(4)}, {Field: proto.Uint32(5)}}),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5},
+		},
+		{
+			NewStoredRow([]byte("budweiser"), 0, []uint64{}, byte('t'), []byte("an american beer")),
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		{
+			NewStoredRow([]byte("budweiser"), 0, []uint64{2, 294, 3078}, byte('t'), []byte("an american beer")),
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0, 2, 166, 2, 134, 24},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		{
+			NewInternalRow([]byte("mapping"), []byte(`{"mapping":"json content"}`)),
+			[]byte{'i', 'm', 'a', 'p', 'p', 'i', 'n', 'g'},
+			[]byte{'{', '"', 'm', 'a', 'p', 'p', 'i', 'n', 'g', '"', ':', '"', 'j', 's', 'o', 'n', ' ', 'c', 'o', 'n', 't', 'e', 'n', 't', '"', '}'},
+		},
+	}
+
+	// test going from struct to k/v bytes
+	for i, test := range tests {
+		rk := test.input.Key()
+		if !reflect.DeepEqual(rk, test.outKey) {
+			t.Errorf("Expected key to be %v got: %v", test.outKey, rk)
+		}
+		rv := test.input.Value()
+		if !reflect.DeepEqual(rv, test.outVal) {
+			t.Errorf("Expected value to be %v got: %v for %d", test.outVal, rv, i)
+		}
+	}
+
+	// now test going back from k/v bytes to struct
+	for i, test := range tests {
+		row, err := ParseFromKeyValue(test.outKey, test.outVal)
+		if err != nil {
+			t.Errorf("error parsking key/value: %v", err)
+		}
+		if !reflect.DeepEqual(row, test.input) {
+			t.Errorf("Expected: %#v got: %#v for %d", test.input, row, i)
+		}
+	}
+
+}
+
+func TestInvalidRows(t *testing.T) {
+	tests := []struct {
+		key []byte
+		val []byte
+	}{
+		// empty key
+		{
+			[]byte{},
+			[]byte{},
+		},
+		// no such type q
+		{
+			[]byte{'q'},
+			[]byte{},
+		},
+		// type v, invalid empty value
+		{
+			[]byte{'v'},
+			[]byte{},
+		},
+		// type f, invalid key
+		{
+			[]byte{'f'},
+			[]byte{},
+		},
+		// type f, valid key, invalid value
+		{
+			[]byte{'f', 0, 0},
+			[]byte{},
+		},
+		// type t, invalid key (missing field)
+		{
+			[]byte{'t'},
+			[]byte{},
+		},
+		// type t, invalid key (missing term)
+		{
+			[]byte{'t', 0, 0},
+			[]byte{},
+		},
+		// type t, invalid key (missing id)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator},
+			[]byte{},
+		},
+		// type t, invalid val (missing freq)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{},
+		},
+		// type t, invalid val (missing norm)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3},
+		},
+		// type t, invalid val (half missing tv field, full missing is valid (no term vectors))
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 255},
+		},
+		// type t, invalid val (missing tv pos)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0},
+		},
+		// type t, invalid val (missing tv start)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0, 0},
+		},
+		// type t, invalid val (missing tv end)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0, 0, 0},
+		},
+		// type b, invalid key (missing id)
+		{
+			[]byte{'b'},
+			[]byte{'b', 'e', 'e', 'r', ByteSeparator, 0, 0},
+		},
+		// type b, invalid val (missing field)
+		{
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{'g', 'a', 'r', 'b', 'a', 'g', 'e'},
+		},
+		// type s, invalid key (missing id)
+		{
+			[]byte{'s'},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		// type b, invalid val (missing field)
+		{
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+	}
+
+	for _, test := range tests {
+		_, err := ParseFromKeyValue(test.key, test.val)
+		if err == nil {
+			t.Errorf("expected error, got nil")
+		}
+	}
+}
+
+func TestDictionaryRowValueBug197(t *testing.T) {
+	// this was the smallest value that would trigger a crash
+	dr := &DictionaryRow{
+		field: 0,
+		term:  []byte("marty"),
+		count: 72057594037927936,
+	}
+	dr.Value()
+	// this is the maximum possible value
+	dr = &DictionaryRow{
+		field: 0,
+		term:  []byte("marty"),
+		count: math.MaxUint64,
+	}
+	dr.Value()
+	// neither of these should panic
+}
+
+func BenchmarkTermFrequencyRowEncode(b *testing.B) {
+	row := NewTermFrequencyRowWithTermVectors(
+		[]byte{'b', 'e', 'e', 'r'},
+		0,
+		[]byte("budweiser"),
+		3,
+		3.14,
+		[]*TermVector{
+			{
+				field: 0,
+				pos:   1,
+				start: 3,
+				end:   11,
+			},
+			{
+				field: 0,
+				pos:   2,
+				start: 23,
+				end:   31,
+			},
+			{
+				field: 0,
+				pos:   3,
+				start: 43,
+				end:   51,
+			},
+		})
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		row.Key()
+		row.Value()
+	}
+}
+
+func BenchmarkTermFrequencyRowDecode(b *testing.B) {
+	k := []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}
+	v := []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 0, 2, 23, 31, 0, 0, 3, 43, 51, 0}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := NewTermFrequencyRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkBackIndexRowEncode(b *testing.B) {
+	field := uint32(1)
+	t1 := "term1"
+	row := NewBackIndexRow([]byte("beername"),
+		[]*BackIndexTermsEntry{
+			{
+				Field: &field,
+				Terms: []string{t1},
+			},
+		},
+		[]*BackIndexStoreEntry{
+			{
+				Field: &field,
+			},
+		})
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		row.Key()
+		row.Value()
+		b.Logf("%#v", row.Value())
+	}
+}
+
+func BenchmarkBackIndexRowDecode(b *testing.B) {
+	k := []byte{0x62, 0x62, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65}
+	v := []byte{0xa, 0x9, 0x8, 0x1, 0x12, 0x5, 0x74, 0x65, 0x72, 0x6d, 0x31, 0x12, 0x2, 0x8, 0x1}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := NewBackIndexRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStoredRowEncode(b *testing.B) {
+	row := NewStoredRow([]byte("budweiser"), 0, []uint64{}, byte('t'), []byte("an american beer"))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		row.Key()
+		row.Value()
+	}
+}
+
+func BenchmarkStoredRowDecode(b *testing.B) {
+	k := []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0}
+	v := []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := NewStoredRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestVisitBackIndexRow(t *testing.T) {
+	expected := map[uint32][]byte{
+		0: []byte("beer"),
+		1: []byte("beat"),
+	}
+	val := []byte{10, 8, 8, 0, 18, 4, 'b', 'e', 'e', 'r', 10, 8, 8, 1, 18, 4, 'b', 'e', 'a', 't', 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5}
+	err := visitBackIndexRow(val, func(field uint32, term []byte) {
+		if reflect.DeepEqual(expected[field], term) {
+			delete(expected, field)
+		}
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(expected) > 0 {
+		t.Errorf("expected visitor to see these but did not %v", expected)
+	}
+}

index/upsidedown/stats.go

@@ -0,0 +1,55 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upsidedown
+
+import (
+	"sync/atomic"
+
+	"github.com/blevesearch/bleve/v2/util"
+	"github.com/blevesearch/upsidedown_store_api"
+)
+
+type indexStat struct {
+	updates, deletes, batches, errors uint64
+	analysisTime, indexTime           uint64
+	termSearchersStarted              uint64
+	termSearchersFinished             uint64
+	numPlainTextBytesIndexed          uint64
+	i                                 *UpsideDownCouch
+}
+
+func (i *indexStat) statsMap() map[string]interface{} {
+	m := map[string]interface{}{}
+	m["updates"] = atomic.LoadUint64(&i.updates)
+	m["deletes"] = atomic.LoadUint64(&i.deletes)
+	m["batches"] = atomic.LoadUint64(&i.batches)
+	m["errors"] = atomic.LoadUint64(&i.errors)
+	m["analysis_time"] = atomic.LoadUint64(&i.analysisTime)
+	m["index_time"] = atomic.LoadUint64(&i.indexTime)
+	m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted)
+	m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished)
+	m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed)
+
+	if o, ok := i.i.store.(store.KVStoreStats); ok {
+		m["kv"] = o.StatsMap()
+	}
+
+	return m
+}
+
+func (i *indexStat) MarshalJSON() ([]byte, error) {
+	m := i.statsMap()
+	return util.MarshalJSON(m)
+}

index/upsidedown/store/boltdb/iterator.go

@@ -0,0 +1,85 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"bytes"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+type Iterator struct {
+	store  *Store
+	tx     *bolt.Tx
+	cursor *bolt.Cursor
+	prefix []byte
+	start  []byte
+	end    []byte
+	valid  bool
+	key    []byte
+	val    []byte
+}
+
+func (i *Iterator) updateValid() {
+	i.valid = (i.key != nil)
+	if i.valid {
+		if i.prefix != nil {
+			i.valid = bytes.HasPrefix(i.key, i.prefix)
+		} else if i.end != nil {
+			i.valid = bytes.Compare(i.key, i.end) < 0
+		}
+	}
+}
+
+func (i *Iterator) Seek(k []byte) {
+	if i.start != nil && bytes.Compare(k, i.start) < 0 {
+		k = i.start
+	}
+	if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {
+		if bytes.Compare(k, i.prefix) < 0 {
+			k = i.prefix
+		} else {
+			i.valid = false
+			return
+		}
+	}
+	i.key, i.val = i.cursor.Seek(k)
+	i.updateValid()
+}
+
+func (i *Iterator) Next() {
+	i.key, i.val = i.cursor.Next()
+	i.updateValid()
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	return i.key, i.val, i.valid
+}
+
+func (i *Iterator) Key() []byte {
+	return i.key
+}
+
+func (i *Iterator) Value() []byte {
+	return i.val
+}
+
+func (i *Iterator) Valid() bool {
+	return i.valid
+}
+
+func (i *Iterator) Close() error {
+	return nil
+}

index/upsidedown/store/boltdb/reader.go

@@ -0,0 +1,73 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	store "github.com/blevesearch/upsidedown_store_api"
+	bolt "go.etcd.io/bbolt"
+)
+
+type Reader struct {
+	store  *Store
+	tx     *bolt.Tx
+	bucket *bolt.Bucket
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	var rv []byte
+	v := r.bucket.Get(key)
+	if v != nil {
+		rv = make([]byte, len(v))
+		copy(rv, v)
+	}
+	return rv, nil
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
+	cursor := r.bucket.Cursor()
+
+	rv := &Iterator{
+		store:  r.store,
+		tx:     r.tx,
+		cursor: cursor,
+		prefix: prefix,
+	}
+
+	rv.Seek(prefix)
+	return rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	cursor := r.bucket.Cursor()
+
+	rv := &Iterator{
+		store:  r.store,
+		tx:     r.tx,
+		cursor: cursor,
+		start:  start,
+		end:    end,
+	}
+
+	rv.Seek(start)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.tx.Rollback()
+}

index/upsidedown/store/boltdb/stats.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"github.com/blevesearch/bleve/v2/util"
+)
+
+type stats struct {
+	s *Store
+}
+
+func (s *stats) MarshalJSON() ([]byte, error) {
+	bs := s.s.db.Stats()
+	return util.MarshalJSON(bs)
+}

index/upsidedown/store/boltdb/store.go

@@ -0,0 +1,184 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
+// following options:
+//
+// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
+//
+// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
+// operations in exchange of losing integrity guarantees if indexation aborts
+// without closing the index. Use it when rebuilding indexes from zero.
+package boltdb
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	store "github.com/blevesearch/upsidedown_store_api"
+	bolt "go.etcd.io/bbolt"
+)
+
+const (
+	Name                    = "boltdb"
+	defaultCompactBatchSize = 100
+)
+
+type Store struct {
+	path        string
+	bucket      string
+	db          *bolt.DB
+	noSync      bool
+	fillPercent float64
+	mo          store.MergeOperator
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
+
+	bucket, ok := config["bucket"].(string)
+	if !ok {
+		bucket = "bleve"
+	}
+
+	noSync, _ := config["nosync"].(bool)
+
+	fillPercent, ok := config["fillPercent"].(float64)
+	if !ok {
+		fillPercent = bolt.DefaultFillPercent
+	}
+
+	bo := &bolt.Options{}
+	ro, ok := config["read_only"].(bool)
+	if ok {
+		bo.ReadOnly = ro
+	}
+
+	if initialMmapSize, ok := config["initialMmapSize"].(int); ok {
+		bo.InitialMmapSize = initialMmapSize
+	} else if initialMmapSize, ok := config["initialMmapSize"].(float64); ok {
+		bo.InitialMmapSize = int(initialMmapSize)
+	}
+
+	db, err := bolt.Open(path, 0600, bo)
+	if err != nil {
+		return nil, err
+	}
+	db.NoSync = noSync
+
+	if !bo.ReadOnly {
+		err = db.Update(func(tx *bolt.Tx) error {
+			_, err := tx.CreateBucketIfNotExists([]byte(bucket))
+
+			return err
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	rv := Store{
+		path:        path,
+		bucket:      bucket,
+		db:          db,
+		mo:          mo,
+		noSync:      noSync,
+		fillPercent: fillPercent,
+	}
+	return &rv, nil
+}
+
+func (bs *Store) Close() error {
+	return bs.db.Close()
+}
+
+func (bs *Store) Reader() (store.KVReader, error) {
+	tx, err := bs.db.Begin(false)
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{
+		store:  bs,
+		tx:     tx,
+		bucket: tx.Bucket([]byte(bs.bucket)),
+	}, nil
+}
+
+func (bs *Store) Writer() (store.KVWriter, error) {
+	return &Writer{
+		store: bs,
+	}, nil
+}
+
+func (bs *Store) Stats() json.Marshaler {
+	return &stats{
+		s: bs,
+	}
+}
+
+// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
+// Removing entries is a workaround for github issue #374.
+func (bs *Store) CompactWithBatchSize(batchSize int) error {
+	for {
+		cnt := 0
+		err := bs.db.Batch(func(tx *bolt.Tx) error {
+			c := tx.Bucket([]byte(bs.bucket)).Cursor()
+			prefix := []byte("d")
+
+			for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+				if bytes.Equal(v, []byte{0}) {
+					cnt++
+					if err := c.Delete(); err != nil {
+						return err
+					}
+					if cnt == batchSize {
+						break
+					}
+				}
+
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if cnt == 0 {
+			break
+		}
+	}
+	return nil
+}
+
+// Compact calls CompactWithBatchSize with a default batch size of 100.  This is a workaround
+// for github issue #374.
+func (bs *Store) Compact() error {
+	return bs.CompactWithBatchSize(defaultCompactBatchSize)
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/boltdb/store_test.go

@@ -0,0 +1,148 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !darwin || !arm64
+
+package boltdb
+
+import (
+	"os"
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+	bolt "go.etcd.io/bbolt"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, map[string]interface{}{"path": "test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.RemoveAll("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestBoltDBKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestBoltDBReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestBoltDBReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestBoltDBWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestBoltDBPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestBoltDBPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestBoltDBRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestBoltDBRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestBoltDBMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}
+
+func TestBoltDBConfig(t *testing.T) {
+	var tests = []struct {
+		in          map[string]interface{}
+		path        string
+		bucket      string
+		noSync      bool
+		fillPercent float64
+	}{
+		{
+			map[string]interface{}{"path": "test", "bucket": "mybucket", "nosync": true, "fillPercent": 0.75},
+			"test",
+			"mybucket",
+			true,
+			0.75,
+		},
+		{
+			map[string]interface{}{"path": "test"},
+			"test",
+			"bleve",
+			false,
+			bolt.DefaultFillPercent,
+		},
+	}
+
+	for _, test := range tests {
+		kv, err := New(nil, test.in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bs, ok := kv.(*Store)
+		if !ok {
+			t.Fatal("failed type assertion to *boltdb.Store")
+		}
+		if bs.path != test.path {
+			t.Fatalf("path: expected %q, got %q", test.path, bs.path)
+		}
+		if bs.bucket != test.bucket {
+			t.Fatalf("bucket: expected %q, got %q", test.bucket, bs.bucket)
+		}
+		if bs.noSync != test.noSync {
+			t.Fatalf("noSync: expected %t, got %t", test.noSync, bs.noSync)
+		}
+		if bs.fillPercent != test.fillPercent {
+			t.Fatalf("fillPercent: expected %f, got %f", test.fillPercent, bs.fillPercent)
+		}
+		cleanup(t, kv)
+	}
+}

index/upsidedown/store/boltdb/writer.go

@@ -0,0 +1,95 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boltdb
+
+import (
+	"fmt"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w.store.mo)
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	return make([]byte, options.TotalBytes), w.NewBatch(), nil
+}
+
+func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {
+
+	emulatedBatch, ok := batch.(*store.EmulatedBatch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	tx, err := w.store.db.Begin(true)
+	if err != nil {
+		return
+	}
+	// defer function to ensure that once started,
+	// we either Commit tx or Rollback
+	defer func() {
+		// if nothing went wrong, commit
+		if err == nil {
+			// careful to catch error here too
+			err = tx.Commit()
+		} else {
+			// caller should see error that caused abort,
+			// not success or failure of Rollback itself
+			_ = tx.Rollback()
+		}
+	}()
+
+	bucket := tx.Bucket([]byte(w.store.bucket))
+	bucket.FillPercent = w.store.fillPercent
+
+	for k, mergeOps := range emulatedBatch.Merger.Merges {
+		kb := []byte(k)
+		existingVal := bucket.Get(kb)
+		mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			err = fmt.Errorf("merge operator returned failure")
+			return
+		}
+		err = bucket.Put(kb, mergedVal)
+		if err != nil {
+			return
+		}
+	}
+
+	for _, op := range emulatedBatch.Ops {
+		if op.V != nil {
+			err = bucket.Put(op.K, op.V)
+			if err != nil {
+				return
+			}
+		} else {
+			err = bucket.Delete(op.K)
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func (w *Writer) Close() error {
+	return nil
+}

index/upsidedown/store/goleveldb/batch.go

@@ -0,0 +1,50 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/goleveldb/leveldb"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Batch struct {
+	store *Store
+	merge *store.EmulatedMerge
+	batch *leveldb.Batch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.batch.Put(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.batch.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Reset() {
+	b.batch.Reset()
+	b.merge = store.NewEmulatedMerge(b.store.mo)
+}
+
+func (b *Batch) Close() error {
+	b.batch.Reset()
+	b.batch = nil
+	b.merge = nil
+	return nil
+}

index/upsidedown/store/goleveldb/config.go

@@ -0,0 +1,66 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/goleveldb/leveldb/filter"
+	"github.com/blevesearch/goleveldb/leveldb/opt"
+)
+
+func applyConfig(o *opt.Options, config map[string]interface{}) (*opt.Options, error) {
+
+	ro, ok := config["read_only"].(bool)
+	if ok {
+		o.ReadOnly = ro
+	}
+
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		o.ErrorIfMissing = !cim
+	}
+
+	eie, ok := config["error_if_exists"].(bool)
+	if ok {
+		o.ErrorIfExist = eie
+	}
+
+	wbs, ok := config["write_buffer_size"].(float64)
+	if ok {
+		o.WriteBuffer = int(wbs)
+	}
+
+	bs, ok := config["block_size"].(float64)
+	if ok {
+		o.BlockSize = int(bs)
+	}
+
+	bri, ok := config["block_restart_interval"].(float64)
+	if ok {
+		o.BlockRestartInterval = int(bri)
+	}
+
+	lcc, ok := config["lru_cache_capacity"].(float64)
+	if ok {
+		o.BlockCacheCapacity = int(lcc)
+	}
+
+	bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
+	if ok {
+		bf := filter.NewBloomFilter(int(bfbpk))
+		o.Filter = bf
+	}
+
+	return o, nil
+}

index/upsidedown/store/goleveldb/iterator.go

@@ -0,0 +1,54 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import "github.com/blevesearch/goleveldb/leveldb/iterator"
+
+type Iterator struct {
+	store    *Store
+	iterator iterator.Iterator
+}
+
+func (ldi *Iterator) Seek(key []byte) {
+	ldi.iterator.Seek(key)
+}
+
+func (ldi *Iterator) Next() {
+	ldi.iterator.Next()
+}
+
+func (ldi *Iterator) Current() ([]byte, []byte, bool) {
+	if ldi.Valid() {
+		return ldi.Key(), ldi.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (ldi *Iterator) Key() []byte {
+	return ldi.iterator.Key()
+}
+
+func (ldi *Iterator) Value() []byte {
+	return ldi.iterator.Value()
+}
+
+func (ldi *Iterator) Valid() bool {
+	return ldi.iterator.Valid()
+}
+
+func (ldi *Iterator) Close() error {
+	ldi.iterator.Release()
+	return nil
+}

index/upsidedown/store/goleveldb/reader.go

@@ -0,0 +1,68 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/goleveldb/leveldb"
+	"github.com/blevesearch/goleveldb/leveldb/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *leveldb.Snapshot
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	b, err := r.snapshot.Get(key, r.store.defaultReadOptions)
+	if err == leveldb.ErrNotFound {
+		return nil, nil
+	}
+	return b, err
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
+	byteRange := util.BytesPrefix(prefix)
+	iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)
+	iter.First()
+	rv := Iterator{
+		store:    r.store,
+		iterator: iter,
+	}
+	return &rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	byteRange := &util.Range{
+		Start: start,
+		Limit: end,
+	}
+	iter := r.snapshot.NewIterator(byteRange, r.store.defaultReadOptions)
+	iter.First()
+	rv := Iterator{
+		store:    r.store,
+		iterator: iter,
+	}
+	return &rv
+}
+
+func (r *Reader) Close() error {
+	r.snapshot.Release()
+	return nil
+}

index/upsidedown/store/goleveldb/store.go

@@ -0,0 +1,152 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/goleveldb/leveldb"
+	"github.com/blevesearch/goleveldb/leveldb/opt"
+	"github.com/blevesearch/goleveldb/leveldb/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+const (
+	Name                    = "goleveldb"
+	defaultCompactBatchSize = 250
+)
+
+type Store struct {
+	path string
+	opts *opt.Options
+	db   *leveldb.DB
+	mo   store.MergeOperator
+
+	defaultWriteOptions *opt.WriteOptions
+	defaultReadOptions  *opt.ReadOptions
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
+
+	opts, err := applyConfig(&opt.Options{}, config)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := leveldb.OpenFile(path, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	rv := Store{
+		path:                path,
+		opts:                opts,
+		db:                  db,
+		mo:                  mo,
+		defaultReadOptions:  &opt.ReadOptions{},
+		defaultWriteOptions: &opt.WriteOptions{},
+	}
+	rv.defaultWriteOptions.Sync = true
+	return &rv, nil
+}
+
+func (ldbs *Store) Close() error {
+	return ldbs.db.Close()
+}
+
+func (ldbs *Store) Reader() (store.KVReader, error) {
+	snapshot, _ := ldbs.db.GetSnapshot()
+	return &Reader{
+		store:    ldbs,
+		snapshot: snapshot,
+	}, nil
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return &Writer{
+		store: ldbs,
+	}, nil
+}
+
+// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches), then
+// compacts the underlying goleveldb store.  Removing entries is a workaround for github issue #374.
+func (ldbs *Store) CompactWithBatchSize(batchSize int) error {
+	// workaround for github issue #374 - remove DictionaryTerm keys with count=0
+	batch := &leveldb.Batch{}
+	for {
+		t, err := ldbs.db.OpenTransaction()
+		if err != nil {
+			return err
+		}
+		iter := t.NewIterator(util.BytesPrefix([]byte("d")), ldbs.defaultReadOptions)
+
+		for iter.Next() {
+			if bytes.Equal(iter.Value(), []byte{0}) {
+				k := append([]byte{}, iter.Key()...)
+				batch.Delete(k)
+			}
+			if batch.Len() == batchSize {
+				break
+			}
+		}
+		iter.Release()
+		if iter.Error() != nil {
+			t.Discard()
+			return iter.Error()
+		}
+
+		if batch.Len() > 0 {
+			err := t.Write(batch, ldbs.defaultWriteOptions)
+			if err != nil {
+				t.Discard()
+				return err
+			}
+			err = t.Commit()
+			if err != nil {
+				return err
+			}
+		} else {
+			t.Discard()
+			break
+		}
+		batch.Reset()
+	}
+
+	return ldbs.db.CompactRange(util.Range{Start: nil, Limit: nil})
+}
+
+// Compact compacts the underlying goleveldb store.  The current implementation includes a workaround
+// for github issue #374 (see CompactWithBatchSize).
+func (ldbs *Store) Compact() error {
+	return ldbs.CompactWithBatchSize(defaultCompactBatchSize)
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/goleveldb/store_test.go

@@ -0,0 +1,99 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"os"
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, map[string]interface{}{
+		"path":              "test",
+		"create_if_missing": true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.RemoveAll("test")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGoLevelDBKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestGoLevelDBReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestGoLevelDBReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestGoLevelDBWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestGoLevelDBPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestGoLevelDBPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestGoLevelDBRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestGoLevelDBRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestGoLevelDBMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/goleveldb/writer.go

@@ -0,0 +1,68 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package goleveldb
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/goleveldb/leveldb"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	rv := Batch{
+		store: w.store,
+		merge: store.NewEmulatedMerge(w.store.mo),
+		batch: new(leveldb.Batch),
+	}
+	return &rv
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	return make([]byte, options.TotalBytes), w.NewBatch(), nil
+}
+
+func (w *Writer) ExecuteBatch(b store.KVBatch) error {
+	batch, ok := b.(*Batch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	// first process merges
+	for k, mergeOps := range batch.merge.Merges {
+		kb := []byte(k)
+		existingVal, err := w.store.db.Get(kb, w.store.defaultReadOptions)
+		if err != nil && err != leveldb.ErrNotFound {
+			return err
+		}
+		mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			return fmt.Errorf("merge operator returned failure")
+		}
+		// add the final merge to this batch
+		batch.batch.Put(kb, mergedVal)
+	}
+
+	// now execute the batch
+	return w.store.db.Write(batch.batch, w.store.defaultWriteOptions)
+}
+
+func (w *Writer) Close() error {
+	return nil
+}

index/upsidedown/store/gtreap/iterator.go

@@ -0,0 +1,152 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"bytes"
+	"sync"
+
+	"github.com/blevesearch/gtreap"
+)
+
+type Iterator struct {
+	t *gtreap.Treap
+
+	m        sync.Mutex
+	cancelCh chan struct{}
+	nextCh   chan *Item
+	curr     *Item
+	currOk   bool
+
+	prefix []byte
+	start  []byte
+	end    []byte
+}
+
+func (w *Iterator) Seek(k []byte) {
+	if w.start != nil && bytes.Compare(k, w.start) < 0 {
+		k = w.start
+	}
+	if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) {
+		if bytes.Compare(k, w.prefix) < 0 {
+			k = w.prefix
+		} else {
+			var end []byte
+			for i := len(w.prefix) - 1; i >= 0; i-- {
+				c := w.prefix[i]
+				if c < 0xff {
+					end = make([]byte, i+1)
+					copy(end, w.prefix)
+					end[i] = c + 1
+					break
+				}
+			}
+			k = end
+		}
+	}
+	w.restart(&Item{k: k})
+}
+
+func (w *Iterator) restart(start *Item) *Iterator {
+	cancelCh := make(chan struct{})
+	nextCh := make(chan *Item, 1)
+
+	w.m.Lock()
+	if w.cancelCh != nil {
+		close(w.cancelCh)
+	}
+	w.cancelCh = cancelCh
+	w.nextCh = nextCh
+	w.curr = nil
+	w.currOk = false
+	w.m.Unlock()
+
+	go func() {
+		if start != nil {
+			w.t.VisitAscend(start, func(itm gtreap.Item) bool {
+				select {
+				case <-cancelCh:
+					return false
+				case nextCh <- itm.(*Item):
+					return true
+				}
+			})
+		}
+		close(nextCh)
+	}()
+
+	w.Next()
+
+	return w
+}
+
+func (w *Iterator) Next() {
+	w.m.Lock()
+	nextCh := w.nextCh
+	w.m.Unlock()
+	w.curr, w.currOk = <-nextCh
+}
+
+func (w *Iterator) Current() ([]byte, []byte, bool) {
+	w.m.Lock()
+	defer w.m.Unlock()
+	if !w.currOk || w.curr == nil {
+		return nil, nil, false
+	}
+	if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) {
+		return nil, nil, false
+	} else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 {
+		return nil, nil, false
+	}
+	return w.curr.k, w.curr.v, w.currOk
+}
+
+func (w *Iterator) Key() []byte {
+	k, _, ok := w.Current()
+	if !ok {
+		return nil
+	}
+	return k
+}
+
+func (w *Iterator) Value() []byte {
+	_, v, ok := w.Current()
+	if !ok {
+		return nil
+	}
+	return v
+}
+
+func (w *Iterator) Valid() bool {
+	_, _, ok := w.Current()
+	return ok
+}
+
+func (w *Iterator) Close() error {
+	w.m.Lock()
+	if w.cancelCh != nil {
+		close(w.cancelCh)
+	}
+	w.cancelCh = nil
+	w.nextCh = nil
+	w.curr = nil
+	w.currOk = false
+	w.m.Unlock()
+
+	return nil
+}

index/upsidedown/store/gtreap/reader.go

@@ -0,0 +1,66 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"github.com/blevesearch/upsidedown_store_api"
+
+	"github.com/blevesearch/gtreap"
+)
+
+type Reader struct {
+	t *gtreap.Treap
+}
+
+func (w *Reader) Get(k []byte) (v []byte, err error) {
+	var rv []byte
+	itm := w.t.Get(&Item{k: k})
+	if itm != nil {
+		rv = make([]byte, len(itm.(*Item).v))
+		copy(rv, itm.(*Item).v)
+		return rv, nil
+	}
+	return nil, nil
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (w *Reader) PrefixIterator(k []byte) store.KVIterator {
+	rv := Iterator{
+		t:      w.t,
+		prefix: k,
+	}
+	rv.restart(&Item{k: k})
+	return &rv
+}
+
+func (w *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	rv := Iterator{
+		t:     w.t,
+		start: start,
+		end:   end,
+	}
+	rv.restart(&Item{k: start})
+	return &rv
+}
+
+func (w *Reader) Close() error {
+	return nil
+}

index/upsidedown/store/gtreap/store.go

@@ -0,0 +1,85 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+
+package gtreap
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/gtreap"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+const Name = "gtreap"
+
+type Store struct {
+	m  sync.Mutex
+	t  *gtreap.Treap
+	mo store.MergeOperator
+}
+
+type Item struct {
+	k []byte
+	v []byte
+}
+
+func itemCompare(a, b interface{}) int {
+	return bytes.Compare(a.(*Item).k, b.(*Item).k)
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path != "" {
+		return nil, os.ErrInvalid
+	}
+
+	rv := Store{
+		t:  gtreap.NewTreap(itemCompare),
+		mo: mo,
+	}
+	return &rv, nil
+}
+
+func (s *Store) Close() error {
+	return nil
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	s.m.Lock()
+	t := s.t
+	s.m.Unlock()
+	return &Reader{t: t}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	return &Writer{s: s}, nil
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/gtreap/store_test.go

@@ -0,0 +1,93 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gtreap
+
+import (
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, map[string]interface{}{
+		"path": "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGTreapKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestGTreapReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestGTreapReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestGTreapWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestGTreapPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestGTreapPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestGTreapRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestGTreapRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestGTreapMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/gtreap/writer.go

@@ -0,0 +1,76 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"fmt"
+	"math/rand"
+
+	"github.com/blevesearch/upsidedown_store_api"
+)
+
+type Writer struct {
+	s *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w.s.mo)
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	return make([]byte, options.TotalBytes), w.NewBatch(), nil
+}
+
+func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
+
+	emulatedBatch, ok := batch.(*store.EmulatedBatch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	w.s.m.Lock()
+	for k, mergeOps := range emulatedBatch.Merger.Merges {
+		kb := []byte(k)
+		var existingVal []byte
+		existingItem := w.s.t.Get(&Item{k: kb})
+		if existingItem != nil {
+			existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v
+		}
+		mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			return fmt.Errorf("merge operator returned failure")
+		}
+		w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int())
+	}
+
+	for _, op := range emulatedBatch.Ops {
+		if op.V != nil {
+			w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int())
+		} else {
+			w.s.t = w.s.t.Delete(&Item{k: op.K})
+		}
+	}
+	w.s.m.Unlock()
+
+	return nil
+}
+
+func (w *Writer) Close() error {
+	w.s = nil
+	return nil
+}

index/upsidedown/store/metrics/batch.go

@@ -0,0 +1,46 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import store "github.com/blevesearch/upsidedown_store_api"
+
+type Batch struct {
+	s *Store
+	o store.KVBatch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.o.Set(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.o.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.s.timerBatchMerge.Time(func() {
+		b.o.Merge(key, val)
+	})
+}
+
+func (b *Batch) Reset() {
+	b.o.Reset()
+}
+
+func (b *Batch) Close() error {
+	err := b.o.Close()
+	b.o = nil
+	return err
+}

index/upsidedown/store/metrics/iterator.go

@@ -0,0 +1,58 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import store "github.com/blevesearch/upsidedown_store_api"
+
+type Iterator struct {
+	s *Store
+	o store.KVIterator
+}
+
+func (i *Iterator) Seek(x []byte) {
+	i.s.timerIteratorSeek.Time(func() {
+		i.o.Seek(x)
+	})
+}
+
+func (i *Iterator) Next() {
+	i.s.timerIteratorNext.Time(func() {
+		i.o.Next()
+	})
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	return i.o.Current()
+}
+
+func (i *Iterator) Key() []byte {
+	return i.o.Key()
+}
+
+func (i *Iterator) Value() []byte {
+	return i.o.Value()
+}
+
+func (i *Iterator) Valid() bool {
+	return i.o.Valid()
+}
+
+func (i *Iterator) Close() error {
+	err := i.o.Close()
+	if err != nil {
+		i.s.AddError("Iterator.Close", err, nil)
+	}
+	return err
+}

index/upsidedown/store/metrics/metrics_test.go

@@ -0,0 +1,141 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap"
+)
+
+func TestMetricsStore(t *testing.T) {
+	_, err := New(nil, map[string]interface{}{})
+	if err == nil {
+		t.Errorf("expected err when bad config")
+	}
+
+	_, err = New(nil, map[string]interface{}{
+		"kvStoreName_actual": "some-invalid-kvstore-name",
+	})
+	if err == nil {
+		t.Errorf("expected err when unknown kvStoreName_actual")
+	}
+
+	s, err := New(nil, map[string]interface{}{
+		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := bytes.NewBuffer(nil)
+	err = s.(*Store).WriteJSON(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteJSON")
+	}
+	var m map[string]interface{}
+	err = json.Unmarshal(b.Bytes(), &m)
+	if err != nil {
+		t.Errorf("expected WriteJSON to be unmarshallable")
+	}
+	if len(m) == 0 {
+		t.Errorf("expected some entries")
+	}
+
+	b = bytes.NewBuffer(nil)
+	s.(*Store).WriteCSVHeader(b)
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteCSVHeader")
+	}
+
+	b = bytes.NewBuffer(nil)
+	s.(*Store).WriteCSV(b)
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteCSV")
+	}
+}
+
+func TestErrors(t *testing.T) {
+	s, err := New(nil, map[string]interface{}{
+		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	x, ok := s.(*Store)
+	if !ok {
+		t.Errorf("expecting a Store")
+	}
+
+	x.AddError("foo", fmt.Errorf("Foo"), []byte("fooKey"))
+	x.AddError("bar", fmt.Errorf("Bar"), nil)
+	x.AddError("baz", fmt.Errorf("Baz"), []byte("bazKey"))
+
+	b := bytes.NewBuffer(nil)
+	err = x.WriteJSON(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var m map[string]interface{}
+	err = json.Unmarshal(b.Bytes(), &m)
+	if err != nil {
+		t.Errorf("expected unmarshallable writeJSON, err: %v, b: %s",
+			err, b.Bytes())
+	}
+
+	errorsi, ok := m["Errors"]
+	if !ok || errorsi == nil {
+		t.Errorf("expected errorsi")
+	}
+	errors, ok := errorsi.([]interface{})
+	if !ok || errors == nil {
+		t.Errorf("expected errorsi is array")
+	}
+	if len(errors) != 3 {
+		t.Errorf("expected errors len 3")
+	}
+
+	e := errors[0].(map[string]interface{})
+	if e["Op"].(string) != "foo" ||
+		e["Err"].(string) != "Foo" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "fooKey" {
+		t.Errorf("expected foo, %#v", e)
+	}
+	e = errors[1].(map[string]interface{})
+	if e["Op"].(string) != "bar" ||
+		e["Err"].(string) != "Bar" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "" {
+		t.Errorf("expected bar, %#v", e)
+	}
+	e = errors[2].(map[string]interface{})
+	if e["Op"].(string) != "baz" ||
+		e["Err"].(string) != "Baz" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "bazKey" {
+		t.Errorf("expected baz, %#v", e)
+	}
+}

index/upsidedown/store/metrics/reader.go

@@ -0,0 +1,64 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import store "github.com/blevesearch/upsidedown_store_api"
+
+type Reader struct {
+	s *Store
+	o store.KVReader
+}
+
+func (r *Reader) Get(key []byte) (v []byte, err error) {
+	r.s.timerReaderGet.Time(func() {
+		v, err = r.o.Get(key)
+		if err != nil {
+			r.s.AddError("Reader.Get", err, key)
+		}
+	})
+	return
+}
+
+func (r *Reader) MultiGet(keys [][]byte) (vals [][]byte, err error) {
+	r.s.timerReaderMultiGet.Time(func() {
+		vals, err = r.o.MultiGet(keys)
+		if err != nil {
+			r.s.AddError("Reader.MultiGet", err, nil)
+		}
+	})
+	return
+}
+
+func (r *Reader) PrefixIterator(prefix []byte) (i store.KVIterator) {
+	r.s.timerReaderPrefixIterator.Time(func() {
+		i = &Iterator{s: r.s, o: r.o.PrefixIterator(prefix)}
+	})
+	return
+}
+
+func (r *Reader) RangeIterator(start, end []byte) (i store.KVIterator) {
+	r.s.timerReaderRangeIterator.Time(func() {
+		i = &Iterator{s: r.s, o: r.o.RangeIterator(start, end)}
+	})
+	return
+}
+
+func (r *Reader) Close() error {
+	err := r.o.Close()
+	if err != nil {
+		r.s.AddError("Reader.Close", err, nil)
+	}
+	return err
+}

index/upsidedown/store/metrics/stats.go

@@ -0,0 +1,50 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type stats struct {
+	s *Store
+}
+
+func (s *stats) statsMap() map[string]interface{} {
+	ms := map[string]interface{}{}
+
+	ms["metrics"] = map[string]interface{}{
+		"reader_get":             TimerMap(s.s.timerReaderGet),
+		"reader_multi_get":       TimerMap(s.s.timerReaderMultiGet),
+		"reader_prefix_iterator": TimerMap(s.s.timerReaderPrefixIterator),
+		"reader_range_iterator":  TimerMap(s.s.timerReaderRangeIterator),
+		"writer_execute_batch":   TimerMap(s.s.timerWriterExecuteBatch),
+		"iterator_seek":          TimerMap(s.s.timerIteratorSeek),
+		"iterator_next":          TimerMap(s.s.timerIteratorNext),
+		"batch_merge":            TimerMap(s.s.timerBatchMerge),
+	}
+
+	if o, ok := s.s.o.(store.KVStoreStats); ok {
+		ms["kv"] = o.StatsMap()
+	}
+
+	return ms
+}
+
+func (s *stats) MarshalJSON() ([]byte, error) {
+	m := s.statsMap()
+	return util.MarshalJSON(m)
+}

index/upsidedown/store/metrics/store.go

@@ -0,0 +1,277 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package metrics provides a bleve.store.KVStore implementation that
+// wraps another, real KVStore implementation, and uses go-metrics to
+// track runtime performance metrics.
+package metrics
+
+import (
+	"container/list"
+	"encoding/json"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/util"
+	"github.com/blevesearch/go-metrics"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+const Name = "metrics"
+
+type Store struct {
+	o store.KVStore
+
+	timerReaderGet            metrics.Timer
+	timerReaderMultiGet       metrics.Timer
+	timerReaderPrefixIterator metrics.Timer
+	timerReaderRangeIterator  metrics.Timer
+	timerWriterExecuteBatch   metrics.Timer
+	timerIteratorSeek         metrics.Timer
+	timerIteratorNext         metrics.Timer
+	timerBatchMerge           metrics.Timer
+
+	m      sync.Mutex // Protects the fields that follow.
+	errors *list.List // Capped list of StoreError's.
+
+	s *stats
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+
+	name, ok := config["kvStoreName_actual"].(string)
+	if !ok || name == "" {
+		return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
+			" config: %#v", config)
+	}
+
+	if name == Name {
+		return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
+	}
+
+	ctr := registry.KVStoreConstructorByName(name)
+	if ctr == nil {
+		return nil, fmt.Errorf("metrics: no kv store constructor,"+
+			" kvStoreName_actual: %s", name)
+	}
+
+	kvs, err := ctr(mo, config)
+	if err != nil {
+		return nil, err
+	}
+
+	rv := &Store{
+		o: kvs,
+
+		timerReaderGet:            metrics.NewTimer(),
+		timerReaderMultiGet:       metrics.NewTimer(),
+		timerReaderPrefixIterator: metrics.NewTimer(),
+		timerReaderRangeIterator:  metrics.NewTimer(),
+		timerWriterExecuteBatch:   metrics.NewTimer(),
+		timerIteratorSeek:         metrics.NewTimer(),
+		timerIteratorNext:         metrics.NewTimer(),
+		timerBatchMerge:           metrics.NewTimer(),
+
+		errors: list.New(),
+	}
+
+	rv.s = &stats{s: rv}
+
+	return rv, nil
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (s *Store) Close() error {
+	return s.o.Close()
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	o, err := s.o.Reader()
+	if err != nil {
+		s.AddError("Reader", err, nil)
+		return nil, err
+	}
+	return &Reader{s: s, o: o}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	o, err := s.o.Writer()
+	if err != nil {
+		s.AddError("Writer", err, nil)
+		return nil, err
+	}
+	return &Writer{s: s, o: o}, nil
+}
+
+// Metric specific code below:
+
+const MaxErrors = 100
+
+type StoreError struct {
+	Time string
+	Op   string
+	Err  string
+	Key  string
+}
+
+func (s *Store) AddError(op string, err error, key []byte) {
+	e := &StoreError{
+		Time: time.Now().Format(time.RFC3339Nano),
+		Op:   op,
+		Err:  fmt.Sprintf("%v", err),
+		Key:  string(key),
+	}
+
+	s.m.Lock()
+	for s.errors.Len() >= MaxErrors {
+		s.errors.Remove(s.errors.Front())
+	}
+	s.errors.PushBack(e)
+	s.m.Unlock()
+}
+
+func (s *Store) WriteJSON(w io.Writer) (err error) {
+	_, err = w.Write([]byte(`{"TimerReaderGet":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerReaderGet)
+	_, err = w.Write([]byte(`,"TimerReaderMultiGet":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerReaderMultiGet)
+	_, err = w.Write([]byte(`,"TimerReaderPrefixIterator":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerReaderPrefixIterator)
+	_, err = w.Write([]byte(`,"TimerReaderRangeIterator":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerReaderRangeIterator)
+	_, err = w.Write([]byte(`,"TimerWriterExecuteBatch":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerWriterExecuteBatch)
+	_, err = w.Write([]byte(`,"TimerIteratorSeek":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerIteratorSeek)
+	_, err = w.Write([]byte(`,"TimerIteratorNext":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerIteratorNext)
+	_, err = w.Write([]byte(`,"TimerBatchMerge":`))
+	if err != nil {
+		return
+	}
+	WriteTimerJSON(w, s.timerBatchMerge)
+
+	_, err = w.Write([]byte(`,"Errors":[`))
+	if err != nil {
+		return
+	}
+	s.m.Lock()
+	defer s.m.Unlock()
+	e := s.errors.Front()
+	i := 0
+	for e != nil {
+		se, ok := e.Value.(*StoreError)
+		if ok && se != nil {
+			if i > 0 {
+				_, err = w.Write([]byte(","))
+				if err != nil {
+					return
+				}
+			}
+			var buf []byte
+			buf, err = util.MarshalJSON(se)
+			if err == nil {
+				_, err = w.Write(buf)
+				if err != nil {
+					return
+				}
+			}
+		}
+		e = e.Next()
+		i = i + 1
+	}
+	_, err = w.Write([]byte(`]`))
+	if err != nil {
+		return
+	}
+
+	// see if the underlying implementation has its own stats
+	if o, ok := s.o.(store.KVStoreStats); ok {
+		storeStats := o.Stats()
+		var storeBytes []byte
+		storeBytes, err = util.MarshalJSON(storeStats)
+		if err != nil {
+			return
+		}
+		_, err = fmt.Fprintf(w, `, "store": %s`, string(storeBytes))
+		if err != nil {
+			return
+		}
+	}
+
+	_, err = w.Write([]byte(`}`))
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (s *Store) WriteCSVHeader(w io.Writer) {
+	WriteTimerCSVHeader(w, "TimerReaderGet")
+	WriteTimerCSVHeader(w, "TimerReaderPrefixIterator")
+	WriteTimerCSVHeader(w, "TimerReaderRangeIterator")
+	WriteTimerCSVHeader(w, "TimerWtierExecuteBatch")
+	WriteTimerCSVHeader(w, "TimerIteratorSeek")
+	WriteTimerCSVHeader(w, "TimerIteratorNext")
+	WriteTimerCSVHeader(w, "TimerBatchMerge")
+}
+
+func (s *Store) WriteCSV(w io.Writer) {
+	WriteTimerCSV(w, s.timerReaderGet)
+	WriteTimerCSV(w, s.timerReaderPrefixIterator)
+	WriteTimerCSV(w, s.timerReaderRangeIterator)
+	WriteTimerCSV(w, s.timerWriterExecuteBatch)
+	WriteTimerCSV(w, s.timerIteratorSeek)
+	WriteTimerCSV(w, s.timerIteratorNext)
+	WriteTimerCSV(w, s.timerBatchMerge)
+}
+
+func (s *Store) Stats() json.Marshaler {
+	return s.s
+}
+
+func (s *Store) StatsMap() map[string]interface{} {
+	return s.s.statsMap()
+}

index/upsidedown/store/metrics/store_test.go

@@ -0,0 +1,95 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/v2/index/upsidedown/store/gtreap"
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, map[string]interface{}{
+		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMetricsKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestMetricsReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestMetricsReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestMetricsWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestMetricsPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestMetricsPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestMetricsRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestMetricsRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestMetricsMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/metrics/util.go

@@ -0,0 +1,135 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/blevesearch/go-metrics"
+)
+
+// NOTE: This is copy & pasted from cbft as otherwise there
+// would be an import cycle.
+
+var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}
+
+func TimerMap(timer metrics.Timer) map[string]interface{} {
+
+	rv := make(map[string]interface{})
+	t := timer.Snapshot()
+	p := t.Percentiles(timerPercentiles)
+
+	percentileKeys := []string{"median", "75%", "95%", "99%", "99.9%"}
+	percentiles := make(map[string]interface{})
+	for i, pi := range p {
+		if !isNanOrInf(pi) {
+			percentileKey := percentileKeys[i]
+			percentiles[percentileKey] = pi
+		}
+	}
+
+	rateKeys := []string{"1-min", "5-min", "15-min", "mean"}
+	rates := make(map[string]interface{})
+	for i, ri := range []float64{t.Rate1(), t.Rate5(), t.Rate15(), t.RateMean()} {
+		if !isNanOrInf(ri) {
+			rateKey := rateKeys[i]
+			rates[rateKey] = ri
+		}
+	}
+
+	rv["count"] = t.Count()
+	rv["min"] = t.Min()
+	rv["max"] = t.Max()
+	mean := t.Mean()
+	if !isNanOrInf(mean) {
+		rv["mean"] = mean
+	}
+	stddev := t.StdDev()
+	if !isNanOrInf(stddev) {
+		rv["stddev"] = stddev
+	}
+	rv["percentiles"] = percentiles
+	rv["rates"] = rates
+
+	return rv
+}
+
+func isNanOrInf(v float64) bool {
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return true
+	}
+	return false
+}
+
+func WriteTimerJSON(w io.Writer, timer metrics.Timer) {
+	t := timer.Snapshot()
+	p := t.Percentiles(timerPercentiles)
+
+	fmt.Fprintf(w, `{"count":%9d,`, t.Count())
+	fmt.Fprintf(w, `"min":%9d,`, t.Min())
+	fmt.Fprintf(w, `"max":%9d,`, t.Max())
+	fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean())
+	fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev())
+	fmt.Fprintf(w, `"percentiles":{`)
+	fmt.Fprintf(w, `"median":%12.2f,`, p[0])
+	fmt.Fprintf(w, `"75%%":%12.2f,`, p[1])
+	fmt.Fprintf(w, `"95%%":%12.2f,`, p[2])
+	fmt.Fprintf(w, `"99%%":%12.2f,`, p[3])
+	fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4])
+	fmt.Fprintf(w, `"rates":{`)
+	fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1())
+	fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5())
+	fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15())
+	fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean())
+}
+
+func WriteTimerCSVHeader(w io.Writer, prefix string) {
+	fmt.Fprintf(w, "%s-count,", prefix)
+	fmt.Fprintf(w, "%s-min,", prefix)
+	fmt.Fprintf(w, "%s-max,", prefix)
+	fmt.Fprintf(w, "%s-mean,", prefix)
+	fmt.Fprintf(w, "%s-stddev,", prefix)
+	fmt.Fprintf(w, "%s-percentile-50%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-75%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-95%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-99%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix)
+	fmt.Fprintf(w, "%s-rate-1-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-5-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-15-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-mean", prefix)
+}
+
+func WriteTimerCSV(w io.Writer, timer metrics.Timer) {
+	t := timer.Snapshot()
+	p := t.Percentiles(timerPercentiles)
+
+	fmt.Fprintf(w, `%d,`, t.Count())
+	fmt.Fprintf(w, `%d,`, t.Min())
+	fmt.Fprintf(w, `%d,`, t.Max())
+	fmt.Fprintf(w, `%f,`, t.Mean())
+	fmt.Fprintf(w, `%f,`, t.StdDev())
+	fmt.Fprintf(w, `%f,`, p[0])
+	fmt.Fprintf(w, `%f,`, p[1])
+	fmt.Fprintf(w, `%f,`, p[2])
+	fmt.Fprintf(w, `%f,`, p[3])
+	fmt.Fprintf(w, `%f,`, p[4])
+	fmt.Fprintf(w, `%f,`, t.Rate1())
+	fmt.Fprintf(w, `%f,`, t.Rate5())
+	fmt.Fprintf(w, `%f,`, t.Rate15())
+	fmt.Fprintf(w, `%f`, t.RateMean())
+}

index/upsidedown/store/metrics/writer.go

@@ -0,0 +1,60 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+	"fmt"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Writer struct {
+	s *Store
+	o store.KVWriter
+}
+
+func (w *Writer) Close() error {
+	err := w.o.Close()
+	if err != nil {
+		w.s.AddError("Writer.Close", err, nil)
+	}
+	return err
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return &Batch{s: w.s, o: w.o.NewBatch()}
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	buf, b, err := w.o.NewBatchEx(options)
+	if err != nil {
+		return nil, nil, err
+	}
+	return buf, &Batch{s: w.s, o: b}, nil
+}
+
+func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {
+	batch, ok := b.(*Batch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+	w.s.timerWriterExecuteBatch.Time(func() {
+		err = w.o.ExecuteBatch(batch.o)
+		if err != nil {
+			w.s.AddError("Writer.ExecuteBatch", err, nil)
+		}
+	})
+	return
+}

index/upsidedown/store/moss/batch.go

@@ -0,0 +1,87 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Batch struct {
+	store   *Store
+	merge   *store.EmulatedMerge
+	batch   moss.Batch
+	buf     []byte // Non-nil when using pre-alloc'ed / NewBatchEx().
+	bufUsed int
+}
+
+func (b *Batch) Set(key, val []byte) {
+	var err error
+	if b.buf != nil {
+		b.bufUsed += len(key) + len(val)
+		err = b.batch.AllocSet(key, val)
+	} else {
+		err = b.batch.Set(key, val)
+	}
+
+	if err != nil {
+		b.store.Logf("bleve moss batch.Set err: %v", err)
+	}
+}
+
+func (b *Batch) Delete(key []byte) {
+	var err error
+	if b.buf != nil {
+		b.bufUsed += len(key)
+		err = b.batch.AllocDel(key)
+	} else {
+		err = b.batch.Del(key)
+	}
+
+	if err != nil {
+		b.store.Logf("bleve moss batch.Delete err: %v", err)
+	}
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	if b.buf != nil {
+		b.bufUsed += len(key) + len(val)
+	}
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Reset() {
+	err := b.Close()
+	if err != nil {
+		b.store.Logf("bleve moss batch.Close err: %v", err)
+		return
+	}
+
+	batch, err := b.store.ms.NewBatch(0, 0)
+	if err == nil {
+		b.batch = batch
+		b.merge = store.NewEmulatedMerge(b.store.mo)
+		b.buf = nil
+		b.bufUsed = 0
+	}
+}
+
+func (b *Batch) Close() error {
+	b.merge = nil
+	err := b.batch.Close()
+	b.batch = nil
+	return err
+}

index/upsidedown/store/moss/iterator.go

@@ -0,0 +1,87 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+)
+
+type Iterator struct {
+	store *Store
+	ss    moss.Snapshot
+	iter  moss.Iterator
+	start []byte
+	end   []byte
+	k     []byte
+	v     []byte
+	err   error
+}
+
+func (x *Iterator) Seek(seekToKey []byte) {
+	_ = x.iter.SeekTo(seekToKey)
+
+	x.k, x.v, x.err = x.iter.Current()
+}
+
+func (x *Iterator) Next() {
+	_ = x.iter.Next()
+
+	x.k, x.v, x.err = x.iter.Current()
+}
+
+func (x *Iterator) Current() ([]byte, []byte, bool) {
+	return x.k, x.v, x.err == nil
+}
+
+func (x *Iterator) Key() []byte {
+	if x.err != nil {
+		return nil
+	}
+
+	return x.k
+}
+
+func (x *Iterator) Value() []byte {
+	if x.err != nil {
+		return nil
+	}
+
+	return x.v
+}
+
+func (x *Iterator) Valid() bool {
+	return x.err == nil
+}
+
+func (x *Iterator) Close() error {
+	var err error
+
+	x.ss = nil
+
+	if x.iter != nil {
+		err = x.iter.Close()
+		x.iter = nil
+	}
+
+	x.k = nil
+	x.v = nil
+	x.err = moss.ErrIteratorDone
+
+	return err
+}
+
+func (x *Iterator) current() {
+	x.k, x.v, x.err = x.iter.Current()
+}

index/upsidedown/store/moss/lower.go

@@ -0,0 +1,571 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package moss provides a KVStore implementation based on the
+// github.com/couchbase/moss library.
+
+package moss
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/couchbase/moss"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+func initLowerLevelStore(
+	config map[string]interface{},
+	lowerLevelStoreName string,
+	lowerLevelStoreConfig map[string]interface{},
+	lowerLevelMaxBatchSize uint64,
+	options moss.CollectionOptions,
+) (moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error) {
+	if lowerLevelStoreConfig == nil {
+		lowerLevelStoreConfig = map[string]interface{}{}
+	}
+
+	for k, v := range config {
+		_, exists := lowerLevelStoreConfig[k]
+		if !exists {
+			lowerLevelStoreConfig[k] = v
+		}
+	}
+
+	if lowerLevelStoreName == "mossStore" {
+		return InitMossStore(lowerLevelStoreConfig, options)
+	}
+
+	constructor := registry.KVStoreConstructorByName(lowerLevelStoreName)
+	if constructor == nil {
+		return nil, nil, nil, nil, fmt.Errorf("moss store, initLowerLevelStore,"+
+			" could not find lower level store: %s", lowerLevelStoreName)
+	}
+
+	kvStore, err := constructor(options.MergeOperator, lowerLevelStoreConfig)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	llStore := &llStore{
+		refs:     0,
+		config:   config,
+		llConfig: lowerLevelStoreConfig,
+		kvStore:  kvStore,
+		logf:     options.Log,
+	}
+
+	llUpdate := func(ssHigher moss.Snapshot) (ssLower moss.Snapshot, err error) {
+		return llStore.update(ssHigher, lowerLevelMaxBatchSize)
+	}
+
+	llSnapshot, err := llUpdate(nil)
+	if err != nil {
+		_ = kvStore.Close()
+		return nil, nil, nil, nil, err
+	}
+
+	return llSnapshot, llUpdate, kvStore, nil, nil // llStore.refs is now 1.
+}
+
+// ------------------------------------------------
+
+// llStore is a lower level store and provides ref-counting around a
+// bleve store.KVStore.
+type llStore struct {
+	kvStore store.KVStore
+
+	config   map[string]interface{}
+	llConfig map[string]interface{}
+
+	logf func(format string, a ...interface{})
+
+	m    sync.Mutex // Protects fields that follow.
+	refs int
+}
+
+// llSnapshot represents a lower-level snapshot, wrapping a bleve
+// store.KVReader, and implements the moss.Snapshot interface.
+type llSnapshot struct {
+	llStore        *llStore // Holds 1 refs on the llStore.
+	kvReader       store.KVReader
+	childSnapshots map[string]*llSnapshot
+
+	m    sync.Mutex // Protects fields that follow.
+	refs int
+}
+
+// llIterator represents a lower-level iterator, wrapping a bleve
+// store.KVIterator, and implements the moss.Iterator interface.
+type llIterator struct {
+	llSnapshot *llSnapshot // Holds 1 refs on the llSnapshot.
+
+	// Some lower-level KVReader implementations need a separate
+	// KVReader clone, due to KVReader single-threaded'ness.
+	kvReader store.KVReader
+
+	kvIterator store.KVIterator
+}
+
+type readerSource interface {
+	Reader() (store.KVReader, error)
+}
+
+// ------------------------------------------------
+
+func (s *llStore) addRef() *llStore {
+	s.m.Lock()
+	s.refs += 1
+	s.m.Unlock()
+
+	return s
+}
+
+func (s *llStore) decRef() {
+	s.m.Lock()
+	s.refs -= 1
+	if s.refs <= 0 {
+		err := s.kvStore.Close()
+		if err != nil {
+			s.logf("llStore kvStore.Close err: %v", err)
+		}
+	}
+	s.m.Unlock()
+}
+
+// update() mutates this lower level store with latest data from the
+// given higher level moss.Snapshot and returns a new moss.Snapshot
+// that the higher level can use which represents this lower level
+// store.
+func (s *llStore) update(ssHigher moss.Snapshot, maxBatchSize uint64) (
+	ssLower moss.Snapshot, err error,
+) {
+	if ssHigher != nil {
+		iter, err := ssHigher.StartIterator(nil, nil, moss.IteratorOptions{
+			IncludeDeletions: true,
+			SkipLowerLevel:   true,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			err = iter.Close()
+			if err != nil {
+				s.logf("llStore iter.Close err: %v", err)
+			}
+		}()
+
+		kvWriter, err := s.kvStore.Writer()
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			err = kvWriter.Close()
+			if err != nil {
+				s.logf("llStore kvWriter.Close err: %v", err)
+			}
+		}()
+
+		batch := kvWriter.NewBatch()
+
+		defer func() {
+			if batch != nil {
+				err = batch.Close()
+				if err != nil {
+					s.logf("llStore batch.Close err: %v", err)
+				}
+			}
+		}()
+
+		var readOptions moss.ReadOptions
+
+		i := uint64(0)
+		for {
+			if i%1000000 == 0 {
+				s.logf("llStore.update, i: %d", i)
+			}
+
+			ex, key, val, err := iter.CurrentEx()
+			if err == moss.ErrIteratorDone {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+
+			switch ex.Operation {
+			case moss.OperationSet:
+				batch.Set(key, val)
+
+			case moss.OperationDel:
+				batch.Delete(key)
+
+			case moss.OperationMerge:
+				val, err = ssHigher.Get(key, readOptions)
+				if err != nil {
+					return nil, err
+				}
+
+				if val != nil {
+					batch.Set(key, val)
+				} else {
+					batch.Delete(key)
+				}
+
+			default:
+				return nil, fmt.Errorf("moss store, update,"+
+					" unexpected operation, ex: %v", ex)
+			}
+
+			i++
+
+			err = iter.Next()
+			if err == moss.ErrIteratorDone {
+				break
+			}
+			if err != nil {
+				return nil, err
+			}
+
+			if maxBatchSize > 0 && i%maxBatchSize == 0 {
+				err = kvWriter.ExecuteBatch(batch)
+				if err != nil {
+					return nil, err
+				}
+
+				err = batch.Close()
+				if err != nil {
+					return nil, err
+				}
+
+				batch = kvWriter.NewBatch()
+			}
+		}
+
+		if i > 0 {
+			s.logf("llStore.update, ExecuteBatch,"+
+				" path: %s, total: %d, start", s.llConfig["path"], i)
+
+			err = kvWriter.ExecuteBatch(batch)
+			if err != nil {
+				return nil, err
+			}
+
+			s.logf("llStore.update, ExecuteBatch,"+
+				" path: %s: total: %d, done", s.llConfig["path"], i)
+		}
+	}
+
+	kvReader, err := s.kvStore.Reader()
+	if err != nil {
+		return nil, err
+	}
+
+	s.logf("llStore.update, new reader")
+
+	return &llSnapshot{
+		llStore:  s.addRef(),
+		kvReader: kvReader,
+		refs:     1,
+	}, nil
+}
+
+// ------------------------------------------------
+
+func (llss *llSnapshot) addRef() *llSnapshot {
+	llss.m.Lock()
+	llss.refs += 1
+	llss.m.Unlock()
+
+	return llss
+}
+
+func (llss *llSnapshot) decRef() {
+	llss.m.Lock()
+	llss.refs -= 1
+	if llss.refs <= 0 {
+		if llss.kvReader != nil {
+			err := llss.kvReader.Close()
+			if err != nil {
+				llss.llStore.logf("llSnapshot kvReader.Close err: %v", err)
+			}
+
+			llss.kvReader = nil
+		}
+
+		if llss.llStore != nil {
+			llss.llStore.decRef()
+			llss.llStore = nil
+		}
+	}
+	llss.m.Unlock()
+}
+
+// ChildCollectionNames returns an array of child collection name strings.
+func (llss *llSnapshot) ChildCollectionNames() ([]string, error) {
+	childCollections := make([]string, len(llss.childSnapshots))
+	idx := 0
+	for name := range llss.childSnapshots {
+		childCollections[idx] = name
+		idx++
+	}
+	return childCollections, nil
+}
+
+// ChildCollectionSnapshot returns a Snapshot on a given child
+// collection by its name.
+func (llss *llSnapshot) ChildCollectionSnapshot(childCollectionName string) (
+	moss.Snapshot, error,
+) {
+	childSnapshot, exists := llss.childSnapshots[childCollectionName]
+	if !exists {
+		return nil, nil
+	}
+	childSnapshot.addRef()
+	return childSnapshot, nil
+}
+
+func (llss *llSnapshot) Close() error {
+	llss.decRef()
+
+	return nil
+}
+
+func (llss *llSnapshot) Get(key []byte,
+	readOptions moss.ReadOptions,
+) ([]byte, error) {
+	rs, ok := llss.kvReader.(readerSource)
+	if ok {
+		r2, err := rs.Reader()
+		if err != nil {
+			return nil, err
+		}
+
+		val, err := r2.Get(key)
+
+		_ = r2.Close()
+
+		return val, err
+	}
+
+	return llss.kvReader.Get(key)
+}
+
+func (llss *llSnapshot) StartIterator(
+	startKeyInclusive, endKeyExclusive []byte,
+	iteratorOptions moss.IteratorOptions,
+) (moss.Iterator, error) {
+	rs, ok := llss.kvReader.(readerSource)
+	if ok {
+		r2, err := rs.Reader()
+		if err != nil {
+			return nil, err
+		}
+
+		i2 := r2.RangeIterator(startKeyInclusive, endKeyExclusive)
+
+		return &llIterator{llSnapshot: llss.addRef(), kvReader: r2, kvIterator: i2}, nil
+	}
+
+	i := llss.kvReader.RangeIterator(startKeyInclusive, endKeyExclusive)
+
+	return &llIterator{llSnapshot: llss.addRef(), kvReader: nil, kvIterator: i}, nil
+}
+
+// ------------------------------------------------
+
+func (lli *llIterator) Close() error {
+	var err0 error
+	if lli.kvIterator != nil {
+		err0 = lli.kvIterator.Close()
+		lli.kvIterator = nil
+	}
+
+	var err1 error
+	if lli.kvReader != nil {
+		err1 = lli.kvReader.Close()
+		lli.kvReader = nil
+	}
+
+	lli.llSnapshot.decRef()
+	lli.llSnapshot = nil
+
+	if err0 != nil {
+		return err0
+	}
+
+	if err1 != nil {
+		return err1
+	}
+
+	return nil
+}
+
+func (lli *llIterator) Next() error {
+	lli.kvIterator.Next()
+
+	return nil
+}
+
+func (lli *llIterator) SeekTo(k []byte) error {
+	lli.kvIterator.Seek(k)
+
+	return nil
+}
+
+func (lli *llIterator) Current() (key, val []byte, err error) {
+	key, val, ok := lli.kvIterator.Current()
+	if !ok {
+		return nil, nil, moss.ErrIteratorDone
+	}
+
+	return key, val, nil
+}
+
+func (lli *llIterator) CurrentEx() (
+	entryEx moss.EntryEx, key, val []byte, err error,
+) {
+	return moss.EntryEx{}, nil, nil, moss.ErrUnimplemented
+}
+
+// ------------------------------------------------
+
+func InitMossStore(config map[string]interface{}, options moss.CollectionOptions) (
+	moss.Snapshot, moss.LowerLevelUpdate, store.KVStore, statsFunc, error,
+) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
+	}
+	if path == "" {
+		return nil, nil, nil, nil, os.ErrInvalid
+	}
+
+	err := os.MkdirAll(path, 0o700)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("lower: InitMossStore mkdir,"+
+			" path: %s, err: %v", path, err)
+	}
+
+	storeOptions := moss.StoreOptions{
+		CollectionOptions: options,
+	}
+	v, ok := config["mossStoreOptions"]
+	if ok {
+		b, err := util.MarshalJSON(v) // Convert from map[string]interface{}.
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+
+		err = util.UnmarshalJSON(b, &storeOptions)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
+
+	s, err := moss.OpenStore(path, storeOptions)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("lower: moss.OpenStore,"+
+			" path: %s, err: %v", path, err)
+	}
+
+	sw := &mossStoreWrapper{s: s}
+
+	llUpdate := func(ssHigher moss.Snapshot) (moss.Snapshot, error) {
+		ss, err := sw.s.Persist(ssHigher, moss.StorePersistOptions{
+			CompactionConcern: moss.CompactionAllow,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sw.AddRef() // Ref-count to be owned by snapshot wrapper.
+
+		return moss.NewSnapshotWrapper(ss, sw), nil
+	}
+
+	llSnapshot, err := llUpdate(nil)
+	if err != nil {
+		_ = s.Close()
+		return nil, nil, nil, nil, err
+	}
+
+	llStats := func() map[string]interface{} {
+		stats, err := s.Stats()
+		if err != nil {
+			return nil
+		}
+		return stats
+	}
+
+	return llSnapshot, llUpdate, sw, llStats, nil
+}
+
+// mossStoreWrapper implements the bleve.index.store.KVStore
+// interface, but only barely enough to allow it to be passed around
+// as a lower-level store.  Advanced apps will likely cast the
+// mossStoreWrapper to access the Actual() method.
+type mossStoreWrapper struct {
+	m    sync.Mutex
+	refs int
+	s    *moss.Store
+}
+
+func (w *mossStoreWrapper) AddRef() {
+	w.m.Lock()
+	w.refs++
+	w.m.Unlock()
+}
+
+func (w *mossStoreWrapper) Close() (err error) {
+	w.m.Lock()
+	w.refs--
+	if w.refs <= 0 {
+		err = w.s.Close()
+		w.s = nil
+	}
+	w.m.Unlock()
+	return err
+}
+
+func (w *mossStoreWrapper) Reader() (store.KVReader, error) {
+	return nil, fmt.Errorf("unexpected")
+}
+
+func (w *mossStoreWrapper) Writer() (store.KVWriter, error) {
+	return nil, fmt.Errorf("unexpected")
+}
+
+func (w *mossStoreWrapper) Actual() *moss.Store {
+	w.m.Lock()
+	rv := w.s
+	w.m.Unlock()
+	return rv
+}
+
+func (w *mossStoreWrapper) histograms() string {
+	var rv string
+	w.m.Lock()
+	if w.s != nil {
+		rv = w.s.Histograms().String()
+	}
+	w.m.Unlock()
+	return rv
+}

index/upsidedown/store/moss/lower_test.go

@@ -0,0 +1,103 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"os"
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func openWithLower(t *testing.T, mo store.MergeOperator) (string, store.KVStore) {
+	tmpDir, _ := os.MkdirTemp("", "mossStore")
+
+	config := map[string]interface{}{
+		"path":                    tmpDir,
+		"mossLowerLevelStoreName": "mossStore",
+	}
+
+	rv, err := New(mo, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tmpDir, rv
+}
+
+func cleanupWithLower(t *testing.T, s store.KVStore, tmpDir string) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.RemoveAll(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMossWithLowerKVCrud(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestMossWithLowerReaderIsolation(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestMossWithLowerReaderOwnsGetBytes(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestMossWithLowerWriterOwnsBytes(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestMossWithLowerPrefixIterator(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestMossWithLowerPrefixIteratorSeek(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestMossWithLowerRangeIterator(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestMossWithLowerRangeIteratorSeek(t *testing.T) {
+	tmpDir, s := openWithLower(t, nil)
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestMossWithLowerMerge(t *testing.T) {
+	tmpDir, s := openWithLower(t, &test.TestMergeCounter{})
+	defer cleanupWithLower(t, s, tmpDir)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/moss/reader.go

@@ -0,0 +1,97 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/couchbase/moss"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type Reader struct {
+	store *Store
+	ss    moss.Snapshot
+}
+
+func (r *Reader) Get(k []byte) (v []byte, err error) {
+	v, err = r.ss.Get(k, moss.ReadOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if v != nil {
+		return append(make([]byte, 0, len(v)), v...), nil
+	}
+	return nil, nil
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
+	kEnd := incrementBytes(k)
+
+	iter, err := r.ss.StartIterator(k, kEnd, moss.IteratorOptions{})
+	if err != nil {
+		return nil
+	}
+
+	rv := &Iterator{
+		store: r.store,
+		ss:    r.ss,
+		iter:  iter,
+		start: k,
+		end:   kEnd,
+	}
+
+	rv.current()
+
+	return rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	iter, err := r.ss.StartIterator(start, end, moss.IteratorOptions{})
+	if err != nil {
+		return nil
+	}
+
+	rv := &Iterator{
+		store: r.store,
+		ss:    r.ss,
+		iter:  iter,
+		start: start,
+		end:   end,
+	}
+
+	rv.current()
+
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.ss.Close()
+}
+
+func incrementBytes(in []byte) []byte {
+	rv := make([]byte, len(in))
+	copy(rv, in)
+	for i := len(rv) - 1; i >= 0; i-- {
+		rv[i] = rv[i] + 1
+		if rv[i] != 0 {
+			return rv // didn't overflow, so stop
+		}
+	}
+	return nil // overflowed
+}

index/upsidedown/store/moss/stats.go

@@ -0,0 +1,58 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+type stats struct {
+	s *Store
+}
+
+func (s *stats) statsMap() map[string]interface{} {
+	ms := map[string]interface{}{}
+
+	var err error
+	ms["moss"], err = s.s.ms.Stats()
+	if err != nil {
+		return ms
+	}
+
+	if s.s.llstore != nil {
+		if o, ok := s.s.llstore.(store.KVStoreStats); ok {
+			ms["kv"] = o.StatsMap()
+		}
+	}
+
+	_, exists := ms["kv"]
+	if !exists && s.s.llstats != nil {
+		ms["kv"] = s.s.llstats()
+	}
+
+	if msw, ok := s.s.llstore.(*mossStoreWrapper); ok {
+		ms["store_histograms"] = msw.histograms()
+	}
+
+	ms["coll_histograms"] = s.s.ms.Histograms().String()
+
+	return ms
+}
+
+func (s *stats) MarshalJSON() ([]byte, error) {
+	m := s.statsMap()
+	return util.MarshalJSON(m)
+}

index/upsidedown/store/moss/store.go

@@ -0,0 +1,231 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package moss provides a KVStore implementation based on the
+// github.com/couchbase/moss library.
+
+package moss
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/couchbase/moss"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/util"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+// RegistryCollectionOptions should be treated as read-only after
+// process init()'ialization.
+var RegistryCollectionOptions = map[string]moss.CollectionOptions{}
+
+const Name = "moss"
+
+type Store struct {
+	m       sync.Mutex
+	ms      moss.Collection
+	mo      store.MergeOperator
+	llstore store.KVStore // May be nil.
+	llstats statsFunc     // May be nil.
+
+	s      *stats
+	config map[string]interface{}
+}
+
+type statsFunc func() map[string]interface{}
+
+// New initializes a moss storage with values from the optional
+// config["mossCollectionOptions"] (a JSON moss.CollectionOptions).
+// Next, values from the RegistryCollectionOptions, named by the
+// optional config["mossCollectionOptionsName"], take precedence.
+// Finally, base case defaults are taken from
+// moss.DefaultCollectionOptions.
+func New(mo store.MergeOperator, config map[string]interface{}) (
+	store.KVStore, error) {
+	options := moss.DefaultCollectionOptions // Copy.
+
+	v, ok := config["mossCollectionOptionsName"]
+	if ok {
+		name, ok := v.(string)
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not parse config[mossCollectionOptionsName]: %v", v)
+		}
+
+		options, ok = RegistryCollectionOptions[name] // Copy.
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not find RegistryCollectionOptions, name: %s", name)
+		}
+	}
+
+	options.MergeOperator = mo
+	options.DeferredSort = true
+
+	v, ok = config["mossCollectionOptions"]
+	if ok {
+		b, err := util.MarshalJSON(v) // Convert from map[string]interface{}.
+		if err != nil {
+			return nil, fmt.Errorf("moss store,"+
+				" could not marshal config[mossCollectionOptions]: %v, err: %v", v, err)
+		}
+
+		err = util.UnmarshalJSON(b, &options)
+		if err != nil {
+			return nil, fmt.Errorf("moss store,"+
+				" could not unmarshal config[mossCollectionOptions]: %v, err: %v", v, err)
+		}
+	}
+
+	// --------------------------------------------------
+
+	if options.Log == nil || options.Debug <= 0 {
+		options.Log = func(format string, a ...interface{}) {}
+	}
+
+	// --------------------------------------------------
+
+	mossLowerLevelStoreName := ""
+	v, ok = config["mossLowerLevelStoreName"]
+	if ok {
+		mossLowerLevelStoreName, ok = v.(string)
+		if !ok {
+			return nil, fmt.Errorf("moss store,"+
+				" could not parse config[mossLowerLevelStoreName]: %v", v)
+		}
+	}
+
+	var llStore store.KVStore
+	var llStats statsFunc
+
+	if options.LowerLevelInit == nil &&
+		options.LowerLevelUpdate == nil &&
+		mossLowerLevelStoreName != "" {
+		mossLowerLevelStoreConfig := map[string]interface{}{}
+		v, ok := config["mossLowerLevelStoreConfig"]
+		if ok {
+			mossLowerLevelStoreConfig, ok = v.(map[string]interface{})
+			if !ok {
+				return nil, fmt.Errorf("moss store, initLowerLevelStore,"+
+					" could parse mossLowerLevelStoreConfig: %v", v)
+			}
+		}
+
+		mossLowerLevelMaxBatchSize := uint64(0)
+		v, ok = config["mossLowerLevelMaxBatchSize"]
+		if ok {
+			mossLowerLevelMaxBatchSizeF, ok := v.(float64)
+			if !ok {
+				return nil, fmt.Errorf("moss store,"+
+					" could not parse config[mossLowerLevelMaxBatchSize]: %v", v)
+			}
+
+			mossLowerLevelMaxBatchSize = uint64(mossLowerLevelMaxBatchSizeF)
+		}
+
+		lowerLevelInit, lowerLevelUpdate, lowerLevelStore, lowerLevelStats, err :=
+			initLowerLevelStore(config,
+				mossLowerLevelStoreName,
+				mossLowerLevelStoreConfig,
+				mossLowerLevelMaxBatchSize,
+				options)
+		if err != nil {
+			return nil, err
+		}
+
+		options.LowerLevelInit = lowerLevelInit
+		options.LowerLevelUpdate = lowerLevelUpdate
+
+		llStore = lowerLevelStore
+		llStats = lowerLevelStats
+	}
+
+	// --------------------------------------------------
+
+	ms, err := moss.NewCollection(options)
+	if err != nil {
+		return nil, err
+	}
+	err = ms.Start()
+	if err != nil {
+		return nil, err
+	}
+	rv := Store{
+		ms:      ms,
+		mo:      mo,
+		llstore: llStore,
+		llstats: llStats,
+		config:  config,
+	}
+	rv.s = &stats{s: &rv}
+	return &rv, nil
+}
+
+func (s *Store) Close() error {
+	if val, ok := s.config["mossAbortCloseEnabled"]; ok {
+		if v, ok := val.(bool); ok && v {
+			if msw, ok := s.llstore.(*mossStoreWrapper); ok {
+				if s := msw.Actual(); s != nil {
+					_ = s.CloseEx(moss.StoreCloseExOptions{Abort: true})
+				}
+			}
+		}
+	}
+	return s.ms.Close()
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	ss, err := s.ms.Snapshot()
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{ss: ss}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	return &Writer{s: s}, nil
+}
+
+func (s *Store) Logf(fmt string, args ...interface{}) {
+	options := s.ms.Options()
+	if options.Log != nil {
+		options.Log(fmt, args...)
+	}
+}
+
+func (s *Store) Stats() json.Marshaler {
+	return s.s
+}
+
+func (s *Store) StatsMap() map[string]interface{} {
+	return s.s.statsMap()
+}
+
+func (s *Store) LowerLevelStore() store.KVStore {
+	return s.llstore
+}
+
+func (s *Store) Collection() moss.Collection {
+	return s.ms
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/moss/store_test.go

@@ -0,0 +1,91 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+	"github.com/blevesearch/upsidedown_store_api/test"
+)
+
+func open(t *testing.T, mo store.MergeOperator) store.KVStore {
+	rv, err := New(mo, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return rv
+}
+
+func cleanup(t *testing.T, s store.KVStore) {
+	err := s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMossKVCrud(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestKVCrud(t, s)
+}
+
+func TestMossReaderIsolation(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderIsolation(t, s)
+}
+
+func TestMossReaderOwnsGetBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestReaderOwnsGetBytes(t, s)
+}
+
+func TestMossWriterOwnsBytes(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestWriterOwnsBytes(t, s)
+}
+
+func TestMossPrefixIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIterator(t, s)
+}
+
+func TestMossPrefixIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestPrefixIteratorSeek(t, s)
+}
+
+func TestMossRangeIterator(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIterator(t, s)
+}
+
+func TestMossRangeIteratorSeek(t *testing.T) {
+	s := open(t, nil)
+	defer cleanup(t, s)
+	test.CommonTestRangeIteratorSeek(t, s)
+}
+
+func TestMossMerge(t *testing.T) {
+	s := open(t, &test.TestMergeCounter{})
+	defer cleanup(t, s)
+	test.CommonTestMerge(t, s)
+}

index/upsidedown/store/moss/writer.go

@@ -0,0 +1,97 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package moss
+
+import (
+	"fmt"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+
+	"github.com/couchbase/moss"
+)
+
+type Writer struct {
+	s *Store
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	b, err := w.s.ms.NewBatch(0, 0)
+	if err != nil {
+		return nil
+	}
+
+	return &Batch{
+		store: w.s,
+		merge: store.NewEmulatedMerge(w.s.mo),
+		batch: b,
+	}
+}
+
+func (w *Writer) NewBatchEx(options store.KVBatchOptions) (
+	[]byte, store.KVBatch, error) {
+	numOps := options.NumSets + options.NumDeletes + options.NumMerges
+
+	b, err := w.s.ms.NewBatch(numOps, options.TotalBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	buf, err := b.Alloc(options.TotalBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return buf, &Batch{
+		store:   w.s,
+		merge:   store.NewEmulatedMerge(w.s.mo),
+		batch:   b,
+		buf:     buf,
+		bufUsed: 0,
+	}, nil
+}
+
+func (w *Writer) ExecuteBatch(b store.KVBatch) (err error) {
+	batch, ok := b.(*Batch)
+	if !ok {
+		return fmt.Errorf("wrong type of batch")
+	}
+
+	for kStr, mergeOps := range batch.merge.Merges {
+		for _, v := range mergeOps {
+			if batch.buf != nil {
+				kLen := len(kStr)
+				vLen := len(v)
+				kBuf := batch.buf[batch.bufUsed : batch.bufUsed+kLen]
+				vBuf := batch.buf[batch.bufUsed+kLen : batch.bufUsed+kLen+vLen]
+				copy(kBuf, kStr)
+				copy(vBuf, v)
+				batch.bufUsed += kLen + vLen
+				err = batch.batch.AllocMerge(kBuf, vBuf)
+			} else {
+				err = batch.batch.Merge([]byte(kStr), v)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return w.s.ms.ExecuteBatch(batch.batch, moss.WriteOptions{})
+}
+
+func (w *Writer) Close() error {
+	w.s = nil
+	return nil
+}

index/upsidedown/store/null/null.go

@@ -0,0 +1,121 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package null
+
+import (
+	"github.com/blevesearch/bleve/v2/registry"
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+const Name = "null"
+
+type Store struct{}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	return &Store{}, nil
+}
+
+func (i *Store) Close() error {
+	return nil
+}
+
+func (i *Store) Reader() (store.KVReader, error) {
+	return &reader{}, nil
+}
+
+func (i *Store) Writer() (store.KVWriter, error) {
+	return &writer{}, nil
+}
+
+type reader struct{}
+
+func (r *reader) Get(key []byte) ([]byte, error) {
+	return nil, nil
+}
+
+func (r *reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return make([][]byte, len(keys)), nil
+}
+
+func (r *reader) PrefixIterator(prefix []byte) store.KVIterator {
+	return &iterator{}
+}
+
+func (r *reader) RangeIterator(start, end []byte) store.KVIterator {
+	return &iterator{}
+}
+
+func (r *reader) Close() error {
+	return nil
+}
+
+type iterator struct{}
+
+func (i *iterator) SeekFirst()    {}
+func (i *iterator) Seek(k []byte) {}
+func (i *iterator) Next()         {}
+
+func (i *iterator) Current() ([]byte, []byte, bool) {
+	return nil, nil, false
+}
+
+func (i *iterator) Key() []byte {
+	return nil
+}
+
+func (i *iterator) Value() []byte {
+	return nil
+}
+
+func (i *iterator) Valid() bool {
+	return false
+}
+
+func (i *iterator) Close() error {
+	return nil
+}
+
+type batch struct{}
+
+func (i *batch) Set(key, val []byte)   {}
+func (i *batch) Delete(key []byte)     {}
+func (i *batch) Merge(key, val []byte) {}
+func (i *batch) Reset()                {}
+func (i *batch) Close() error          { return nil }
+
+type writer struct{}
+
+func (w *writer) NewBatch() store.KVBatch {
+	return &batch{}
+}
+
+func (w *writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
+	return make([]byte, options.TotalBytes), w.NewBatch(), nil
+}
+
+func (w *writer) ExecuteBatch(store.KVBatch) error {
+	return nil
+}
+
+func (w *writer) Close() error {
+	return nil
+}
+
+func init() {
+	err := registry.RegisterKVStore(Name, New)
+	if err != nil {
+		panic(err)
+	}
+}

index/upsidedown/store/null/null_test.go

@@ -0,0 +1,92 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package null
+
+import (
+	"testing"
+
+	store "github.com/blevesearch/upsidedown_store_api"
+)
+
+func TestStore(t *testing.T) {
+	s, err := New(nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	NullTestKVStore(t, s)
+}
+
+// NullTestKVStore has very different expectations
+// compared to CommonTestKVStore
+func NullTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = writer.ExecuteBatch(batch)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.RangeIterator([]byte("b"), nil)
+	key, val, valid := it.Current()
+	if valid {
+		t.Fatalf("valid true, expected false")
+	}
+	if key != nil {
+		t.Fatalf("expected key nil, got %s", key)
+	}
+	if val != nil {
+		t.Fatalf("expected value nil, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

index/upsidedown/upsidedown.pb.go

@@ -0,0 +1,690 @@
+// Code generated by protoc-gen-gogo.
+// source: upsidedown.proto
+// DO NOT EDIT!
+
+/*
+Package upsidedown is a generated protocol buffer package.
+
+It is generated from these files:
+
+	upsidedown.proto
+
+It has these top-level messages:
+
+	BackIndexTermsEntry
+	BackIndexStoreEntry
+	BackIndexRowValue
+*/
+package upsidedown
+
+import proto "github.com/golang/protobuf/proto"
+import math "math"
+
+import io "io"
+import fmt "fmt"
+import github_com_golang_protobuf_proto "github.com/golang/protobuf/proto"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = math.Inf
+
+type BackIndexTermsEntry struct {
+	Field            *uint32  `protobuf:"varint,1,req,name=field" json:"field,omitempty"`
+	Terms            []string `protobuf:"bytes,2,rep,name=terms" json:"terms,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *BackIndexTermsEntry) Reset()         { *m = BackIndexTermsEntry{} }
+func (m *BackIndexTermsEntry) String() string { return proto.CompactTextString(m) }
+func (*BackIndexTermsEntry) ProtoMessage()    {}
+
+func (m *BackIndexTermsEntry) GetField() uint32 {
+	if m != nil && m.Field != nil {
+		return *m.Field
+	}
+	return 0
+}
+
+func (m *BackIndexTermsEntry) GetTerms() []string {
+	if m != nil {
+		return m.Terms
+	}
+	return nil
+}
+
+type BackIndexStoreEntry struct {
+	Field            *uint32  `protobuf:"varint,1,req,name=field" json:"field,omitempty"`
+	ArrayPositions   []uint64 `protobuf:"varint,2,rep,name=arrayPositions" json:"arrayPositions,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *BackIndexStoreEntry) Reset()         { *m = BackIndexStoreEntry{} }
+func (m *BackIndexStoreEntry) String() string { return proto.CompactTextString(m) }
+func (*BackIndexStoreEntry) ProtoMessage()    {}
+
+func (m *BackIndexStoreEntry) GetField() uint32 {
+	if m != nil && m.Field != nil {
+		return *m.Field
+	}
+	return 0
+}
+
+func (m *BackIndexStoreEntry) GetArrayPositions() []uint64 {
+	if m != nil {
+		return m.ArrayPositions
+	}
+	return nil
+}
+
+type BackIndexRowValue struct {
+	TermsEntries     []*BackIndexTermsEntry `protobuf:"bytes,1,rep,name=termsEntries" json:"termsEntries,omitempty"`
+	StoredEntries    []*BackIndexStoreEntry `protobuf:"bytes,2,rep,name=storedEntries" json:"storedEntries,omitempty"`
+	XXX_unrecognized []byte                 `json:"-"`
+}
+
+func (m *BackIndexRowValue) Reset()         { *m = BackIndexRowValue{} }
+func (m *BackIndexRowValue) String() string { return proto.CompactTextString(m) }
+func (*BackIndexRowValue) ProtoMessage()    {}
+
+func (m *BackIndexRowValue) GetTermsEntries() []*BackIndexTermsEntry {
+	if m != nil {
+		return m.TermsEntries
+	}
+	return nil
+}
+
+func (m *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry {
+	if m != nil {
+		return m.StoredEntries
+	}
+	return nil
+}
+
+func (m *BackIndexTermsEntry) Unmarshal(data []byte) error {
+	var hasFields [1]uint64
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		switch fieldNum {
+		case 1:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
+			}
+			var v uint32
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Field = &v
+			hasFields[0] |= uint64(0x00000001)
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Terms", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			postIndex := iNdEx + int(stringLen)
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Terms = append(m.Terms, string(data[iNdEx:postIndex]))
+			iNdEx = postIndex
+		default:
+			var sizeOfWire int
+			for {
+				sizeOfWire++
+				wire >>= 7
+				if wire == 0 {
+					break
+				}
+			}
+			iNdEx -= sizeOfWire
+			skippy, err := skipUpsidedown(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthUpsidedown
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+	if hasFields[0]&uint64(0x00000001) == 0 {
+		return new(github_com_golang_protobuf_proto.RequiredNotSetError)
+	}
+
+	return nil
+}
+func (m *BackIndexStoreEntry) Unmarshal(data []byte) error {
+	var hasFields [1]uint64
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		switch fieldNum {
+		case 1:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType)
+			}
+			var v uint32
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Field = &v
+			hasFields[0] |= uint64(0x00000001)
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ArrayPositions", wireType)
+			}
+			var v uint64
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.ArrayPositions = append(m.ArrayPositions, v)
+		default:
+			var sizeOfWire int
+			for {
+				sizeOfWire++
+				wire >>= 7
+				if wire == 0 {
+					break
+				}
+			}
+			iNdEx -= sizeOfWire
+			skippy, err := skipUpsidedown(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthUpsidedown
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+	if hasFields[0]&uint64(0x00000001) == 0 {
+		return new(github_com_golang_protobuf_proto.RequiredNotSetError)
+	}
+
+	return nil
+}
+func (m *BackIndexRowValue) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field TermsEntries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			postIndex := iNdEx + msglen
+			if msglen < 0 {
+				return ErrInvalidLengthUpsidedown
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{})
+			if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field StoredEntries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			postIndex := iNdEx + msglen
+			if msglen < 0 {
+				return ErrInvalidLengthUpsidedown
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{})
+			if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			var sizeOfWire int
+			for {
+				sizeOfWire++
+				wire >>= 7
+				if wire == 0 {
+					break
+				}
+			}
+			iNdEx -= sizeOfWire
+			skippy, err := skipUpsidedown(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthUpsidedown
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
+			iNdEx += skippy
+		}
+	}
+
+	return nil
+}
+func skipUpsidedown(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for {
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthUpsidedown
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipUpsidedown(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthUpsidedown = fmt.Errorf("proto: negative length found during unmarshaling")
+)
+
+func (m *BackIndexTermsEntry) Size() (n int) {
+	var l int
+	_ = l
+	if m.Field != nil {
+		n += 1 + sovUpsidedown(uint64(*m.Field))
+	}
+	if len(m.Terms) > 0 {
+		for _, s := range m.Terms {
+			l = len(s)
+			n += 1 + l + sovUpsidedown(uint64(l))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *BackIndexStoreEntry) Size() (n int) {
+	var l int
+	_ = l
+	if m.Field != nil {
+		n += 1 + sovUpsidedown(uint64(*m.Field))
+	}
+	if len(m.ArrayPositions) > 0 {
+		for _, e := range m.ArrayPositions {
+			n += 1 + sovUpsidedown(uint64(e))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func (m *BackIndexRowValue) Size() (n int) {
+	var l int
+	_ = l
+	if len(m.TermsEntries) > 0 {
+		for _, e := range m.TermsEntries {
+			l = e.Size()
+			n += 1 + l + sovUpsidedown(uint64(l))
+		}
+	}
+	if len(m.StoredEntries) > 0 {
+		for _, e := range m.StoredEntries {
+			l = e.Size()
+			n += 1 + l + sovUpsidedown(uint64(l))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		n += len(m.XXX_unrecognized)
+	}
+	return n
+}
+
+func sovUpsidedown(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozUpsidedown(x uint64) (n int) {
+	return sovUpsidedown(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *BackIndexTermsEntry) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *BackIndexTermsEntry) MarshalTo(data []byte) (n int, err error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if m.Field == nil {
+		return 0, new(github_com_golang_protobuf_proto.RequiredNotSetError)
+	} else {
+		data[i] = 0x8
+		i++
+		i = encodeVarintUpsidedown(data, i, uint64(*m.Field))
+	}
+	if len(m.Terms) > 0 {
+		for _, s := range m.Terms {
+			data[i] = 0x12
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				data[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			data[i] = uint8(l)
+			i++
+			i += copy(data[i:], s)
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		i += copy(data[i:], m.XXX_unrecognized)
+	}
+	return i, nil
+}
+
+func (m *BackIndexStoreEntry) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *BackIndexStoreEntry) MarshalTo(data []byte) (n int, err error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if m.Field == nil {
+		return 0, new(github_com_golang_protobuf_proto.RequiredNotSetError)
+	} else {
+		data[i] = 0x8
+		i++
+		i = encodeVarintUpsidedown(data, i, uint64(*m.Field))
+	}
+	if len(m.ArrayPositions) > 0 {
+		for _, num := range m.ArrayPositions {
+			data[i] = 0x10
+			i++
+			i = encodeVarintUpsidedown(data, i, uint64(num))
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		i += copy(data[i:], m.XXX_unrecognized)
+	}
+	return i, nil
+}
+
+func (m *BackIndexRowValue) Marshal() (data []byte, err error) {
+	size := m.Size()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *BackIndexRowValue) MarshalTo(data []byte) (n int, err error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.TermsEntries) > 0 {
+		for _, msg := range m.TermsEntries {
+			data[i] = 0xa
+			i++
+			i = encodeVarintUpsidedown(data, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	if len(m.StoredEntries) > 0 {
+		for _, msg := range m.StoredEntries {
+			data[i] = 0x12
+			i++
+			i = encodeVarintUpsidedown(data, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	if m.XXX_unrecognized != nil {
+		i += copy(data[i:], m.XXX_unrecognized)
+	}
+	return i, nil
+}
+
+func encodeFixed64Upsidedown(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Upsidedown(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintUpsidedown(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}

index/upsidedown/upsidedown.proto

@@ -0,0 +1,14 @@
+message BackIndexTermsEntry {
+  required uint32 field = 1;
+	repeated string terms = 2;
+}
+
+message BackIndexStoreEntry {
+	required uint32 field = 1;
+	repeated uint64 arrayPositions = 2;
+}
+
+message BackIndexRowValue {
+	repeated BackIndexTermsEntry termsEntries = 1;
+	repeated BackIndexStoreEntry storedEntries = 2;
+}