Adding upstream version 2.5.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

numeric/bin.go

@@ -0,0 +1,43 @@
+package numeric
+
+var interleaveMagic = []uint64{
+	0x5555555555555555,
+	0x3333333333333333,
+	0x0F0F0F0F0F0F0F0F,
+	0x00FF00FF00FF00FF,
+	0x0000FFFF0000FFFF,
+	0x00000000FFFFFFFF,
+	0xAAAAAAAAAAAAAAAA,
+}
+
+var interleaveShift = []uint{1, 2, 4, 8, 16}
+
+// Interleave the first 32 bits of each uint64
+// apdated from org.apache.lucene.util.BitUtil
+// which was adapted from:
+// http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN
+func Interleave(v1, v2 uint64) uint64 {
+	v1 = (v1 | (v1 << interleaveShift[4])) & interleaveMagic[4]
+	v1 = (v1 | (v1 << interleaveShift[3])) & interleaveMagic[3]
+	v1 = (v1 | (v1 << interleaveShift[2])) & interleaveMagic[2]
+	v1 = (v1 | (v1 << interleaveShift[1])) & interleaveMagic[1]
+	v1 = (v1 | (v1 << interleaveShift[0])) & interleaveMagic[0]
+	v2 = (v2 | (v2 << interleaveShift[4])) & interleaveMagic[4]
+	v2 = (v2 | (v2 << interleaveShift[3])) & interleaveMagic[3]
+	v2 = (v2 | (v2 << interleaveShift[2])) & interleaveMagic[2]
+	v2 = (v2 | (v2 << interleaveShift[1])) & interleaveMagic[1]
+	v2 = (v2 | (v2 << interleaveShift[0])) & interleaveMagic[0]
+	return (v2 << 1) | v1
+}
+
+// Deinterleave the 32-bit value starting at position 0
+// to get the other 32-bit value, shift it by 1 first
+func Deinterleave(b uint64) uint64 {
+	b &= interleaveMagic[0]
+	b = (b ^ (b >> interleaveShift[0])) & interleaveMagic[1]
+	b = (b ^ (b >> interleaveShift[1])) & interleaveMagic[2]
+	b = (b ^ (b >> interleaveShift[2])) & interleaveMagic[3]
+	b = (b ^ (b >> interleaveShift[3])) & interleaveMagic[4]
+	b = (b ^ (b >> interleaveShift[4])) & interleaveMagic[5]
+	return b
+}

numeric/bin_test.go

@@ -0,0 +1,27 @@
+package numeric
+
+import "testing"
+
+func TestInterleaveDeinterleave(t *testing.T) {
+	tests := []struct {
+		v1 uint64
+		v2 uint64
+	}{
+		{0, 0},
+		{1, 1},
+		{27, 39},
+		{1<<32 - 1, 1<<32 - 1}, // largest that should still work
+	}
+
+	for _, test := range tests {
+		i := Interleave(test.v1, test.v2)
+		gotv1 := Deinterleave(i)
+		gotv2 := Deinterleave(i >> 1)
+		if gotv1 != test.v1 {
+			t.Errorf("expected v1: %d, got %d, interleaved was %x", test.v1, gotv1, i)
+		}
+		if gotv2 != test.v2 {
+			t.Errorf("expected v2: %d, got %d, interleaved was %x", test.v2, gotv2, i)
+		}
+	}
+}

numeric/float.go

@@ -0,0 +1,34 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package numeric
+
+import (
+	"math"
+)
+
+func Float64ToInt64(f float64) int64 {
+	fasint := int64(math.Float64bits(f))
+	if fasint < 0 {
+		fasint = fasint ^ 0x7fffffffffffffff
+	}
+	return fasint
+}
+
+func Int64ToFloat64(i int64) float64 {
+	if i < 0 {
+		i ^= 0x7fffffffffffffff
+	}
+	return math.Float64frombits(uint64(i))
+}

numeric/float_test.go

@@ -0,0 +1,64 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package numeric
+
+import (
+	"testing"
+)
+
+// test that the float/sortable int operations work both ways
+// and that the corresponding integers sort the same as
+// the original floats would have
+func TestSortabledFloat64ToInt64(t *testing.T) {
+	tests := []struct {
+		input float64
+	}{
+		{
+			input: -4640094584139352638,
+		},
+		{
+			input: -167.42,
+		},
+		{
+			input: -1.11,
+		},
+		{
+			input: 0,
+		},
+		{
+			input: 3.14,
+		},
+		{
+			input: 167.42,
+		},
+	}
+
+	var lastInt64 *int64
+	for _, test := range tests {
+		actual := Float64ToInt64(test.input)
+		if lastInt64 != nil {
+			// check that this float is greater than the last one
+			if actual <= *lastInt64 {
+				t.Errorf("expected greater than prev, this: %d, last %d", actual, *lastInt64)
+			}
+		}
+		lastInt64 = &actual
+		convertedBack := Int64ToFloat64(actual)
+		// assert that we got back what we started with
+		if convertedBack != test.input {
+			t.Errorf("expected %f, got %f", test.input, convertedBack)
+		}
+	}
+}

numeric/prefix_coded.go

@@ -0,0 +1,111 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package numeric
+
+import "fmt"
+
+const ShiftStartInt64 byte = 0x20
+
+// PrefixCoded is a byte array encoding of
+// 64-bit numeric values shifted by 0-63 bits
+type PrefixCoded []byte
+
+func NewPrefixCodedInt64(in int64, shift uint) (PrefixCoded, error) {
+	rv, _, err := NewPrefixCodedInt64Prealloc(in, shift, nil)
+	return rv, err
+}
+
+func NewPrefixCodedInt64Prealloc(in int64, shift uint, prealloc []byte) (
+	rv PrefixCoded, preallocRest []byte, err error) {
+	if shift > 63 {
+		return nil, prealloc, fmt.Errorf("cannot shift %d, must be between 0 and 63", shift)
+	}
+
+	nChars := ((63 - shift) / 7) + 1
+
+	size := int(nChars + 1)
+	if len(prealloc) >= size {
+		rv = PrefixCoded(prealloc[0:size])
+		preallocRest = prealloc[size:]
+	} else {
+		rv = make(PrefixCoded, size)
+	}
+
+	rv[0] = ShiftStartInt64 + byte(shift)
+
+	sortableBits := int64(uint64(in) ^ 0x8000000000000000)
+	sortableBits = int64(uint64(sortableBits) >> shift)
+	for nChars > 0 {
+		// Store 7 bits per byte for compatibility
+		// with UTF-8 encoding of terms
+		rv[nChars] = byte(sortableBits & 0x7f)
+		nChars--
+		sortableBits = int64(uint64(sortableBits) >> 7)
+	}
+
+	return rv, preallocRest, nil
+}
+
+func MustNewPrefixCodedInt64(in int64, shift uint) PrefixCoded {
+	rv, err := NewPrefixCodedInt64(in, shift)
+	if err != nil {
+		panic(err)
+	}
+	return rv
+}
+
+// Shift returns the number of bits shifted
+// returns 0 if in uninitialized state
+func (p PrefixCoded) Shift() (uint, error) {
+	if len(p) > 0 {
+		shift := p[0] - ShiftStartInt64
+		if shift < 0 || shift < 63 {
+			return uint(shift), nil
+		}
+	}
+	return 0, fmt.Errorf("invalid prefix coded value")
+}
+
+func (p PrefixCoded) Int64() (int64, error) {
+	shift, err := p.Shift()
+	if err != nil {
+		return 0, err
+	}
+	var sortableBits int64
+	for _, inbyte := range p[1:] {
+		sortableBits <<= 7
+		sortableBits |= int64(inbyte)
+	}
+	return int64(uint64((sortableBits << shift)) ^ 0x8000000000000000), nil
+}
+
+func ValidPrefixCodedTerm(p string) (bool, int) {
+	return ValidPrefixCodedTermBytes([]byte(p))
+}
+
+func ValidPrefixCodedTermBytes(p []byte) (bool, int) {
+	if len(p) > 0 {
+		if p[0] < ShiftStartInt64 || p[0] > ShiftStartInt64+63 {
+			return false, 0
+		}
+		shift := p[0] - ShiftStartInt64
+		nChars := ((63 - int(shift)) / 7) + 1
+		if len(p) != nChars+1 {
+			return false, 0
+		}
+		return true, int(shift)
+	}
+	return false, 0
+}

numeric/prefix_coded_test.go

@@ -0,0 +1,158 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package numeric
+
+import (
+	"reflect"
+	"testing"
+)
+
+var tests = []struct {
+	input  int64
+	shift  uint
+	output PrefixCoded
+}{
+	{
+		input:  1,
+		shift:  0,
+		output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
+	},
+	{
+		input:  -1,
+		shift:  0,
+		output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f},
+	},
+	{
+		input:  -94582,
+		shift:  0,
+		output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa},
+	},
+	{
+		input:  314729851,
+		shift:  0,
+		output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b},
+	},
+	{
+		input:  314729851,
+		shift:  4,
+		output: PrefixCoded{0x24, 0x8, 0x0, 0x0, 0x0, 0x0, 0x9, 0x30, 0x4c, 0x57},
+	},
+	{
+		input:  314729851,
+		shift:  8,
+		output: PrefixCoded{0x28, 0x40, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x4, 0x65},
+	},
+	{
+		input:  314729851,
+		shift:  16,
+		output: PrefixCoded{0x30, 0x20, 0x0, 0x0, 0x0, 0x0, 0x25, 0x42},
+	},
+	{
+		input:  314729851,
+		shift:  32,
+		output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0},
+	},
+	{
+		input:  1234729851,
+		shift:  32,
+		output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0},
+	},
+}
+
+// these array encoding values have been verified manually
+// against the lucene implementation
+func TestPrefixCoded(t *testing.T) {
+
+	for _, test := range tests {
+		actual, err := NewPrefixCodedInt64(test.input, test.shift)
+		if err != nil {
+			t.Error(err)
+		}
+		if !reflect.DeepEqual(actual, test.output) {
+			t.Errorf("expected %#v, got %#v", test.output, actual)
+		}
+		checkedShift, err := actual.Shift()
+		if err != nil {
+			t.Error(err)
+		}
+		if checkedShift != test.shift {
+			t.Errorf("expected %d, got %d", test.shift, checkedShift)
+		}
+		// if the shift was 0, make sure we can go back to the original
+		if test.shift == 0 {
+			backToLong, err := actual.Int64()
+			if err != nil {
+				t.Error(err)
+			}
+			if backToLong != test.input {
+				t.Errorf("expected %v, got %v", test.input, backToLong)
+			}
+		}
+	}
+}
+
+func TestPrefixCodedValid(t *testing.T) {
+	// all of the shared tests should be valid
+	for _, test := range tests {
+		valid, _ := ValidPrefixCodedTerm(string(test.output))
+		if !valid {
+			t.Errorf("expected %s to be valid prefix coded, is not", string(test.output))
+		}
+	}
+
+	invalidTests := []struct {
+		data PrefixCoded
+	}{
+		// first byte invalid skip (too low)
+		{
+			data: PrefixCoded{0x19, 'c', 'a', 't'},
+		},
+		// first byte invalid skip (too high)
+		{
+			data: PrefixCoded{0x20 + 64, 'c'},
+		},
+		// length of trailing bytes wrong (too long)
+		{
+			data: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
+		},
+		// length of trailing bytes wrong (too short)
+		{
+			data: PrefixCoded{0x20 + 63},
+		},
+	}
+
+	// all of the shared tests should be valid
+	for _, test := range invalidTests {
+		valid, _ := ValidPrefixCodedTerm(string(test.data))
+		if valid {
+			t.Errorf("expected %s to be invalid prefix coded, it is", string(test.data))
+		}
+	}
+}
+
+func BenchmarkTestPrefixCoded(b *testing.B) {
+
+	for i := 0; i < b.N; i++ {
+		for _, test := range tests {
+			actual, err := NewPrefixCodedInt64(test.input, test.shift)
+			if err != nil {
+				b.Error(err)
+			}
+			if !reflect.DeepEqual(actual, test.output) {
+				b.Errorf("expected %#v, got %#v", test.output, actual)
+			}
+		}
+	}
+}