Adding upstream version 2.5.1.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
c71cb8b61d
commit
982828099e
GPG key ID:
FBB4F0E80A80222F
783 changed files with 150650 additions and 0 deletions
156
search/highlight/fragmenter/simple/simple.go
Normal file
156
search/highlight/fragmenter/simple/simple.go
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package simple
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/registry"
|
||||
"github.com/blevesearch/bleve/v2/search/highlight"
|
||||
)
|
||||
|
||||
const Name = "simple"
|
||||
|
||||
const defaultFragmentSize = 200
|
||||
|
||||
type Fragmenter struct {
|
||||
fragmentSize int
|
||||
}
|
||||
|
||||
func NewFragmenter(fragmentSize int) *Fragmenter {
|
||||
return &Fragmenter{
|
||||
fragmentSize: fragmentSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment {
|
||||
var rv []*highlight.Fragment
|
||||
maxbegin := 0
|
||||
OUTER:
|
||||
for currTermIndex, termLocation := range ot {
|
||||
// start with this
|
||||
// it should be the highest scoring fragment with this term first
|
||||
start := termLocation.Start
|
||||
end := start
|
||||
used := 0
|
||||
for end < len(orig) && used < s.fragmentSize {
|
||||
r, size := utf8.DecodeRune(orig[end:])
|
||||
if r == utf8.RuneError {
|
||||
continue OUTER // bail
|
||||
}
|
||||
end += size
|
||||
used++
|
||||
}
|
||||
|
||||
// if we still have more characters available to us
|
||||
// push back towards beginning
|
||||
// without cross maxbegin
|
||||
for start > 0 && used < s.fragmentSize {
|
||||
if start > len(orig) {
|
||||
// bail if out of bounds, possibly due to token replacement
|
||||
// e.g with a regexp replacement
|
||||
continue OUTER
|
||||
}
|
||||
r, size := utf8.DecodeLastRune(orig[0:start])
|
||||
if r == utf8.RuneError {
|
||||
continue OUTER // bail
|
||||
}
|
||||
if start-size >= maxbegin {
|
||||
start -= size
|
||||
used++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// however, we'd rather have the tokens centered more in the frag
|
||||
// lets try to do that as best we can, without affecting the score
|
||||
// find the end of the last term in this fragment
|
||||
minend := end
|
||||
for _, innerTermLocation := range ot[currTermIndex:] {
|
||||
if innerTermLocation.End > end {
|
||||
break
|
||||
}
|
||||
minend = innerTermLocation.End
|
||||
}
|
||||
|
||||
// find the smaller of the two rooms to move
|
||||
roomToMove := utf8.RuneCount(orig[minend:end])
|
||||
roomToMoveStart := 0
|
||||
if start >= maxbegin {
|
||||
roomToMoveStart = utf8.RuneCount(orig[maxbegin:start])
|
||||
}
|
||||
if roomToMoveStart < roomToMove {
|
||||
roomToMove = roomToMoveStart
|
||||
}
|
||||
|
||||
offset := roomToMove / 2
|
||||
|
||||
for offset > 0 {
|
||||
r, size := utf8.DecodeLastRune(orig[0:start])
|
||||
if r == utf8.RuneError {
|
||||
continue OUTER // bail
|
||||
}
|
||||
start -= size
|
||||
|
||||
r, size = utf8.DecodeLastRune(orig[0:end])
|
||||
if r == utf8.RuneError {
|
||||
continue OUTER // bail
|
||||
}
|
||||
end -= size
|
||||
offset--
|
||||
}
|
||||
|
||||
rv = append(rv, &highlight.Fragment{Orig: orig, Start: start - offset, End: end - offset})
|
||||
// set maxbegin to the end of the current term location
|
||||
// so that next one won't back up to include it
|
||||
maxbegin = termLocation.End
|
||||
|
||||
}
|
||||
if len(ot) == 0 {
|
||||
// if there were no terms to highlight
|
||||
// produce a single fragment from the beginning
|
||||
start := 0
|
||||
end := start
|
||||
used := 0
|
||||
for end < len(orig) && used < s.fragmentSize {
|
||||
r, size := utf8.DecodeRune(orig[end:])
|
||||
if r == utf8.RuneError {
|
||||
break
|
||||
}
|
||||
end += size
|
||||
used++
|
||||
}
|
||||
rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end})
|
||||
}
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Fragmenter, error) {
|
||||
size := defaultFragmentSize
|
||||
sizeVal, ok := config["size"].(float64)
|
||||
if ok {
|
||||
size = int(sizeVal)
|
||||
}
|
||||
return NewFragmenter(size), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
err := registry.RegisterFragmenter(Name, Constructor)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
311
search/highlight/fragmenter/simple/simple_test.go
Normal file
311
search/highlight/fragmenter/simple/simple_test.go
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package simple
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search/highlight"
|
||||
)
|
||||
|
||||
func TestSimpleFragmenter(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
orig []byte
|
||||
fragments []*highlight.Fragment
|
||||
ot highlight.TermLocations
|
||||
size int
|
||||
}{
|
||||
{
|
||||
orig: []byte("this is a test"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("this is a test"),
|
||||
Start: 0,
|
||||
End: 14,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "test",
|
||||
Pos: 4,
|
||||
Start: 10,
|
||||
End: 14,
|
||||
},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
{
|
||||
orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"),
|
||||
Start: 0,
|
||||
End: 100,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
|
||||
Pos: 1,
|
||||
Start: 0,
|
||||
End: 100,
|
||||
},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
{
|
||||
orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 0,
|
||||
End: 100,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 10,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 20,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 30,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 40,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 50,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 60,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 70,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 80,
|
||||
End: 101,
|
||||
},
|
||||
{
|
||||
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
|
||||
Start: 90,
|
||||
End: 101,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 1,
|
||||
Start: 0,
|
||||
End: 10,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 2,
|
||||
Start: 10,
|
||||
End: 20,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 3,
|
||||
Start: 20,
|
||||
End: 30,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 4,
|
||||
Start: 30,
|
||||
End: 40,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 5,
|
||||
Start: 40,
|
||||
End: 50,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 6,
|
||||
Start: 50,
|
||||
End: 60,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 7,
|
||||
Start: 60,
|
||||
End: 70,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 8,
|
||||
Start: 70,
|
||||
End: 80,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 9,
|
||||
Start: 80,
|
||||
End: 90,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "0123456789",
|
||||
Pos: 10,
|
||||
Start: 90,
|
||||
End: 100,
|
||||
},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
{
|
||||
orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"),
|
||||
Start: 0,
|
||||
End: 411,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "पानी",
|
||||
Pos: 1,
|
||||
Start: 2,
|
||||
End: 14,
|
||||
},
|
||||
},
|
||||
size: 200,
|
||||
},
|
||||
{
|
||||
orig: []byte("交换机"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("交换机"),
|
||||
Start: 0,
|
||||
End: 9,
|
||||
},
|
||||
{
|
||||
Orig: []byte("交换机"),
|
||||
Start: 3,
|
||||
End: 9,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "交换",
|
||||
Pos: 1,
|
||||
Start: 0,
|
||||
End: 6,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "换机",
|
||||
Pos: 2,
|
||||
Start: 3,
|
||||
End: 9,
|
||||
},
|
||||
},
|
||||
size: 200,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
fragmenter := NewFragmenter(test.size)
|
||||
fragments := fragmenter.Fragment(test.orig, test.ot)
|
||||
if !reflect.DeepEqual(fragments, test.fragments) {
|
||||
t.Errorf("expected %#v, got %#v", test.fragments, fragments)
|
||||
for _, fragment := range fragments {
|
||||
t.Logf("frag: %s", fragment.Orig[fragment.Start:fragment.End])
|
||||
t.Logf("frag: %d - %d", fragment.Start, fragment.End)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimpleFragmenterWithSize(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
orig []byte
|
||||
fragments []*highlight.Fragment
|
||||
ot highlight.TermLocations
|
||||
}{
|
||||
{
|
||||
orig: []byte("this is a test"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("this is a test"),
|
||||
Start: 0,
|
||||
End: 5,
|
||||
},
|
||||
{
|
||||
Orig: []byte("this is a test"),
|
||||
Start: 9,
|
||||
End: 14,
|
||||
},
|
||||
},
|
||||
ot: highlight.TermLocations{
|
||||
&highlight.TermLocation{
|
||||
Term: "this",
|
||||
Pos: 1,
|
||||
Start: 0,
|
||||
End: 5,
|
||||
},
|
||||
&highlight.TermLocation{
|
||||
Term: "test",
|
||||
Pos: 4,
|
||||
Start: 10,
|
||||
End: 14,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
orig: []byte("避免出现 rune 越界问题"),
|
||||
fragments: []*highlight.Fragment{
|
||||
{
|
||||
Orig: []byte("避免出现 rune 越界问题"),
|
||||
Start: 0,
|
||||
End: 13,
|
||||
},
|
||||
},
|
||||
ot: nil,
|
||||
},
|
||||
}
|
||||
|
||||
fragmenter := NewFragmenter(5)
|
||||
for _, test := range tests {
|
||||
fragments := fragmenter.Fragment(test.orig, test.ot)
|
||||
if !reflect.DeepEqual(fragments, test.fragments) {
|
||||
t.Errorf("expected %#v, got %#v", test.fragments, fragments)
|
||||
for _, fragment := range fragments {
|
||||
t.Logf("frag: %#v", fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue