Adding upstream version 2.5.1.
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
parent
c71cb8b61d
commit
982828099e
GPG key ID:
FBB4F0E80A80222F
783 changed files with 150650 additions and 0 deletions
130
analysis/datetime/sanitized/sanitized.go
Normal file
130
analysis/datetime/sanitized/sanitized.go
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package sanitized
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/analysis"
|
||||
"github.com/blevesearch/bleve/v2/registry"
|
||||
)
|
||||
|
||||
const Name = "sanitizedgo"
|
||||
|
||||
var validMagicNumbers = map[string]struct{}{
|
||||
"2006": {},
|
||||
"06": {}, // Year
|
||||
"01": {},
|
||||
"1": {},
|
||||
"_1": {},
|
||||
"January": {},
|
||||
"Jan": {}, // Month
|
||||
"02": {},
|
||||
"2": {},
|
||||
"_2": {},
|
||||
"__2": {},
|
||||
"002": {},
|
||||
"Monday": {},
|
||||
"Mon": {}, // Day
|
||||
"15": {},
|
||||
"3": {},
|
||||
"03": {}, // Hour
|
||||
"4": {},
|
||||
"04": {}, // Minute
|
||||
"5": {},
|
||||
"05": {}, // Second
|
||||
"0700": {},
|
||||
"070000": {},
|
||||
"07": {},
|
||||
"00": {},
|
||||
"": {},
|
||||
}
|
||||
|
||||
var layoutSplitRegex = regexp.MustCompile("[\\+\\-= :T,Z\\.<>;\\?!`~@#$%\\^&\\*|'\"\\(\\){}\\[\\]/\\\\]")
|
||||
|
||||
var layoutStripRegex = regexp.MustCompile(`PM|pm|\.9+|\.0+|MST`)
|
||||
|
||||
type DateTimeParser struct {
|
||||
layouts []string
|
||||
}
|
||||
|
||||
func New(layouts []string) *DateTimeParser {
|
||||
return &DateTimeParser{
|
||||
layouts: layouts,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *DateTimeParser) ParseDateTime(input string) (time.Time, string, error) {
|
||||
for _, layout := range p.layouts {
|
||||
rv, err := time.Parse(layout, input)
|
||||
if err == nil {
|
||||
return rv, layout, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, "", analysis.ErrInvalidDateTime
|
||||
}
|
||||
|
||||
// date time layouts must be a combination of constants specified in golang time package
|
||||
// https://pkg.go.dev/time#pkg-constants
|
||||
// this validation verifies that only these constants are used in the custom layout
|
||||
// for compatibility with the golang time package
|
||||
func validateLayout(layout string) bool {
|
||||
// first we strip out commonly used constants
|
||||
// such as "PM" which can be present in the layout
|
||||
// right after a time component, e.g. 03:04PM;
|
||||
// because regex split cannot separate "03:04PM" into
|
||||
// "03:04" and "PM". We also strip out ".9+" and ".0+"
|
||||
// which represent fractional seconds.
|
||||
layout = layoutStripRegex.ReplaceAllString(layout, "")
|
||||
// then we split the layout by non-constant characters
|
||||
// which is a regex and verify that each split is a valid magic number
|
||||
split := layoutSplitRegex.Split(layout, -1)
|
||||
for i := range split {
|
||||
_, found := validMagicNumbers[split[i]]
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) {
|
||||
layouts, ok := config["layouts"].([]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("must specify layouts")
|
||||
}
|
||||
var layoutStrs []string
|
||||
for _, layout := range layouts {
|
||||
layoutStr, ok := layout.(string)
|
||||
if ok {
|
||||
if !validateLayout(layoutStr) {
|
||||
return nil, fmt.Errorf("invalid datetime parser layout: %s,"+
|
||||
" please refer to https://pkg.go.dev/time#pkg-constants for supported"+
|
||||
" layouts", layoutStr)
|
||||
}
|
||||
layoutStrs = append(layoutStrs, layoutStr)
|
||||
}
|
||||
}
|
||||
return New(layoutStrs), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
err := registry.RegisterDateTimeParser(Name, DateTimeParserConstructor)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
109
analysis/datetime/sanitized/sanitized_test.go
Normal file
109
analysis/datetime/sanitized/sanitized_test.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package sanitized
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLayoutValidatorRegex(t *testing.T) {
|
||||
splitRegexTests := []struct {
|
||||
input string
|
||||
output []string
|
||||
}{
|
||||
{
|
||||
input: "2014-08-03",
|
||||
output: []string{"2014", "08", "03"},
|
||||
},
|
||||
{
|
||||
input: "2014-08-03T15:59:30",
|
||||
output: []string{"2014", "08", "03", "15", "59", "30"},
|
||||
},
|
||||
{
|
||||
input: "2014.08-03 15/59`30",
|
||||
output: []string{"2014", "08", "03", "15", "59", "30"},
|
||||
},
|
||||
{
|
||||
input: "2014/08/03T15:59:30Z08:00",
|
||||
output: []string{"2014", "08", "03", "15", "59", "30", "08", "00"},
|
||||
},
|
||||
{
|
||||
input: "2014\\08|03T15=59.30.999999999+08*00",
|
||||
output: []string{"2014", "08", "03", "15", "59", "30", "999999999", "08", "00"},
|
||||
},
|
||||
{
|
||||
input: "2006-01-02T15:04:05.999999999Z07:00",
|
||||
output: []string{"2006", "01", "02", "15", "04", "05", "999999999", "07", "00"},
|
||||
},
|
||||
{
|
||||
input: "A-B C:DTE,FZG.H<I>J;K?L!M`N~O@P#Q$R%S^U&V*W|X'Y\"A(B)C{D}E[F]G/H\\I+J=L",
|
||||
output: []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P",
|
||||
"Q", "R", "S", "U", "V", "W", "X", "Y", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "L"},
|
||||
},
|
||||
}
|
||||
regex := layoutSplitRegex
|
||||
for _, test := range splitRegexTests {
|
||||
t.Run(test.input, func(t *testing.T) {
|
||||
actualOutput := regex.Split(test.input, -1)
|
||||
if !reflect.DeepEqual(actualOutput, test.output) {
|
||||
t.Fatalf("expected output %v, got %v", test.output, actualOutput)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
stripRegexTests := []struct {
|
||||
input string
|
||||
output string
|
||||
}{
|
||||
{
|
||||
input: "3PM",
|
||||
output: "3",
|
||||
},
|
||||
{
|
||||
input: "3.0PM",
|
||||
output: "3",
|
||||
},
|
||||
{
|
||||
input: "3.9AM",
|
||||
output: "3AM",
|
||||
},
|
||||
{
|
||||
input: "3.999999999pm",
|
||||
output: "3",
|
||||
},
|
||||
{
|
||||
input: "2006-01-02T15:04:05.999999999Z07:00MST",
|
||||
output: "2006-01-02T15:04:05Z07:00",
|
||||
},
|
||||
{
|
||||
input: "Jan _2 15:04:05.0000000+07:00MST",
|
||||
output: "Jan _2 15:04:05+07:00",
|
||||
},
|
||||
{
|
||||
input: "15:04:05.99PM+07:00MST",
|
||||
output: "15:04:05+07:00",
|
||||
},
|
||||
}
|
||||
regex = layoutStripRegex
|
||||
for _, test := range stripRegexTests {
|
||||
t.Run(test.input, func(t *testing.T) {
|
||||
actualOutput := layoutStripRegex.ReplaceAllString(test.input, "")
|
||||
if !reflect.DeepEqual(actualOutput, test.output) {
|
||||
t.Fatalf("expected output %v, got %v", test.output, actualOutput)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue