Adding upstream version 0.28.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

tools/picker/excerpt_modifier.go

@@ -0,0 +1,150 @@
+package picker
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+
+	"github.com/pocketbase/pocketbase/tools/list"
+	"github.com/spf13/cast"
+	"golang.org/x/net/html"
+)
+
+func init() {
+	Modifiers["excerpt"] = func(args ...string) (Modifier, error) {
+		return newExcerptModifier(args...)
+	}
+}
+
+var whitespaceRegex = regexp.MustCompile(`\s+`)
+
+var excludeTags = []string{
+	"head", "style", "script", "iframe", "embed", "applet", "object",
+	"svg", "img", "picture", "dialog", "template", "button", "form",
+	"textarea", "input", "select", "option",
+}
+
+var inlineTags = []string{
+	"a", "abbr", "acronym", "b", "bdo", "big", "br", "button",
+	"cite", "code", "em", "i", "label", "q", "small", "span",
+	"strong", "strike", "sub", "sup", "time",
+}
+
+var _ Modifier = (*excerptModifier)(nil)
+
+type excerptModifier struct {
+	max          int  // approximate max excerpt length
+	withEllipsis bool // if enabled will add ellipsis when the plain text length > max
+}
+
+// newExcerptModifier validates the specified raw string arguments and
+// initializes a new excerptModifier.
+//
+// This method is usually invoked in initModifer().
+func newExcerptModifier(args ...string) (*excerptModifier, error) {
+	totalArgs := len(args)
+
+	if totalArgs == 0 {
+		return nil, errors.New("max argument is required - expected (max, withEllipsis?)")
+	}
+
+	if totalArgs > 2 {
+		return nil, errors.New("too many arguments - expected (max, withEllipsis?)")
+	}
+
+	max := cast.ToInt(args[0])
+	if max == 0 {
+		return nil, errors.New("max argument must be > 0")
+	}
+
+	var withEllipsis bool
+	if totalArgs > 1 {
+		withEllipsis = cast.ToBool(args[1])
+	}
+
+	return &excerptModifier{max, withEllipsis}, nil
+}
+
+// Modify implements the [Modifier.Modify] interface method.
+//
+// It returns a plain text excerpt/short-description from a formatted
+// html string (non-string values are kept untouched).
+func (m *excerptModifier) Modify(value any) (any, error) {
+	strValue, ok := value.(string)
+	if !ok {
+		// not a string -> return as it is without applying the modifier
+		// (we don't throw an error because the modifier could be applied for a missing expand field)
+		return value, nil
+	}
+
+	var builder strings.Builder
+
+	doc, err := html.Parse(strings.NewReader(strValue))
+	if err != nil {
+		return "", err
+	}
+
+	var hasPrevSpace bool
+
+	// for all node types and more details check
+	// https://pkg.go.dev/golang.org/x/net/html#Parse
+	var stripTags func(*html.Node)
+	stripTags = func(n *html.Node) {
+		switch n.Type {
+		case html.TextNode:
+			// collapse multiple spaces into one
+			txt := whitespaceRegex.ReplaceAllString(n.Data, " ")
+
+			if hasPrevSpace {
+				txt = strings.TrimLeft(txt, " ")
+			}
+
+			if txt != "" {
+				hasPrevSpace = strings.HasSuffix(txt, " ")
+
+				builder.WriteString(txt)
+			}
+		}
+
+		// excerpt max has been reached => no need to further iterate
+		// (+2 for the extra whitespace suffix/prefix that will be trimmed later)
+		if builder.Len() > m.max+2 {
+			return
+		}
+
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			if c.Type != html.ElementNode || !list.ExistInSlice(c.Data, excludeTags) {
+				isBlock := c.Type == html.ElementNode && !list.ExistInSlice(c.Data, inlineTags)
+
+				if isBlock && !hasPrevSpace {
+					builder.WriteString(" ")
+					hasPrevSpace = true
+				}
+
+				stripTags(c)
+
+				if isBlock && !hasPrevSpace {
+					builder.WriteString(" ")
+					hasPrevSpace = true
+				}
+			}
+		}
+	}
+	stripTags(doc)
+
+	result := strings.TrimSpace(builder.String())
+
+	if len(result) > m.max {
+		// note: casted to []rune to properly account for multi-byte chars
+		runes := []rune(result)
+		if len(runes) > m.max {
+			result = string(runes[:m.max])
+			result = strings.TrimSpace(result)
+			if m.withEllipsis {
+				result += "..."
+			}
+		}
+	}
+
+	return result, nil
+}

tools/picker/excerpt_modifier_test.go

@@ -0,0 +1,170 @@
+package picker
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/spf13/cast"
+)
+
+func TestNewExcerptModifier(t *testing.T) {
+	scenarios := []struct {
+		name        string
+		args        []string
+		expectError bool
+	}{
+		{
+			"no arguments",
+			nil,
+			true,
+		},
+		{
+			"too many arguments",
+			[]string{"12", "false", "something"},
+			true,
+		},
+		{
+			"non-numeric max argument",
+			[]string{"something"}, // should fallback to 0 which is not allowed
+			true,
+		},
+		{
+			"numeric max argument",
+			[]string{"12"},
+			false,
+		},
+		{
+			"non-bool withEllipsis argument",
+			[]string{"12", "something"}, // should fallback to false which is allowed
+			false,
+		},
+		{
+			"truthy withEllipsis argument",
+			[]string{"12", "t"},
+			false,
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			m, err := newExcerptModifier(s.args...)
+
+			hasErr := err != nil
+			if hasErr != s.expectError {
+				t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
+			}
+
+			if hasErr {
+				if m != nil {
+					t.Fatalf("Expected nil modifier, got %v", m)
+				}
+
+				return
+			}
+
+			var argMax int
+			if len(s.args) > 0 {
+				argMax = cast.ToInt(s.args[0])
+			}
+
+			var argWithEllipsis bool
+			if len(s.args) > 1 {
+				argWithEllipsis = cast.ToBool(s.args[1])
+			}
+
+			if m.max != argMax {
+				t.Fatalf("Expected max %d, got %d", argMax, m.max)
+			}
+
+			if m.withEllipsis != argWithEllipsis {
+				t.Fatalf("Expected withEllipsis %v, got %v", argWithEllipsis, m.withEllipsis)
+			}
+		})
+	}
+}
+
+func TestExcerptModifierModify(t *testing.T) {
+	html := ` <script>var a = 123;</script>   <p>Hello</p><div id="test_id">t   est<b>12
+	3</b><span>456</span></div><span>word <b>7</b> 89<span>!<b>?</b><b> a </b><b>b </b>c</span>#<h1>title</h1>`
+
+	plainText := "Hello t est12 3456 word 7 89!? a b c# title"
+
+	scenarios := []struct {
+		name     string
+		args     []string
+		value    string
+		expected string
+	}{
+		// without ellipsis
+		{
+			"only max < len(plainText)",
+			[]string{"2"},
+			html,
+			plainText[:2],
+		},
+		{
+			"only max = len(plainText)",
+			[]string{fmt.Sprint(len(plainText))},
+			html,
+			plainText,
+		},
+		{
+			"only max > len(plainText)",
+			[]string{fmt.Sprint(len(plainText) + 5)},
+			html,
+			plainText,
+		},
+
+		// with ellipsis
+		{
+			"with ellipsis and max < len(plainText)",
+			[]string{"2", "t"},
+			html,
+			plainText[:2] + "...",
+		},
+		{
+			"with ellipsis and max = len(plainText)",
+			[]string{fmt.Sprint(len(plainText)), "t"},
+			html,
+			plainText,
+		},
+		{
+			"with ellipsis and max > len(plainText)",
+			[]string{fmt.Sprint(len(plainText) + 5), "t"},
+			html,
+			plainText,
+		},
+
+		// multibyte chars
+		{
+			"mutibyte chars <= max",
+			[]string{"4", "t"},
+			"аб\nв ",
+			"аб в",
+		},
+		{
+			"mutibyte chars > max",
+			[]string{"3", "t"},
+			"аб\nв ",
+			"аб...",
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			m, err := newExcerptModifier(s.args...)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			raw, err := m.Modify(s.value)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if v := cast.ToString(raw); v != s.expected {
+				t.Fatalf("Expected %q, got %q", s.expected, v)
+			}
+		})
+	}
+}

tools/picker/modifiers.go

@@ -0,0 +1,41 @@
+package picker
+
+import (
+	"fmt"
+
+	"github.com/pocketbase/pocketbase/tools/tokenizer"
+)
+
+var Modifiers = map[string]ModifierFactoryFunc{}
+
+type ModifierFactoryFunc func(args ...string) (Modifier, error)
+
+type Modifier interface {
+	// Modify executes the modifier and returns a new modified value.
+	Modify(value any) (any, error)
+}
+
+func initModifer(rawModifier string) (Modifier, error) {
+	t := tokenizer.NewFromString(rawModifier)
+	t.Separators('(', ')', ',', ' ')
+	t.IgnoreParenthesis(true)
+
+	parts, err := t.ScanAll()
+	if err != nil {
+		return nil, err
+	}
+
+	if len(parts) == 0 {
+		return nil, fmt.Errorf("invalid or empty modifier expression %q", rawModifier)
+	}
+
+	name := parts[0]
+	args := parts[1:]
+
+	factory, ok := Modifiers[name]
+	if !ok {
+		return nil, fmt.Errorf("missing or invalid modifier %q", name)
+	}
+
+	return factory(args...)
+}

tools/picker/pick.go

@@ -0,0 +1,184 @@
+package picker
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/pocketbase/pocketbase/tools/search"
+	"github.com/pocketbase/pocketbase/tools/tokenizer"
+)
+
+// Pick converts data into a []any, map[string]any, etc. (using json marshal->unmarshal)
+// containing only the fields from the parsed rawFields expression.
+//
+// rawFields is a comma separated string of the fields to include.
+// Nested fields should be listed with dot-notation.
+// Fields value modifiers are also supported using the `:modifier(args)` format (see Modifiers).
+//
+// Example:
+//
+//	data := map[string]any{"a": 1, "b": 2, "c": map[string]any{"c1": 11, "c2": 22}}
+//	Pick(data, "a,c.c1") // map[string]any{"a": 1, "c": map[string]any{"c1": 11}}
+func Pick(data any, rawFields string) (any, error) {
+	parsedFields, err := parseFields(rawFields)
+	if err != nil {
+		return nil, err
+	}
+
+	// marshalize the provided data to ensure that the related json.Marshaler
+	// implementations are invoked, and then convert it back to a plain
+	// json value that we can further operate on.
+	//
+	// @todo research other approaches to avoid the double serialization
+	// ---
+	encoded, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+
+	var decoded any
+	if err := json.Unmarshal(encoded, &decoded); err != nil {
+		return nil, err
+	}
+	// ---
+
+	// special cases to preserve the same fields format when used with single item or search results data.
+	var isSearchResult bool
+	switch data.(type) {
+	case search.Result, *search.Result:
+		isSearchResult = true
+	}
+
+	if isSearchResult {
+		if decodedMap, ok := decoded.(map[string]any); ok {
+			pickParsedFields(decodedMap["items"], parsedFields)
+		}
+	} else {
+		pickParsedFields(decoded, parsedFields)
+	}
+
+	return decoded, nil
+}
+
+func parseFields(rawFields string) (map[string]Modifier, error) {
+	t := tokenizer.NewFromString(rawFields)
+
+	fields, err := t.ScanAll()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]Modifier, len(fields))
+
+	for _, f := range fields {
+		parts := strings.SplitN(strings.TrimSpace(f), ":", 2)
+
+		if len(parts) > 1 {
+			m, err := initModifer(parts[1])
+			if err != nil {
+				return nil, err
+			}
+			result[parts[0]] = m
+		} else {
+			result[parts[0]] = nil
+		}
+	}
+
+	return result, nil
+}
+
+func pickParsedFields(data any, fields map[string]Modifier) error {
+	switch v := data.(type) {
+	case map[string]any:
+		pickMapFields(v, fields)
+	case []map[string]any:
+		for _, item := range v {
+			if err := pickMapFields(item, fields); err != nil {
+				return err
+			}
+		}
+	case []any:
+		if len(v) == 0 {
+			return nil // nothing to pick
+		}
+
+		if _, ok := v[0].(map[string]any); !ok {
+			return nil // for now ignore non-map values
+		}
+
+		for _, item := range v {
+			if err := pickMapFields(item.(map[string]any), fields); err != nil {
+				return nil
+			}
+		}
+	}
+
+	return nil
+}
+
+func pickMapFields(data map[string]any, fields map[string]Modifier) error {
+	if len(fields) == 0 {
+		return nil // nothing to pick
+	}
+
+	if m, ok := fields["*"]; ok {
+		// append all missing root level data keys
+		for k := range data {
+			var exists bool
+
+			for f := range fields {
+				if strings.HasPrefix(f+".", k+".") {
+					exists = true
+					break
+				}
+			}
+
+			if !exists {
+				fields[k] = m
+			}
+		}
+	}
+
+DataLoop:
+	for k := range data {
+		matchingFields := make(map[string]Modifier, len(fields))
+		for f, m := range fields {
+			if strings.HasPrefix(f+".", k+".") {
+				matchingFields[f] = m
+				continue
+			}
+		}
+
+		if len(matchingFields) == 0 {
+			delete(data, k)
+			continue DataLoop
+		}
+
+		// remove the current key from the matching fields path
+		for f, m := range matchingFields {
+			remains := strings.TrimSuffix(strings.TrimPrefix(f+".", k+"."), ".")
+
+			// final key
+			if remains == "" {
+				if m != nil {
+					var err error
+					data[k], err = m.Modify(data[k])
+					if err != nil {
+						return err
+					}
+				}
+				continue DataLoop
+			}
+
+			// cleanup the old field key and continue with the rest of the field path
+			delete(matchingFields, f)
+			matchingFields[remains] = m
+		}
+
+		if err := pickParsedFields(data[k], matchingFields); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

tools/picker/pick_test.go

@@ -0,0 +1,276 @@
+package picker_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/pocketbase/pocketbase/tools/picker"
+	"github.com/pocketbase/pocketbase/tools/search"
+)
+
+func TestPickFields(t *testing.T) {
+	scenarios := []struct {
+		name        string
+		data        any
+		fields      string
+		expectError bool
+		result      string
+	}{
+		{
+			"empty fields",
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"",
+			false,
+			`{"a":1,"b":2,"c":"test"}`,
+		},
+		{
+			"missing fields",
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"missing",
+			false,
+			`{}`,
+		},
+		{
+			"non map data",
+			"test",
+			"a,b,test",
+			false,
+			`"test"`,
+		},
+		{
+			"non slice of map data",
+			[]any{"a", "b", "test"},
+			"a,test",
+			false,
+			`["a","b","test"]`,
+		},
+		{
+			"map with no matching field",
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"missing", // test individual fields trim
+			false,
+			`{}`,
+		},
+		{
+			"map with existing and missing fields",
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"a,  c  ,missing", // test individual fields trim
+			false,
+			`{"a":1,"c":"test"}`,
+		},
+		{
+			"slice of maps with existing and missing fields",
+			[]any{
+				map[string]any{"a": 11, "b": 11, "c": "test1"},
+				map[string]any{"a": 22, "b": 22, "c": "test2"},
+			},
+			"a,  c  ,missing", // test individual fields trim
+			false,
+			`[{"a":11,"c":"test1"},{"a":22,"c":"test2"}]`,
+		},
+		{
+			"nested fields with mixed map and any slices",
+			map[string]any{
+				"a": 1,
+				"b": 2,
+				"c": "test",
+				"anySlice": []any{
+					map[string]any{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+						"D": map[string]any{
+							"DA": 1,
+							"DB": 2,
+						},
+					},
+					map[string]any{
+						"A": "test",
+					},
+				},
+				"mapSlice": []map[string]any{
+					{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+						"D": []any{
+							map[string]any{"DA": 1},
+						},
+					},
+					{
+						"B": []any{"1", "2", 3},
+						"D": []any{
+							map[string]any{"DA": 2},
+							map[string]any{"DA": 3},
+							map[string]any{"DB": 4}, // will result to empty since there is no DA
+						},
+					},
+				},
+				"fullMap": []map[string]any{
+					{
+						"A": []int{1, 2, 3},
+						"B": []any{"1", "2", 3},
+						"C": "test",
+					},
+					{
+						"B": []any{"1", "2", 3},
+						"D": []any{
+							map[string]any{"DA": 2},
+							map[string]any{"DA": 3}, // will result to empty since there is no DA
+						},
+					},
+				},
+			},
+			"a, c, anySlice.A, mapSlice.C, mapSlice.D.DA, anySlice.D,fullMap",
+			false,
+			`{"a":1,"anySlice":[{"A":[1,2,3],"D":{"DA":1,"DB":2}},{"A":"test"}],"c":"test","fullMap":[{"A":[1,2,3],"B":["1","2",3],"C":"test"},{"B":["1","2",3],"D":[{"DA":2},{"DA":3}]}],"mapSlice":[{"C":"test","D":[{"DA":1}]},{"D":[{"DA":2},{"DA":3},{}]}]}`,
+		},
+		{
+			"SearchResult",
+			search.Result{
+				Page:       1,
+				PerPage:    10,
+				TotalItems: 20,
+				TotalPages: 30,
+				Items: []any{
+					map[string]any{"a": 11, "b": 11, "c": "test1"},
+					map[string]any{"a": 22, "b": 22, "c": "test2"},
+				},
+			},
+			"a,c,missing",
+			false,
+			`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
+		},
+		{
+			"*SearchResult",
+			&search.Result{
+				Page:       1,
+				PerPage:    10,
+				TotalItems: 20,
+				TotalPages: 30,
+				Items: []any{
+					map[string]any{"a": 11, "b": 11, "c": "test1"},
+					map[string]any{"a": 22, "b": 22, "c": "test2"},
+				},
+			},
+			"a,c",
+			false,
+			`{"items":[{"a":11,"c":"test1"},{"a":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
+		},
+		{
+			"root wildcard",
+			&search.Result{
+				Page:       1,
+				PerPage:    10,
+				TotalItems: 20,
+				TotalPages: 30,
+				Items: []any{
+					map[string]any{"a": 11, "b": 11, "c": "test1"},
+					map[string]any{"a": 22, "b": 22, "c": "test2"},
+				},
+			},
+			"*",
+			false,
+			`{"items":[{"a":11,"b":11,"c":"test1"},{"a":22,"b":22,"c":"test2"}],"page":1,"perPage":10,"totalItems":20,"totalPages":30}`,
+		},
+		{
+			"root wildcard with nested exception",
+			map[string]any{
+				"id":    "123",
+				"title": "lorem",
+				"rel": map[string]any{
+					"id":    "456",
+					"title": "rel_title",
+				},
+			},
+			"*,rel.id",
+			false,
+			`{"id":"123","rel":{"id":"456"},"title":"lorem"}`,
+		},
+		{
+			"sub wildcard",
+			map[string]any{
+				"id":    "123",
+				"title": "lorem",
+				"rel": map[string]any{
+					"id":    "456",
+					"title": "rel_title",
+					"sub": map[string]any{
+						"id":    "789",
+						"title": "sub_title",
+					},
+				},
+			},
+			"id,rel.*",
+			false,
+			`{"id":"123","rel":{"id":"456","sub":{"id":"789","title":"sub_title"},"title":"rel_title"}}`,
+		},
+		{
+			"sub wildcard with nested exception",
+			map[string]any{
+				"id":    "123",
+				"title": "lorem",
+				"rel": map[string]any{
+					"id":    "456",
+					"title": "rel_title",
+					"sub": map[string]any{
+						"id":    "789",
+						"title": "sub_title",
+					},
+				},
+			},
+			"id,rel.*,rel.sub.id",
+			false,
+			`{"id":"123","rel":{"id":"456","sub":{"id":"789"},"title":"rel_title"}}`,
+		},
+		{
+			"invalid excerpt modifier",
+			map[string]any{"a": 1, "b": 2, "c": "test"},
+			"*:excerpt",
+			true,
+			`{"a":1,"b":2,"c":"test"}`,
+		},
+		{
+			"valid excerpt modifier",
+			map[string]any{
+				"id":    "123",
+				"title": "lorem",
+				"rel": map[string]any{
+					"id":    "456",
+					"title": "<p>rel_title</p>",
+					"sub": map[string]any{
+						"id":    "789",
+						"title": "sub_title",
+					},
+				},
+			},
+			"*:excerpt(2),rel.title:excerpt(3, true)",
+			false,
+			`{"id":"12","rel":{"title":"rel..."},"title":"lo"}`,
+		},
+	}
+
+	for _, s := range scenarios {
+		t.Run(s.name, func(t *testing.T) {
+			result, err := picker.Pick(s.data, s.fields)
+
+			hasErr := err != nil
+			if hasErr != s.expectError {
+				t.Fatalf("Expected hasErr %v, got %v (%v)", s.expectError, hasErr, err)
+			}
+
+			if hasErr {
+				return
+			}
+
+			serialized, err := json.Marshal(result)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if v := string(serialized); v != s.result {
+				t.Fatalf("Expected body\n%s \ngot \n%s", s.result, v)
+			}
+		})
+	}
+}