diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..0442b43 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + labels: ["dependencies"] + schedule: + interval: "daily" + - package-ecosystem: "gomod" + directory: "/" + labels: ["dependencies"] + schedule: + interval: "daily" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..516849b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,21 @@ +name: test +on: + pull_request: + paths-ignore: + - '*.md' + push: + branches: + - master + paths-ignore: + - '*.md' + - '.github/*' +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/setup-go@v5 + with: + go-version: 1.23.3 + - uses: actions/checkout@v4 + - run: go test ./... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..175bc2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +# IDE +*.iml +.idea +.vscode + +# OS +.DS_Store + +# JS +node_modules + +# Go +/vendor diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..49a3d72 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 TwiN + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..978050f --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# deepmerge +![test](https://github.com/TwiN/deepmerge/workflows/test/badge.svg?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/TwiN/deepmerge)](https://goreportcard.com/report/github.com/TwiN/deepmerge) +[![Go version](https://img.shields.io/github/go-mod/go-version/TwiN/deepmerge.svg)](https://github.com/TwiN/deepmerge) +[![Go Reference](https://pkg.go.dev/badge/github.com/TwiN/deepmerge.svg)](https://pkg.go.dev/github.com/TwiN/deepmerge) + +Go library for deep merging YAML or JSON files. + + +## Usage + +### YAML +```go +package main + +import ( + "github.com/TwiN/deepmerge" +) + +func main() { + dst := ` +debug: true +client: + insecure: true +users: + - id: 1 + firstName: John + lastName: Doe + - id: 2 + firstName: Jane + lastName: Doe` + src := ` +client: + timeout: 5s +users: + - id: 3 + firstName: Bob + lastName: Smith` + output, err := deepmerge.YAML([]byte(dst), []byte(src)) + if err != nil { + panic(err) + } + println(string(output)) +} +``` + +Output: +```yaml +client: + insecure: true + timeout: 5s +debug: true +users: + - firstName: John + id: 1 + lastName: Doe + - firstName: Jane + id: 2 + lastName: Doe + - firstName: Bob + id: 3 + lastName: Smith +``` + +### JSON +```go +package main + +import ( + "github.com/TwiN/deepmerge" +) + +func main() { + dst := `{ + "debug": true, + "client": { + "insecure": true + }, + "users": [ + { + "id": 1, + "firstName": "John", + "lastName": "Doe" + }, + { + "id": 2, + "firstName": "Jane", + "lastName": "Doe" + } + ] +}` + src := `{ + "client": { + "timeout": "5s" + }, + "users": [ + { + "id": 3, + "firstName": "Bob", + "lastName": "Smith" + } + ] +}` + output, err := deepmerge.JSON([]byte(dst), []byte(src)) + if err != nil { + panic(err) + } + println(string(output)) +} +``` + +Output: +```json +{ + "client": { + "insecure": true, + "timeout": "5s" + }, + "debug": true, + "users": [ + { + "firstName": "John", + "id": 1, + "lastName": "Doe" + }, + { + "firstName": "Jane", + "id": 2, + "lastName": "Doe" + }, + { + "firstName": "Bob", + "id": 3, + "lastName": "Smith" + } + ] +} +``` \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..d84c5b6 --- /dev/null +++ b/config.go @@ -0,0 +1,10 @@ +package deepmerge + +type Config struct { + // PreventMultipleDefinitionsOfKeysWithPrimitiveValue causes the return of an error if dst and src define + // the same key and if said key has a value with a primitive type + // This does not apply to slices or maps. + // + // Defaults to true + PreventMultipleDefinitionsOfKeysWithPrimitiveValue bool +} diff --git a/deepmerge.go b/deepmerge.go new file mode 100644 index 0000000..11f8226 --- /dev/null +++ b/deepmerge.go @@ -0,0 +1,48 @@ +package deepmerge + +import ( + "errors" +) + +var ( + ErrKeyWithPrimitiveValueDefinedMoreThanOnce = errors.New("error due to parameter with value of primitive type: only maps and slices/arrays can be merged, which means you cannot have define the same key twice for parameters that are not maps or slices/arrays") +) + +func DeepMerge(dst, src map[string]interface{}, config Config) error { + for srcKey, srcValue := range src { + if srcValueAsMap, ok := srcValue.(map[string]interface{}); ok { // handle maps + if dstValue, ok := dst[srcKey]; ok { + if dstValueAsMap, ok := dstValue.(map[string]interface{}); ok { + err := DeepMerge(dstValueAsMap, srcValueAsMap, config) + if err != nil { + return err + } + continue + } + } else { + dst[srcKey] = make(map[string]interface{}) + } + err := DeepMerge(dst[srcKey].(map[string]interface{}), srcValueAsMap, config) + if err != nil { + return err + } + } else if srcValueAsSlice, ok := srcValue.([]interface{}); ok { // handle slices + if dstValue, ok := dst[srcKey]; ok { + if dstValueAsSlice, ok := dstValue.([]interface{}); ok { + // If both src and dst are slices, we'll copy the elements from that src slice over to the dst slice + dst[srcKey] = append(dstValueAsSlice, srcValueAsSlice...) + continue + } + } + dst[srcKey] = srcValueAsSlice + } else { // handle primitives + if config.PreventMultipleDefinitionsOfKeysWithPrimitiveValue { + if _, ok := dst[srcKey]; ok { + return ErrKeyWithPrimitiveValueDefinedMoreThanOnce + } + } + dst[srcKey] = srcValue + } + } + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e25506b --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/TwiN/deepmerge + +go 1.23.3 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/json.go b/json.go new file mode 100644 index 0000000..fb8abea --- /dev/null +++ b/json.go @@ -0,0 +1,31 @@ +package deepmerge + +import ( + "encoding/json" +) + +// JSON merges the contents of src into dst +func JSON(dst, src []byte, optionalConfig ...Config) ([]byte, error) { + var cfg Config + if len(optionalConfig) > 0 { + cfg = optionalConfig[0] + } else { + cfg = Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: true} + } + var dstMap, srcMap map[string]interface{} + err := json.Unmarshal(dst, &dstMap) + if err != nil { + return nil, err + } + err = json.Unmarshal(src, &srcMap) + if err != nil { + return nil, err + } + if dstMap == nil { + dstMap = make(map[string]interface{}) + } + if err = DeepMerge(dstMap, srcMap, cfg); err != nil { + return nil, err + } + return json.Marshal(dstMap) +} diff --git a/json_test.go b/json_test.go new file mode 100644 index 0000000..1082a24 --- /dev/null +++ b/json_test.go @@ -0,0 +1,285 @@ +package deepmerge_test + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/TwiN/deepmerge" +) + +func TestJSON(t *testing.T) { + scenarios := []struct { + name string + config deepmerge.Config + dst string + src string + expected string + expectedErr error + }{ + { + name: "invalid-dst", + dst: ``, + src: `{}`, + expected: `{}`, + expectedErr: errors.New("unexpected end of JSON input"), + }, + { + name: "invalid-src", + dst: `{}`, + src: ``, + expected: `{}`, + expectedErr: errors.New("unexpected end of JSON input"), + }, + { + name: "simple-endpoint-merge", + dst: `{ + "endpoints": [ + { + "name": "one", + "url": "https://example.com", + "client": { + "timeout": "5s" + }, + "conditions": [ + "[CONNECTED] == true", + "[STATUS] == 200" + ], + "alerts": [ + { + "type": "slack", + "failure-threshold": 5 + } + ] + }, + { + "name": "two", + "url": "https://example.org", + "conditions": [ + "len([BODY]) > 0" + ] + } + ] +}`, + src: `{ + "endpoints": [ + { + "name": "three", + "url": "https://twin.sh/health", + "conditions": [ + "[STATUS] == 200", + "[BODY].status == UP" + ] + } + ] +}`, + expected: `{ + "endpoints": [ + { + "name": "one", + "url": "https://example.com", + "client": { + "timeout": "5s" + }, + "conditions": [ + "[CONNECTED] == true", + "[STATUS] == 200" + ], + "alerts": [ + { + "type": "slack", + "failure-threshold": 5 + } + ] + }, + { + "name": "two", + "url": "https://example.org", + "conditions": [ + "len([BODY]) > 0" + ] + }, + { + "name": "three", + "url": "https://twin.sh/health", + "conditions": [ + "[STATUS] == 200", + "[BODY].status == UP" + ] + } + ] +}`, + }, + { + name: "deep-merge-with-map-slice-and-primitive", + dst: `{ + "metrics": true, + "alerting": { + "slack": { + "webhook-url": "https://hooks.slack.com/services/xxx/yyy/zzz", + "default-alert": { + "description": "health check failed", + "send-on-resolved": true, + "failure-threshold": 5, + "success-threshold": 5 + } + } + }, + "endpoints": [ + { + "name": "example", + "url": "https://example.org", + "interval": "5s" + } + ] +}`, + src: `{ + "debug": true, + "alerting": { + "discord": { + "webhook-url": "https://discord.com/api/webhooks/xxx/yyy" + } + }, + "endpoints": [ + { + "name": "frontend", + "url": "https://example.com" + } + ] +}`, + expected: `{ + "metrics": true, + "debug": true, + "alerting": { + "discord": { + "webhook-url": "https://discord.com/api/webhooks/xxx/yyy" + }, + "slack": { + "webhook-url": "https://hooks.slack.com/services/xxx/yyy/zzz", + "default-alert": { + "description": "health check failed", + "send-on-resolved": true, + "failure-threshold": 5, + "success-threshold": 5 + } + } + }, + "endpoints": [ + { + "interval": "5s", + "name": "example", + "url": "https://example.org" + }, + { + "name": "frontend", + "url": "https://example.com" + } + ] +}`, + }, + { // only maps and slices can be merged. If there are duplicate keys that have a primitive value, then that's an error. + name: "duplicate-key-with-primitive-value", + config: deepmerge.Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: true}, // NOTE: true is the default + dst: `{"metrics": true}`, + src: `{"metrics": false}`, + + expectedErr: deepmerge.ErrKeyWithPrimitiveValueDefinedMoreThanOnce, + }, + { + name: "duplicate-key-with-primitive-value-with-PreventMultipleDefinitionsOfKeysWithPrimitiveValue-set-to-false", + config: deepmerge.Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: false}, + dst: `{"metrics": true, "debug": true}`, + src: `{"metrics": false}`, + expected: `{"metrics": false, "debug": true}`, + }, + { + name: "readme-example", + dst: `{ + "debug": true, + "client": { + "insecure": true + }, + "users": [ + { + "id": 1, + "firstName": "John", + "lastName": "Doe" + }, + { + "id": 2, + "firstName": "Jane", + "lastName": "Doe" + } + ] +}`, + src: `{ + "client": { + "timeout": "5s" + }, + "users": [ + { + "id": 3, + "firstName": "Bob", + "lastName": "Smith" + } + ] +}`, + expected: `{ + "client": { + "insecure": true, + "timeout": "5s" + }, + "debug": true, + "users": [ + { + "firstName": "John", + "id": 1, + "lastName": "Doe" + }, + { + "firstName": "Jane", + "id": 2, + "lastName": "Doe" + }, + { + "firstName": "Bob", + "id": 3, + "lastName": "Smith" + } + ] +}`, + }, + } + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + output, err := deepmerge.JSON([]byte(scenario.dst), []byte(scenario.src), scenario.config) + if !errors.Is(err, scenario.expectedErr) && !(scenario.expectedErr != nil && err.Error() == scenario.expectedErr.Error()) { + t.Errorf("[%s] expected error %v, got %v", scenario.name, scenario.expectedErr, err) + } + // Just so we don't have to worry about the formatting, we'll unmarshal the output and marshal it again. + expectedAsMap, outputAsMap := make(map[string]interface{}), make(map[string]interface{}) + if len(output) > 0 { + if err := json.Unmarshal(output, &outputAsMap); err != nil { + t.Errorf("[%s] failed to unmarshal output: %v", scenario.name, err) + } + } + if len(scenario.expected) > 0 { + if err := json.Unmarshal([]byte(scenario.expected), &expectedAsMap); err != nil { + t.Errorf("[%s] failed to unmarshal expected: %v", scenario.name, err) + } + } + formattedOutput, err := json.Marshal(outputAsMap) + if err != nil { + t.Errorf("[%s] should've been able to re-marshal output: %v", scenario.name, err) + } + formattedExpected, err := json.Marshal(expectedAsMap) + if err != nil { + t.Errorf("[%s] should've been able to re-marshal expected: %v", scenario.name, err) + } + // Compare what we got vs what we expected + if string(formattedOutput) != string(formattedExpected) { + t.Errorf("[%s] expected:\n%s\n\ngot:\n%s", scenario.name, string(formattedExpected), string(formattedOutput)) + } + }) + } +} diff --git a/yaml.go b/yaml.go new file mode 100644 index 0000000..7575e1e --- /dev/null +++ b/yaml.go @@ -0,0 +1,31 @@ +package deepmerge + +import ( + "gopkg.in/yaml.v3" +) + +// YAML merges the contents of src into dst +func YAML(dst, src []byte, optionalConfig ...Config) ([]byte, error) { + var cfg Config + if len(optionalConfig) > 0 { + cfg = optionalConfig[0] + } else { + cfg = Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: true} + } + var dstMap, srcMap map[string]interface{} + err := yaml.Unmarshal(dst, &dstMap) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(src, &srcMap) + if err != nil { + return nil, err + } + if dstMap == nil { + dstMap = make(map[string]interface{}) + } + if err = DeepMerge(dstMap, srcMap, cfg); err != nil { + return nil, err + } + return yaml.Marshal(dstMap) +} diff --git a/yaml_test.go b/yaml_test.go new file mode 100644 index 0000000..d60b366 --- /dev/null +++ b/yaml_test.go @@ -0,0 +1,217 @@ +package deepmerge_test + +import ( + "errors" + "testing" + + "github.com/TwiN/deepmerge" + "gopkg.in/yaml.v3" +) + +func TestYAML(t *testing.T) { + scenarios := []struct { + name string + config deepmerge.Config + dst string + src string + expected string + expectedErr error + }{ + { + name: "invalid-dst", + dst: `wat`, + src: ``, + expected: ``, + expectedErr: errors.New("yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wat` into map[string]interface {}"), + }, + { + name: "invalid-src", + dst: ``, + src: `wat`, + expected: ``, + expectedErr: errors.New("yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `wat` into map[string]interface {}"), + }, + { + name: "simple-endpoint-merge", + dst: `endpoints: + - name: one + url: https://example.com + client: + timeout: 5s + conditions: + - "[CONNECTED] == true" + - "[STATUS] == 200" + alerts: + - type: slack + failure-threshold: 5 + + - name: two + url: https://example.org + conditions: + - "len([BODY]) > 0"`, + src: `endpoints: + - name: three + url: https://twin.sh/health + conditions: + - "[STATUS] == 200" + - "[BODY].status == UP"`, + expected: `endpoints: + - name: one + url: https://example.com + client: + timeout: 5s + conditions: + - "[CONNECTED] == true" + - "[STATUS] == 200" + alerts: + - type: slack + failure-threshold: 5 + + - name: two + url: https://example.org + conditions: + - "len([BODY]) > 0" + + - name: three + url: https://twin.sh/health + conditions: + - "[STATUS] == 200" + - "[BODY].status == UP" +`, + }, + { + name: "deep-merge-with-map-slice-and-primitive", + dst: ` +metrics: true + +alerting: + slack: + webhook-url: https://hooks.slack.com/services/xxx/yyy/zzz + default-alert: + description: "health check failed" + send-on-resolved: true + failure-threshold: 5 + success-threshold: 5 + +endpoints: + - name: example + url: https://example.org + interval: 5s`, + src: ` +debug: true + +alerting: + discord: + webhook-url: https://discord.com/api/webhooks/xxx/yyy + +endpoints: + - name: frontend + url: https://example.com`, + expected: ` +metrics: true +debug: true + +alerting: + discord: + webhook-url: https://discord.com/api/webhooks/xxx/yyy + slack: + webhook-url: https://hooks.slack.com/services/xxx/yyy/zzz + default-alert: + description: "health check failed" + send-on-resolved: true + failure-threshold: 5 + success-threshold: 5 + +endpoints: + - interval: 5s + name: example + url: https://example.org + + - name: frontend + url: https://example.com +`, + }, + { // only maps and slices can be merged. If there are duplicate keys that have a primitive value, then that's an error. + name: "duplicate-key-with-primitive-value", + config: deepmerge.Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: true}, // NOTE: true is the default + dst: `metrics: true`, + src: `metrics: false`, + expectedErr: deepmerge.ErrKeyWithPrimitiveValueDefinedMoreThanOnce, + }, + { + name: "duplicate-key-with-primitive-value-with-preventDuplicateKeysWithPrimitiveValue-set-to-false", + config: deepmerge.Config{PreventMultipleDefinitionsOfKeysWithPrimitiveValue: false}, + dst: `metrics: true +debug: true`, + src: `metrics: false`, + expected: `metrics: false +debug: true`, + }, + { + name: "readme-example", + dst: `debug: true +client: + insecure: true +users: + - id: 1 + firstName: John + lastName: Doe + - id: 2 + firstName: Jane + lastName: Doe`, + src: `client: + timeout: 5s +users: + - id: 3 + firstName: Bob + lastName: Smith`, + expected: ` +client: + insecure: true + timeout: 5s +debug: true +users: + - firstName: John + id: 1 + lastName: Doe + - firstName: Jane + id: 2 + lastName: Doe + - firstName: Bob + id: 3 + lastName: Smith`, + }, + } + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + output, err := deepmerge.YAML([]byte(scenario.dst), []byte(scenario.src), scenario.config) + if !errors.Is(err, scenario.expectedErr) && !(scenario.expectedErr != nil && err.Error() == scenario.expectedErr.Error()) { + t.Errorf("[%s] expected error %v, got %v", scenario.name, scenario.expectedErr, err) + } + // Just so we don't have to worry about the formatting, we'll unmarshal the output and marshal it again. + expectedAsMap, outputAsMap := make(map[string]interface{}), make(map[string]interface{}) + if len(output) > 0 { + if err := yaml.Unmarshal(output, &outputAsMap); err != nil { + t.Errorf("[%s] failed to unmarshal output: %v", scenario.name, err) + } + } + if len(scenario.expected) > 0 { + if err := yaml.Unmarshal([]byte(scenario.expected), &expectedAsMap); err != nil { + t.Errorf("[%s] failed to unmarshal expected: %v", scenario.name, err) + } + } + formattedOutput, err := yaml.Marshal(outputAsMap) + if err != nil { + t.Errorf("[%s] should've been able to re-marshal output: %v", scenario.name, err) + } + formattedExpected, err := yaml.Marshal(expectedAsMap) + if err != nil { + t.Errorf("[%s] should've been able to re-marshal expected: %v", scenario.name, err) + } + // Compare what we got vs what we expected + if string(formattedOutput) != string(formattedExpected) { + t.Errorf("[%s] expected:\n%s\n\ngot:\n%s", scenario.name, string(formattedExpected), string(formattedOutput)) + } + }) + } +}