feat: add new Equivalent option

This option allows to skip the generation of operations for unequal
arrays of similar length and content.

Author: wI2L

README.md

@@ -280,6 +280,28 @@ Finally, as a side example, if we were to use the `Rationalize()` option in the
 ]
 ```
 
+#### Equivalence
+
+Some data types, such as arrays, can be deeply unequal and equivalent at the same time.
+
+Take the following JSON documents:
+```json
+[
+    "a", "b", "c", "d"
+]
+```
+```json
+[
+    "d", "c", "b", "a"
+]
+```
+
+The root arrays of each document are not equal because the values differ at each index. However, they are equivalent in terms of content:
+- they have the same length
+- the elements of the first can be found in the second, the same number of times for each
+
+For such situations, you can use the `Equivalent()` option to instruct the diff generator to skip the generation of operations that would otherwise be added to the patch to represent the differences between the two arrays.
+
 ## Benchmarks
 
 Performance is not the primary target of the package, instead it strives for correctness. A simple benchmark that compare the performance of available options is provided to give a rough estimate of the cost of each option. You can find the JSON documents used by this benchmark in the directory [testdata/benchs](testdata/benchs).

bench_test.go

@@ -11,33 +11,41 @@ func BenchmarkCompare(b *testing.B) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	var before interface{}
-	err = json.Unmarshal(beforeBytes, &before)
+	afterBytesOrdered, err := ioutil.ReadFile("testdata/benchs/after-ordered.json")
 	if err != nil {
 		b.Fatal(err)
 	}
-	afterBytes, err := ioutil.ReadFile("testdata/benchs/after.json")
-	if err != nil {
-		b.Fatal(err)
-	}
-	var after interface{}
-	err = json.Unmarshal(afterBytes, &after)
+	afterBytesUnordered, err := ioutil.ReadFile("testdata/benchs/after-unordered.json")
 	if err != nil {
 		b.Fatal(err)
 	}
 	makeopts := func(opts ...Option) []Option { return opts }
 
 	for _, bb := range []struct {
-		name string
-		opts []Option
+		name       string
+		opts       []Option
+		afterBytes []byte
 	}{
-		{"default", nil},
-		{"invertible", makeopts(Invertible())},
-		{"factorize", makeopts(Factorize())},
-		{"rationalize", makeopts(Rationalize())},
-		{"factor+ratio", makeopts(Factorize(), Rationalize())},
-		{"all-options", makeopts(Factorize(), Rationalize(), Invertible())},
+		{"default-ordered", nil, afterBytesOrdered},
+		{"default-unordered", nil, afterBytesUnordered},
+		{"invertible", makeopts(Invertible()), afterBytesOrdered},
+		{"factorize", makeopts(Factorize()), afterBytesOrdered},
+		{"rationalize", makeopts(Rationalize()), afterBytesOrdered},
+		{"equivalent-ordered", makeopts(Equivalent()), afterBytesOrdered},
+		{"equivalent-unordered", makeopts(Equivalent()), afterBytesUnordered},
+		{"factor+ratio", makeopts(Factorize(), Rationalize()), afterBytesOrdered},
+		{"all-options-ordered", makeopts(Factorize(), Rationalize(), Invertible(), Equivalent()), afterBytesOrdered},
+		{"all-options-unordered", makeopts(Factorize(), Rationalize(), Invertible(), Equivalent()), afterBytesUnordered},
 	} {
+		var before, after interface{}
+		err = json.Unmarshal(beforeBytes, &before)
+		if err != nil {
+			b.Fatal(err)
+		}
+		err = json.Unmarshal(bb.afterBytes, &after)
+		if err != nil {
+			b.Fatal(err)
+		}
 		b.Run("Compare/"+bb.name, func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
 				patch, err := CompareOpts(before, after, bb.opts...)
@@ -49,7 +57,7 @@ func BenchmarkCompare(b *testing.B) {
 		})
 		b.Run("CompareJSON/"+bb.name, func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
-				patch, err := CompareJSONOpts(beforeBytes, afterBytes, bb.opts...)
+				patch, err := CompareJSONOpts(beforeBytes, bb.afterBytes, bb.opts...)
 				if err != nil {
 					b.Error(err)
 				}
@@ -59,7 +67,7 @@ func BenchmarkCompare(b *testing.B) {
 		b.Run("differ_diff/"+bb.name, func(b *testing.B) {
 			for i := 0; i < b.N; i++ {
 				d := differ{
-					targetBytes: afterBytes,
+					targetBytes: bb.afterBytes,
 				}
 				for _, opt := range bb.opts {
 					opt(&d)

compare.go

@@ -42,12 +42,18 @@ func CompareJSONOpts(source, target []byte, opts ...Option) (Patch, error) {
 
 // Factorize enables factorization of operations.
 func Factorize() Option {
-	return func(o *differ) { o.factorize = true }
+	return func(o *differ) { o.opts.factorize = true }
 }
 
 // Rationalize enables rationalization of operations.
 func Rationalize() Option {
-	return func(o *differ) { o.rationalize = true }
+	return func(o *differ) { o.opts.rationalize = true }
+}
+
+// Equivalent disables the generation of operations for
+// arrays of equal length and content that are not ordered.
+func Equivalent() Option {
+	return func(o *differ) { o.opts.equivalent = true }
 }
 
 // Invertible enables the generation of an invertible
@@ -58,7 +64,7 @@ func Rationalize() Option {
 // such, using this option disable the usage of copy
 // operation in favor of add operations.
 func Invertible() Option {
-	return func(o *differ) { o.invertible = true }
+	return func(o *differ) { o.opts.invertible = true }
 }
 
 func compare(d *differ, src, tgt interface{}) (Patch, error) {

differ.go

@@ -9,14 +9,19 @@ type differ struct {
 	patch       Patch
 	hasher      hasher
 	hashmap     map[uint64]jsonNode
+	targetBytes []byte
+	opts        options
+}
+
+type options struct {
 	factorize   bool
 	rationalize bool
 	invertible  bool
-	targetBytes []byte
+	equivalent  bool
 }
 
 func (d *differ) diff(src, tgt interface{}) {
-	if d.factorize {
+	if d.opts.factorize {
 		d.prepare(emptyPtr, src, tgt)
 	}
 	d.compare(emptyPtr, src, tgt)
@@ -60,7 +65,7 @@ func (d *differ) compare(ptr pointer, src, tgt interface{}) {
 		}
 	}
 	// Rationalize any new operations.
-	if d.rationalize && len(d.patch) > size {
+	if d.opts.rationalize && len(d.patch) > size {
 		d.rationalizeLastOps(ptr, src, tgt, size)
 	}
 }
@@ -108,7 +113,7 @@ func (d *differ) prepare(ptr pointer, src, tgt interface{}) {
 func (d *differ) rationalizeLastOps(ptr pointer, src, tgt interface{}, lastOpIdx int) {
 	newOps := make(Patch, 0, 2)
 
-	if d.invertible {
+	if d.opts.invertible {
 		newOps = newOps.append(OperationTest, emptyPtr, ptr, nil, src)
 	}
 	// replaceOp represents a single operation that
@@ -162,8 +167,8 @@ func (d *differ) compareObjects(ptr pointer, src, tgt map[string]interface{}) {
 
 // compareArrays generates the patch operations that
 // represents the differences between two JSON arrays.
-func (d *differ) compareArrays(ptr pointer, src, dst []interface{}) {
-	size := min(len(src), len(dst))
+func (d *differ) compareArrays(ptr pointer, src, tgt []interface{}) {
+	size := min(len(src), len(tgt))
 
 	// When the source array contains more elements
 	// than the target, entries are being removed
@@ -172,29 +177,58 @@ func (d *differ) compareArrays(ptr pointer, src, dst []interface{}) {
 	for i := size; i < len(src); i++ {
 		d.remove(ptr.appendIndex(size), src[i])
 	}
+	if d.opts.equivalent && d.unorderedDeepEqualSlice(src, tgt) {
+		goto next
+	}
 	// Compare the elements at each index present in
 	// both the source and destination arrays.
 	for i := 0; i < size; i++ {
-		d.compare(ptr.appendIndex(i), src[i], dst[i])
+		d.compare(ptr.appendIndex(i), src[i], tgt[i])
 	}
+next:
 	// When the target array contains more elements
 	// than the source, entries are appended to the
 	// destination.
-	for i := size; i < len(dst); i++ {
-		d.add(ptr.appendKey("-"), dst[i])
+	for i := size; i < len(tgt); i++ {
+		d.add(ptr.appendKey("-"), tgt[i])
 	}
 }
 
+func (d *differ) unorderedDeepEqualSlice(src, tgt []interface{}) bool {
+	if len(src) != len(tgt) {
+		return false
+	}
+	diff := make(map[uint64]int, len(src))
+
+	for _, v := range src {
+		k := d.hasher.digest(v)
+		diff[k]++
+	}
+	for _, v := range tgt {
+		k := d.hasher.digest(v)
+		// If the digest hash if not in the diff,
+		// return early.
+		if _, ok := diff[k]; !ok {
+			return false
+		}
+		diff[k] -= 1
+		if diff[k] == 0 {
+			delete(diff, k)
+		}
+	}
+	return len(diff) == 0
+}
+
 func (d *differ) add(ptr pointer, v interface{}) {
-	if !d.factorize {
+	if !d.opts.factorize {
 		d.patch = d.patch.append(OperationAdd, emptyPtr, ptr, nil, v)
 		return
 	}
 	idx := d.findRemoved(v)
 	if idx != -1 {
 		op := d.patch[idx]
 
-		// https://tools.ietf.org/html/rfc6902#section-4.4
+		// https://tools.ietf.org/html/rfc6902#section-4.4f
 		// The "from" location MUST NOT be a proper prefix
 		// of the "path" location; i.e., a location cannot
 		// be moved into one of its children.
@@ -205,29 +239,22 @@ func (d *differ) add(ptr pointer, v interface{}) {
 		return
 	}
 	uptr := d.findUnchanged(v)
-	if !uptr.isRoot() && !d.invertible {
+	if !uptr.isRoot() && !d.opts.invertible {
 		d.patch = d.patch.append(OperationCopy, uptr, ptr, nil, v)
 	} else {
 		d.patch = d.patch.append(OperationAdd, emptyPtr, ptr, nil, v)
 	}
 }
 
-// areComparable returns whether the interface values
-// i1 and i2 can be compared. The values are comparable
-// only if they are both non-nil and share the same kind.
-func areComparable(i1, i2 interface{}) bool {
-	return typeSwitchKind(i1) == typeSwitchKind(i2)
-}
-
 func (d *differ) replace(ptr pointer, src, tgt interface{}) {
-	if d.invertible {
+	if d.opts.invertible {
 		d.patch = d.patch.append(OperationTest, emptyPtr, ptr, nil, src)
 	}
 	d.patch = d.patch.append(OperationReplace, emptyPtr, ptr, src, tgt)
 }
 
 func (d *differ) remove(ptr pointer, v interface{}) {
-	if d.invertible {
+	if d.opts.invertible {
 		d.patch = d.patch.append(OperationTest, emptyPtr, ptr, nil, v)
 	}
 	d.patch = d.patch.append(OperationRemove, emptyPtr, ptr, v, nil)

differ_test.go

@@ -32,7 +32,8 @@ func TestOptions(t *testing.T) {
 		{"testdata/tests/options/invertible.json", makeopts(Invertible())},
 		{"testdata/tests/options/factorization.json", makeopts(Factorize())},
 		{"testdata/tests/options/rationalization.json", makeopts(Rationalize())},
-		{"testdata/tests/options/all.json", makeopts(Factorize(), Rationalize(), Invertible())},
+		{"testdata/tests/options/equivalence.json", makeopts(Equivalent())},
+		{"testdata/tests/options/all.json", makeopts(Factorize(), Rationalize(), Invertible(), Equivalent())},
 	} {
 		var (
 			ext  = filepath.Ext(tt.testfile)

equal.go

@@ -1,6 +1,16 @@
 package jsondiff
 
-import "reflect"
+import (
+	"fmt"
+	"reflect"
+)
+
+// areComparable returns whether the interface values
+// i1 and i2 can be compared. The values are comparable
+// only if they are both non-nil and share the same kind.
+func areComparable(i1, i2 interface{}) bool {
+	return typeSwitchKind(i1) == typeSwitchKind(i2)
+}
 
 // typeSwitchKind returns the reflect.Kind of
 // the interface i using a type switch statement.
@@ -19,7 +29,7 @@ func typeSwitchKind(i interface{}) reflect.Kind {
 	case map[string]interface{}:
 		return reflect.Map
 	default:
-		panic("unknown json type")
+		panic(fmt.Sprintf("invalid json type %T", i))
 	}
 }