Extended ZSET benchmarks with ZRANGESTORE and extra ZADD benchmarks (#280)

* Extended ZSET benchmarks with ZRANGESTORE and extra ZADD benchmarks

* Fixed failing memtier args test (#281)

* Fixed wrong test arg order on redis_benchmarks_specification (#282)

* Fixed failing memtier args test

* Fixed wrong test arg order on redis_benchmarks_specification/__self_contained_coordinator__/clients.py

* Fix test issues: order of arguments in prepare_memtier_benchmark_parameters and escape sequence warning

* Fix duplicate --json-out-file parameter in prepare_memtier_benchmark_parameters

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.260"
+version = "0.1.263"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__self_contained_coordinator__/clients.py

@@ -8,12 +8,12 @@ def prepare_memtier_benchmark_parameters(
 ):
     benchmark_command = [
         full_benchmark_path,
+        "--json-out-file",
+        local_benchmark_output_filename,
         "--port",
         "{}".format(port),
         "--server",
         "{}".format(server),
-        "--json-out-file",
-        local_benchmark_output_filename,
     ]
     if oss_cluster_api_enabled is True:
         benchmark_command.append("--cluster-mode")

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 2.0 element-1-simple-score 62.0 element-2-medium-score 2047.9999999999998 element-3-long-score 1.0e+30 element-4-very-long-score 1.0e+60 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different valid hexadecimal floating-point numbers, gradually increasing in length of the string to parse.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 0x1p+1 element-1-simple-score 0x1.fp+5 element-2-medium-score 0x1.fffffffffffffp+10 element-3-long-score 0x1.ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+100 element-4-very-long-score 0x1.fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+200 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements
+description: Runs memtier_benchmark, for a keyspace length of 1 key doing zrangestore on a high cardinality sorted set
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zrangestore
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 1000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zrangestore
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 300000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

utils/tests/test_runner.py

@@ -265,7 +265,7 @@ def test_extract_testsuites():
             "--test-suites-folder",
             "./utils/tests/test_data/test-suites",
             "--tests-regex",
-            ".*\.yml",
+            r".*\.yml",
         ]
     )
     tests = extract_testsuites(args)

utils/tests/test_self_contained_coordinator_memtier.py

@@ -1076,7 +1076,7 @@ def test_prepare_memtier_benchmark_parameters():
         )
         assert (
             benchmark_command_str
-            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+            == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
         )
         oss_api_enabled = True
         (
@@ -1092,7 +1092,7 @@ def test_prepare_memtier_benchmark_parameters():
         )
         assert (
             benchmark_command_str
-            == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
+            == 'memtier_benchmark --json-out-file 1.json --port 12000 --server localhost --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __data__" --command-key-pattern="R" --command "SET __key__ __data__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
         )