Merge remote-tracking branch 'origin/main' into clean-topology-filter

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.260"
+version = "0.1.263"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__cli__/stats.py

@@ -108,6 +108,8 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
         benchmark_config = {}
         requires_override = False
         test_result = True
+        tested_groups_match_origin = True
+
         with open(test_file, "r") as stream:
 
             try:
@@ -291,7 +293,7 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     )
 
                 if tested_groups != origin_tested_groups:
-                    requires_override = True
+                    tested_groups_match_origin = False
                     benchmark_config["tested-groups"] = tested_groups
                     logging.warn(
                         "there is a difference between specified test-groups in the yaml (name={}) and the ones we've detected {}!={}".format(
@@ -312,7 +314,15 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
             test_result = False
         overall_result &= test_result
 
-        if requires_override and override_enabled:
+        if not tested_groups_match_origin:
+            if len(tested_groups) > 0:
+                overall_result = False
+            else:
+                logging.warn(
+                    "difference between specified and detected test-groups was ignored since command info is not available in this benchmark version"
+                )
+
+        if (requires_override or not tested_groups_match_origin) and override_enabled:
             logging.info(
                 "Saving a new version of the file {} with the overrided data".format(
                     test_file

redis_benchmarks_specification/__self_contained_coordinator__/clients.py

@@ -8,12 +8,12 @@ def prepare_memtier_benchmark_parameters(
 ):
     benchmark_command = [
         full_benchmark_path,
+        "--json-out-file",
+        local_benchmark_output_filename,
         "--port",
         "{}".format(port),
         "--server",
         "{}".format(server),
-        "--json-out-file",
-        local_benchmark_output_filename,
     ]
     if oss_cluster_api_enabled is True:
         benchmark_command.append("--cluster-mode")

redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix.yml

@@ -1,34 +1,28 @@
 version: 0.4
 name: memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix
-description: Runs memtier_benchmark, for a keyspace length of 10M keys with a data size of 10 Bytes for each key.
+description: memtier_benchmark, 10M keys, string GET, 10B value size, pipeline=100, no key prefix
 dbconfig:
   configuration-parameters:
     save: '""'
   check:
     keyspacelen: 10000000
-  preload_tool:
-    run_image: redislabs/memtier_benchmark:edge
-    tool: memtier_benchmark
-    arguments: '--key-maximum  10000000 -n allkeys --key-prefix "" --data-size 10 --ratio 1:0 --key-pattern P:P -c 50 -t 2 --hide-histogram --key-minimum 1'
   resources:
     requests:
-      memory: 1g
+      memory: 10g
+tested-groups:
+- string
 tested-commands:
 - get
 redis-topologies:
 - oss-standalone
-build-variants:
-- gcc:8.5.0-amd64-debian-buster-default
-- dockerhub
+- oss-cluster-3-primaries
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '--key-maximum  10000000 --key-minimum 1 --pipeline 100 --key-prefix "" --distinct-client-seed --data-size 10 --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120'
+  arguments: '"--data-size" "10" --command "GET __key__" --command-key-pattern="R" --key-prefix="" -c 50 -t 2 --pipeline 100 --hide-histogram --test-time 180'
   resources:
     requests:
-      cpus: '10'
+      cpus: '3'
       memory: 2g
 
-tested-groups:
-- string
-priority: 1
+priority: 33

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix.yml

@@ -1,30 +1,28 @@
 version: 0.4
 name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix
-description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes.
+description: memtier_benchmark, 1M keys, string SET, 10B value size, pipeline=100, no key prefix
 dbconfig:
   configuration-parameters:
     save: '""'
   check:
-    keyspacelen: 0
+    keyspacelen: 1000000
   resources:
     requests:
       memory: 1g
+tested-groups:
+- string
 tested-commands:
 - set
 redis-topologies:
 - oss-standalone
-build-variants:
-- gcc:8.5.0-amd64-debian-buster-default
-- dockerhub
+- oss-cluster-3-primaries
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--pipeline" "100" "--data-size" "10" --distinct-client-seed --key-prefix "" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 120 -c 10 -t 10 --hide-histogram'
+  arguments: '"--data-size" "10" --command "SET __key__ __data__" --command-key-pattern="R" --key-prefix="" -c 50 -t 2 --pipeline 100 --hide-histogram --test-time 180'
   resources:
     requests:
-      cpus: '10'
+      cpus: '3'
       memory: 2g
 
-tested-groups:
-- string
-priority: 17
+priority: 33

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-10B-pipeline-100-nokeyprefix.yml

@@ -1,34 +1,28 @@
 version: 0.4
 name: memtier_benchmark-1Mkeys-string-get-10B-pipeline-100-nokeyprefix
-description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 10 Bytes for each key.
+description: memtier_benchmark, 1M keys, string GET, 10B value size, pipeline=100, no key prefix
 dbconfig:
   configuration-parameters:
     save: '""'
   check:
     keyspacelen: 1000000
-  preload_tool:
-    run_image: redislabs/memtier_benchmark:edge
-    tool: memtier_benchmark
-    arguments: '--key-maximum  1000000 -n allkeys --key-prefix "" --data-size 10 --ratio 1:0 --key-pattern P:P -c 50 -t 2 --hide-histogram --key-minimum 1'
   resources:
     requests:
       memory: 1g
+tested-groups:
+- string
 tested-commands:
 - get
 redis-topologies:
 - oss-standalone
-build-variants:
-- gcc:8.5.0-amd64-debian-buster-default
-- dockerhub
+- oss-cluster-3-primaries
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '--key-maximum  1000000 --key-minimum 1 --pipeline 100 --key-prefix "" --distinct-client-seed --data-size 10 --ratio 0:1 --key-pattern R:R -c 10 -t 10 --hide-histogram --test-time 120'
+  arguments: '"--data-size" "10" --command "GET __key__" --command-key-pattern="R" --key-prefix="" -c 50 -t 2 --pipeline 100 --hide-histogram --test-time 180'
   resources:
     requests:
-      cpus: '10'
+      cpus: '3'
       memory: 2g
 
-tested-groups:
-- string
-priority: 1
+priority: 33

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-float-score
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 2.0 element-1-simple-score 62.0 element-2-medium-score 2047.9999999999998 element-3-long-score 1.0e+30 element-4-very-long-score 1.0e+60 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1key-load-zset-with-5-elements-parsing-hexa-score
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different valid hexadecimal floating-point numbers, gradually increasing in length of the string to parse.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 50 -t 4 --command "ZADD zset 0x1p+1 element-1-simple-score 0x1.fp+5 element-2-medium-score 0x1.fffffffffffffp+10 element-3-long-score 0x1.ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+100 element-4-very-long-score 0x1.fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffp+200 element-5-extra-large-score" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zset-600K-elements-zrangestore-1K-elements
+description: Runs memtier_benchmark, for a keyspace length of 1 key doing zrangestore on a high cardinality sorted set
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zrangestore
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 1000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zset-600K-elements-zrangestore-300K-elements
+description: Runs memtier_benchmark, for a keyspace length of 1 key loading the sorted set with 5 elements with different floating-point numbers, gradually increasing in the size of the float to parse. This is a control benchmark for the one named memtier_benchmark-1key-load-zset-with-5-elements-hexa-score
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '-n allkeys "--data-size" "10" --key-prefix "" "--command" "ZADD zset __key__ __key__" "--command-key-pattern" "P" "-c" "1" "-t" "1" "--hide-histogram" "--key-minimum" "1" "--key-maximum" "600001"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- zrangestore
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 -c 1 -t 1 --command "ZRANGESTORE zset1 zset 0 300000" --command-key-pattern="P" --key-minimum=1 --key-maximum 1 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- sorted-set
+priority: 12