From 59da33e2dfad2ea52fddcbcada45b6209f3bd9f4 Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Thu, 21 Nov 2024 11:56:44 -0800 Subject: [PATCH 1/2] Test new preview repair metrics --- repair_tests/preview_repair_test.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/repair_tests/preview_repair_test.py b/repair_tests/preview_repair_test.py index df03b3604e..6276e4f51f 100644 --- a/repair_tests/preview_repair_test.py +++ b/repair_tests/preview_repair_test.py @@ -43,11 +43,15 @@ def test_preview(self): session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") + desynchronized_bytes = preview_desynchronized_token_ranges_count(node1) # everything should be in sync result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout assert_no_repair_history(session) assert preview_failure_count(node1) == 0 + assert preview_desynchronized_bytes_count(node1) == 0 + assert preview_desynchronized_token_ranges_count(node1) == 0 + assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes # make data inconsistent between nodes stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") @@ -72,15 +76,21 @@ def test_preview(self): node1.start(wait_for_binary_proto=True) # data should not be in sync for full and unrepaired previews + desynchronized_bytes = preview_desynchronized_bytes_count(node1) result = node1.repair(options=['ks', '--preview']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 1 + assert preview_desynchronized_token_ranges_count(node1) == 80 + assert preview_desynchronized_bytes_count(node1) > desynchronized_bytes + desynchronized_bytes = preview_desynchronized_bytes_count(node1) result = node1.repair(options=['ks', '--preview', '--full']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 2 + assert preview_desynchronized_token_ranges_count(node1) == 160 + assert preview_desynchronized_bytes_count(node1) > desynchronized_bytes # repaired data should be in sync anyway result = node1.repair(options=['ks', '--validate']) @@ -94,20 +104,30 @@ def test_preview(self): node.nodetool('compact ks tbl') # ...and everything should be in sync + desynchronized_bytes = preview_desynchronized_bytes_count(node1) result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout # data is repaired, previewFailure metric should remain same assert preview_failure_count(node1) == 2 + assert preview_desynchronized_token_ranges_count(node1) == 160 + assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes + desynchronized_bytes = preview_desynchronized_bytes_count(node1) result = node1.repair(options=['ks', '--preview', '--full']) assert "Previewed data was in sync" in result.stdout assert preview_failure_count(node1) == 2 + assert preview_desynchronized_token_ranges_count(node1) == 160 + assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes result = node1.repair(options=['ks', '--validate']) assert "Repaired data is in sync" in result.stdout assert preview_failure_count(node2) == 0 + assert preview_desynchronized_bytes_count(node2) == 0 + assert preview_desynchronized_token_ranges_count(node2) == 0 assert preview_failure_count(node3) == 0 + assert preview_desynchronized_bytes_count(node3) == 0 + assert preview_desynchronized_token_ranges_count(node3) == 0 def assert_no_repair_history(session): @@ -121,3 +141,13 @@ def preview_failure_count(node): mbean = make_mbean('metrics', type='Repair', name='PreviewFailures') with JolokiaAgent(node) as jmx: return jmx.read_attribute(mbean, 'Count') + +def preview_desynchronized_token_ranges_count(node): + mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='PreviewedDesynchronizedTokenRanges') + with JolokiaAgent(node) as jmx: + return jmx.read_attribute(mbean, 'Count') + +def preview_desynchronized_bytes_count(node): + mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='PreviewedDesynchronizedBytes') + with JolokiaAgent(node) as jmx: + return jmx.read_attribute(mbean, 'Count') From 5428300243c886da232bbf33a1c0b67eb9c9de1b Mon Sep 17 00:00:00 2001 From: Kristijonas Zalys Date: Mon, 9 Dec 2024 21:52:02 -0800 Subject: [PATCH 2/2] Add bytes previewed metric --- repair_tests/preview_repair_test.py | 62 ++++++++++++++++++----------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/repair_tests/preview_repair_test.py b/repair_tests/preview_repair_test.py index 6276e4f51f..864f05e9b1 100644 --- a/repair_tests/preview_repair_test.py +++ b/repair_tests/preview_repair_test.py @@ -43,15 +43,15 @@ def test_preview(self): session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") - desynchronized_bytes = preview_desynchronized_token_ranges_count(node1) # everything should be in sync result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout assert_no_repair_history(session) assert preview_failure_count(node1) == 0 - assert preview_desynchronized_bytes_count(node1) == 0 - assert preview_desynchronized_token_ranges_count(node1) == 0 - assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes + assert bytes_previewed_desynchronized_count(node1) == 0 + assert token_ranges_previewed_desynchronized_count(node1) == 0 + assert bytes_previewed_count(node1) == 0 + assert bytes_previewed_desynchronized_count(node1) == 0 # make data inconsistent between nodes stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") @@ -76,21 +76,24 @@ def test_preview(self): node1.start(wait_for_binary_proto=True) # data should not be in sync for full and unrepaired previews - desynchronized_bytes = preview_desynchronized_bytes_count(node1) + desynchronized_bytes = bytes_previewed_desynchronized_count(node1) + previewed_bytes = bytes_previewed_count(node1) result = node1.repair(options=['ks', '--preview']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 1 - assert preview_desynchronized_token_ranges_count(node1) == 80 - assert preview_desynchronized_bytes_count(node1) > desynchronized_bytes + assert token_ranges_previewed_desynchronized_count(node1) == 80 + assert bytes_previewed_desynchronized_count(node1) > previewed_bytes - desynchronized_bytes = preview_desynchronized_bytes_count(node1) + desynchronized_bytes = bytes_previewed_desynchronized_count(node1) + previewed_bytes = bytes_previewed_count(node1) result = node1.repair(options=['ks', '--preview', '--full']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 2 - assert preview_desynchronized_token_ranges_count(node1) == 160 - assert preview_desynchronized_bytes_count(node1) > desynchronized_bytes + assert token_ranges_previewed_desynchronized_count(node1) == 160 + assert bytes_previewed_desynchronized_count(node1) > desynchronized_bytes + assert bytes_previewed_count(node1) > previewed_bytes # repaired data should be in sync anyway result = node1.repair(options=['ks', '--validate']) @@ -104,30 +107,36 @@ def test_preview(self): node.nodetool('compact ks tbl') # ...and everything should be in sync - desynchronized_bytes = preview_desynchronized_bytes_count(node1) + desynchronized_bytes = bytes_previewed_desynchronized_count(node1) + previewed_bytes = bytes_previewed_count(node1) result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout # data is repaired, previewFailure metric should remain same assert preview_failure_count(node1) == 2 - assert preview_desynchronized_token_ranges_count(node1) == 160 - assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes + assert token_ranges_previewed_desynchronized_count(node1) == 160 + assert bytes_previewed_desynchronized_count(node1) == desynchronized_bytes + assert bytes_previewed_count(node1) == previewed_bytes - desynchronized_bytes = preview_desynchronized_bytes_count(node1) + desynchronized_bytes = bytes_previewed_desynchronized_count(node1) + previewed_bytes = bytes_previewed_count(node1) result = node1.repair(options=['ks', '--preview', '--full']) assert "Previewed data was in sync" in result.stdout assert preview_failure_count(node1) == 2 - assert preview_desynchronized_token_ranges_count(node1) == 160 - assert preview_desynchronized_bytes_count(node1) == desynchronized_bytes + assert token_ranges_previewed_desynchronized_count(node1) == 160 + assert bytes_previewed_desynchronized_count(node1) == desynchronized_bytes + assert bytes_previewed_count(node1) > previewed_bytes result = node1.repair(options=['ks', '--validate']) assert "Repaired data is in sync" in result.stdout assert preview_failure_count(node2) == 0 - assert preview_desynchronized_bytes_count(node2) == 0 - assert preview_desynchronized_token_ranges_count(node2) == 0 + assert bytes_previewed_desynchronized_count(node2) == 0 + assert token_ranges_previewed_desynchronized_count(node2) == 0 assert preview_failure_count(node3) == 0 - assert preview_desynchronized_bytes_count(node3) == 0 - assert preview_desynchronized_token_ranges_count(node3) == 0 + assert bytes_previewed_desynchronized_count(node3) == 0 + assert token_ranges_previewed_desynchronized_count(node3) == 0 + assert bytes_previewed_count(node2) > 0 + assert bytes_previewed_count(node3) > 0 def assert_no_repair_history(session): @@ -142,12 +151,17 @@ def preview_failure_count(node): with JolokiaAgent(node) as jmx: return jmx.read_attribute(mbean, 'Count') -def preview_desynchronized_token_ranges_count(node): - mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='PreviewedDesynchronizedTokenRanges') +def token_ranges_previewed_desynchronized_count(node): + mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='TokenRangesPreviewedDesynchronized') with JolokiaAgent(node) as jmx: return jmx.read_attribute(mbean, 'Count') -def preview_desynchronized_bytes_count(node): - mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='PreviewedDesynchronizedBytes') +def bytes_previewed_desynchronized_count(node): + mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='BytesPreviewedDesynchronized') + with JolokiaAgent(node) as jmx: + return jmx.read_attribute(mbean, 'Count') + +def bytes_previewed_count(node): + mbean = make_mbean('metrics', type='Table', keyspace='ks', scope='tbl', name='BytesPreviewed') with JolokiaAgent(node) as jmx: return jmx.read_attribute(mbean, 'Count')