From 1977ab59e95bc5587f3a06e0cdfdc846c64c0fda Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Thu, 23 Jan 2025 16:19:09 +0300 Subject: [PATCH 1/3] Add test for storage exhaustion when deleting rows --- ydb/tests/olap/test_quota_exhaustion.py | 64 +++++++++++++++++++------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/ydb/tests/olap/test_quota_exhaustion.py b/ydb/tests/olap/test_quota_exhaustion.py index 2f17b5d3c1b2..579bbbec1e27 100644 --- a/ydb/tests/olap/test_quota_exhaustion.py +++ b/ydb/tests/olap/test_quota_exhaustion.py @@ -22,15 +22,14 @@ def setup_class(cls): def teardown_class(cls): cls.cluster.stop() - def test(self): - """As per https://github.com/ydb-platform/ydb/issues/13529""" - + def make_session(self): driver = ydb.Driver(endpoint=f'grpc://localhost:{self.cluster.nodes[1].grpc_port}', database='/Root') session = ydb.QuerySessionPool(driver) driver.wait(5, fail_fast=True) + return session - def create_table(table): - return session.execute_with_retries(f""" + def create_test_table(self, session, table): + return session.execute_with_retries(f""" CREATE TABLE {table} ( k Int32 NOT NULL, v Uint64, @@ -38,8 +37,8 @@ def create_table(table): ) WITH (STORE = COLUMN) """) - def upsert_chunk(table, chunk_id, retries=10): - return session.execute_with_retries(f""" + def upsert_test_chunk(self, session, table, chunk_id, retries=10): + return session.execute_with_retries(f""" $n = {ROWS_CHUNK_SIZE}; $values_list = ListReplicate(42ul, $n); $rows_list = ListFoldMap($values_list, {chunk_id * ROWS_CHUNK_SIZE}, ($val, $i) -> ((<|k:$i, v:$val|>, $i + 1))); @@ -48,17 +47,54 @@ def upsert_chunk(table, chunk_id, retries=10): SELECT * FROM AS_TABLE($rows_list); """, None, ydb.retries.RetrySettings(max_retries=retries)) - create_table('huge') - + def upsert_until_overload(self, session, table): try: for i in range(ROWS_CHUNKS_COUNT): - res = upsert_chunk('huge', i, retries=0) - print(f"query #{i} ok, result:", res, file=sys.stderr) + res = self.upsert_test_chunk(session, table, i, retries=0) + print(f"upsert #{i} ok, result:", res, file=sys.stderr) except ydb.issues.Overloaded: - print('got overload issue', file=sys.stderr) + print('upsert: got overload issue', file=sys.stderr) + + def test(self): + """As per https://github.com/ydb-platform/ydb/issues/13529""" + session = self.make_session() + # Overflow the database + self.create_test_table(session, 'huge') + self.upsert_until_overload(session, 'huge') + + # Cleanup session.execute_with_retries("""DROP TABLE huge""") # Check database health after cleanup - create_table('small') - upsert_chunk('small', 0) + self.create_test_table(session, 'small') + self.upsert_test_chunk(session, 'small', 0) + + def delete_test_chunk(self, session, table, chunk_id, retries=10): + session.execute_with_retries(f""" + DELETE FROM {table} + WHERE {chunk_id * ROWS_CHUNK_SIZE} <= k AND k <= {chunk_id * ROWS_CHUNK_SIZE + ROWS_CHUNK_SIZE} + """, None, ydb.retries.RetrySettings(max_retries=retries)) + + def delete_until_overload(self, session, table): + for i in range(ROWS_CHUNKS_COUNT): + try: + self.delete_test_chunk(session, table, i, retries=0) + print(f"delete #{i} ok", file=sys.stderr) + except ydb.issues.Overloaded: + print('delete: got overload issue', file=sys.stderr) + return i + + def test_delete(self): + """As per https://github.com/ydb-platform/ydb/issues/13653""" + session = self.make_session() + + # Overflow the database + self.create_test_table(session, 'huge') + self.upsert_until_overload(session, 'huge') + + # Check that deletions will lead to overflow, too + i = self.delete_until_overload(session, 'huge') + + # Try to wait until deletion works again (after compaction) + self.delete_test_chunk(session, 'huge', i) From b1b659610fec68fb6bd6c8ccdfb08bf39aadd9c2 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Fri, 24 Jan 2025 15:22:32 +0300 Subject: [PATCH 2/3] Add quota limits --- ydb/tests/olap/test_quota_exhaustion.py | 47 ++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/ydb/tests/olap/test_quota_exhaustion.py b/ydb/tests/olap/test_quota_exhaustion.py index 579bbbec1e27..157c73ce1127 100644 --- a/ydb/tests/olap/test_quota_exhaustion.py +++ b/ydb/tests/olap/test_quota_exhaustion.py @@ -1,4 +1,7 @@ +import os +import subprocess import sys +import time import ydb from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator @@ -85,15 +88,57 @@ def delete_until_overload(self, session, table): print('delete: got overload issue', file=sys.stderr) return i + def ydbcli_db_schema_exec(self, node, operation_proto): + endpoint = f"{node.host}:{node.port}" + args = [ + node.binary_path, + f"--server=grpc://{endpoint}", + "db", + "schema", + "exec", + operation_proto, + ] + command = subprocess.run(args, capture_output=True) + assert command.returncode == 0, command.stderr.decode("utf-8") + + + def alter_database_quotas(self, node, database_path, database_quotas): + alter_proto = """ModifyScheme { + OperationType: ESchemeOpAlterSubDomain + WorkingDir: "%s" + SubDomain { + Name: "%s" + DatabaseQuotas { + %s + } + } + }""" % ( + os.path.dirname(database_path), + os.path.basename(database_path), + database_quotas, + ) + + self.ydbcli_db_schema_exec(node, alter_proto) + def test_delete(self): """As per https://github.com/ydb-platform/ydb/issues/13653""" session = self.make_session() + # Set soft and hard quotas to 6GB + self.alter_database_quotas(self.cluster.nodes[1], '/Root', """ + data_size_hard_quota: 6000000000 + data_size_soft_quota: 6000000000 + """) + # Overflow the database self.create_test_table(session, 'huge') self.upsert_until_overload(session, 'huge') - # Check that deletions will lead to overflow, too + # Check that deletion works at least first time + # self.delete_test_chunk(session, 'huge', 0) + # ^ uncomment after fixing https://github.com/ydb-platform/ydb/issues/13808 + + # Check that deletions will lead to overflow at some moment i = self.delete_until_overload(session, 'huge') # Try to wait until deletion works again (after compaction) From 701cbfb7bac49a76e4373b352c1268f9ce3f0b84 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Thu, 13 Feb 2025 18:42:35 +0300 Subject: [PATCH 3/3] Fix codestyle --- ydb/tests/olap/test_quota_exhaustion.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ydb/tests/olap/test_quota_exhaustion.py b/ydb/tests/olap/test_quota_exhaustion.py index 157c73ce1127..a8464775f4b1 100644 --- a/ydb/tests/olap/test_quota_exhaustion.py +++ b/ydb/tests/olap/test_quota_exhaustion.py @@ -1,7 +1,6 @@ import os import subprocess import sys -import time import ydb from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator @@ -101,7 +100,6 @@ def ydbcli_db_schema_exec(self, node, operation_proto): command = subprocess.run(args, capture_output=True) assert command.returncode == 0, command.stderr.decode("utf-8") - def alter_database_quotas(self, node, database_path, database_quotas): alter_proto = """ModifyScheme { OperationType: ESchemeOpAlterSubDomain