Skip to content

Commit d4f9aa1

Browse files
EnmkEnmk
authored
Merge pull request #961 from Altinity/stateless_broken_unknown_db
Antalya 25.6.5 test crossouts
2 parents 0488d00 + 625c3bd commit d4f9aa1

File tree

13 files changed

+193
-426
lines changed

13 files changed

+193
-426
lines changed

‎.github/workflows/master.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3595,7 +3595,7 @@ jobs:
35953595
35963596
stateless_tests_amd_coverage_1_6:
35973597
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3598-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3598+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
35993599
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMS82KQ==') }}
36003600
name: "Stateless tests (amd_coverage,1/6)"
36013601
outputs:
@@ -3640,7 +3640,7 @@ jobs:
36403640
36413641
stateless_tests_amd_coverage_2_6:
36423642
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3643-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3643+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
36443644
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMi82KQ==') }}
36453645
name: "Stateless tests (amd_coverage,2/6)"
36463646
outputs:
@@ -3685,7 +3685,7 @@ jobs:
36853685
36863686
stateless_tests_amd_coverage_3_6:
36873687
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3688-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3688+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
36893689
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMy82KQ==') }}
36903690
name: "Stateless tests (amd_coverage,3/6)"
36913691
outputs:
@@ -3730,7 +3730,7 @@ jobs:
37303730
37313731
stateless_tests_amd_coverage_4_6:
37323732
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3733-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3733+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
37343734
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNC82KQ==') }}
37353735
name: "Stateless tests (amd_coverage,4/6)"
37363736
outputs:
@@ -3775,7 +3775,7 @@ jobs:
37753775
37763776
stateless_tests_amd_coverage_5_6:
37773777
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3778-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3778+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
37793779
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNS82KQ==') }}
37803780
name: "Stateless tests (amd_coverage,5/6)"
37813781
outputs:
@@ -3820,7 +3820,7 @@ jobs:
38203820
38213821
stateless_tests_amd_coverage_6_6:
38223822
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
3823-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
3823+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, build_arm_coverage]
38243824
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNi82KQ==') }}
38253825
name: "Stateless tests (amd_coverage,6/6)"
38263826
outputs:

‎.github/workflows/pull_request.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2022,7 +2022,7 @@ jobs:
20222022
20232023
stateless_tests_amd_coverage_1_6:
20242024
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2025-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2025+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
20262026
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMS82KQ==') }}
20272027
name: "Stateless tests (amd_coverage,1/6)"
20282028
outputs:
@@ -2067,7 +2067,7 @@ jobs:
20672067
20682068
stateless_tests_amd_coverage_2_6:
20692069
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2070-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2070+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
20712071
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMi82KQ==') }}
20722072
name: "Stateless tests (amd_coverage,2/6)"
20732073
outputs:
@@ -2112,7 +2112,7 @@ jobs:
21122112
21132113
stateless_tests_amd_coverage_3_6:
21142114
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2115-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2115+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
21162116
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsMy82KQ==') }}
21172117
name: "Stateless tests (amd_coverage,3/6)"
21182118
outputs:
@@ -2157,7 +2157,7 @@ jobs:
21572157
21582158
stateless_tests_amd_coverage_4_6:
21592159
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2160-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2160+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
21612161
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNC82KQ==') }}
21622162
name: "Stateless tests (amd_coverage,4/6)"
21632163
outputs:
@@ -2202,7 +2202,7 @@ jobs:
22022202
22032203
stateless_tests_amd_coverage_5_6:
22042204
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2205-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2205+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
22062206
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNS82KQ==') }}
22072207
name: "Stateless tests (amd_coverage,5/6)"
22082208
outputs:
@@ -2247,7 +2247,7 @@ jobs:
22472247
22482248
stateless_tests_amd_coverage_6_6:
22492249
runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64]
2250-
needs: [config_workflow, dockers_build_amd, dockers_build_arm, stateless_tests_amd_asan_distributed_plan_1_2, stateless_tests_amd_asan_distributed_plan_2_2, build_arm_coverage]
2250+
needs: [config_workflow, dockers_build_amd, dockers_build_arm, build_arm_coverage]
22512251
if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfY292ZXJhZ2UsNi82KQ==') }}
22522252
name: "Stateless tests (amd_coverage,6/6)"
22532253
outputs:

‎ci/defs/job_configs.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"./tests/clickhouse-test",
3434
"./tests/config",
3535
"./tests/*.txt",
36+
"./tests/broken_tests.json",
3637
],
3738
),
3839
result_name_for_cidb="Tests",
@@ -588,6 +589,7 @@ class JobConfigs:
588589
"./tests/ci/integration_tests_runner.py",
589590
"./tests/integration/",
590591
"./docker",
592+
"./tests/broken_tests.json",
591593
],
592594
),
593595
).parametrize(
@@ -610,6 +612,7 @@ class JobConfigs:
610612
"./tests/ci/integration_tests_runner.py",
611613
"./tests/integration/",
612614
"./docker",
615+
"./tests/broken_tests.json",
613616
],
614617
),
615618
).parametrize(
@@ -645,6 +648,7 @@ class JobConfigs:
645648
"./tests/ci/integration_tests_runner.py",
646649
"./tests/integration/",
647650
"./docker",
651+
"./tests/broken_tests.json",
648652
],
649653
),
650654
allow_merge_on_failure=True,
@@ -670,6 +674,7 @@ class JobConfigs:
670674
"./tests/ci/integration_tests_runner.py",
671675
"./tests/integration/",
672676
"./docker",
677+
"./tests/broken_tests.json",
673678
],
674679
),
675680
requires=["Build (amd_asan)"],

‎ci/jobs/scripts/functional_tests_results.py

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import dataclasses
2+
import json
3+
import os
24
import traceback
35
from typing import List
46

@@ -28,6 +30,16 @@
2830
# out.writerow(status)
2931

3032

33+
def get_broken_tests_list() -> dict:
34+
file_path = "tests/broken_tests.json"
35+
if not os.path.isfile(file_path) or os.path.getsize(file_path) == 0:
36+
return {}
37+
38+
with open(file_path, "r", encoding="utf-8") as skip_list_file:
39+
skip_list_tests = json.load(skip_list_file)
40+
return skip_list_tests
41+
42+
3143
class FTResultsProcessor:
3244
@dataclasses.dataclass
3345
class Summary:
@@ -36,6 +48,7 @@ class Summary:
3648
unknown: int
3749
failed: int
3850
success: int
51+
broken: int
3952
test_results: List[Result]
4053
hung: bool = False
4154
server_died: bool = False
@@ -54,13 +67,16 @@ def _process_test_output(self):
5467
unknown = 0
5568
failed = 0
5669
success = 0
70+
broken = 0
5771
hung = False
5872
server_died = False
5973
retries = False
6074
success_finish = False
6175
test_results = []
6276
test_end = True
6377

78+
known_broken_tests = get_broken_tests_list()
79+
6480
with open(self.tests_output_file, "r", encoding="utf-8") as test_file:
6581
for line in test_file:
6682
original_line = line
@@ -133,6 +149,26 @@ def _process_test_output(self):
133149
info="".join(test[3])[:16384],
134150
)
135151
)
152+
153+
if test[1] == "FAIL":
154+
broken_message = None
155+
if test[0] in known_broken_tests.keys():
156+
if known_broken_tests[test[0]].get("message"):
157+
if (
158+
known_broken_tests[test[0]]["message"]
159+
in test_results_[-1].info
160+
):
161+
broken_message = f"\nMarked as broken, matched message: '{known_broken_tests[test[0]]['message']}'"
162+
else:
163+
broken_message = f"\nMarked as broken, no message specified"
164+
165+
if broken_message:
166+
broken += 1
167+
failed -= 1
168+
test_results_[-1].set_status(Result.StatusExtended.BROKEN)
169+
test_results_[-1].set_label(Result.Label.BROKEN)
170+
test_results_[-1].info += broken_message
171+
136172
except Exception as e:
137173
print(f"ERROR: Failed to parse test results: [{test}]")
138174
traceback.print_exc()
@@ -153,6 +189,7 @@ def _process_test_output(self):
153189
unknown=unknown,
154190
failed=failed,
155191
success=success,
192+
broken=broken,
156193
test_results=test_results,
157194
hung=hung,
158195
server_died=server_died,
@@ -220,7 +257,7 @@ def run(self):
220257
pass
221258

222259
if not info:
223-
info = f"Failed: {s.failed}, Passed: {s.success}, Skipped: {s.skipped}"
260+
info = f"Failed: {s.failed}, Passed: {s.success}, Skipped: {s.skipped}, Broken: {s.broken}"
224261

225262
# TODO: !!!
226263
# def test_result_comparator(item):

‎ci/praktika/result.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class StatusExtended:
5252
OK = "OK"
5353
FAIL = "FAIL"
5454
SKIPPED = "SKIPPED"
55+
BROKEN = "BROKEN"
5556

5657
class Label:
5758
REQUIRED = "required"
@@ -170,6 +171,7 @@ def is_ok(self):
170171
Result.Status.SUCCESS,
171172
Result.StatusExtended.OK,
172173
Result.StatusExtended.SKIPPED,
174+
Result.StatusExtended.BROKEN,
173175
)
174176

175177
def is_error(self):

‎ci/workflows/master.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,7 @@
3131
*JobConfigs.functional_tests_jobs_azure_master_only,
3232
*JobConfigs.integration_test_jobs_required,
3333
*JobConfigs.integration_test_jobs_non_required,
34-
*[
35-
job.set_dependency(REQUIRED_STATELESS_TESTS_JOB_NAMES)
36-
for job in JobConfigs.functional_tests_jobs_coverage
37-
],
34+
*JobConfigs.functional_tests_jobs_coverage,
3835
*JobConfigs.stress_test_jobs,
3936
# *JobConfigs.stress_test_azure_master_jobs, # NOTE (strtgbb): disabled due to ASAN build failure
4037
*JobConfigs.ast_fuzzer_jobs,

‎ci/workflows/pull_request.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,7 @@
4141
*JobConfigs.compatibility_test_jobs,
4242
*JobConfigs.functional_tests_jobs_required,
4343
*JobConfigs.functional_tests_jobs_non_required,
44-
*[
45-
job.set_dependency(REQUIRED_STATELESS_TESTS_JOB_NAMES)
46-
for job in JobConfigs.functional_tests_jobs_coverage
47-
],
44+
*JobConfigs.functional_tests_jobs_coverage,
4845
JobConfigs.bugfix_validation_it_job.set_dependency(
4946
[
5047
# JobNames.STYLE_CHECK, # NOTE (strtgbb): we don't run style check

0 commit comments

Comments
 (0)