Shared metadata

.github/actions/build_and_test_ya/action.yml

@@ -51,20 +51,45 @@ inputs:
   vars:
     type: string
     default: ""
+  custom_branch_name:
+    description: "Custom branch name required when workflow branch != checkout branch"
+    type: string
+    required: false
 defaults:
   run:
     shell: bash
 runs:
   using: "composite"
   steps:
+    - name: Prepare folder prefix
+      id: prepare_prefix
+      shell: bash
+      run: |
+        # Check if custom_branch_name is set and not empty
+        if [ -n "${{ inputs.custom_branch_name }}" ]; then
+          # Extract and sanitize custom_branch_name
+          CUSTOM_BRANCH_NAME="${{ inputs.custom_branch_name }}"
+          # Replace all unsupported characters with hyphens
+          SANITIZED_NAME="${CUSTOM_BRANCH_NAME//[^a-zA-Z0-9-]/-}"
+          # Optionally limit the length to, say, 50 characters
+          SANITIZED_NAME="${SANITIZED_NAME:0:50}"
+          # Assign the sanitized name to the folder_prefix
+          FOLDER_PREFIX="ya-${SANITIZED_NAME}-"
+        else
+          # If the branch name is not provided, use a default prefix
+          FOLDER_PREFIX='ya-'
+        fi
+        # Output the folder_prefix for use in subsequent steps
+        echo "folder_prefix=${FOLDER_PREFIX}" >> $GITHUB_ENV
+
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd
       with:
         s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
         s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
         s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
         s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
-        folder_prefix: ya-
+        folder_prefix: ${{ env.folder_prefix }}
         build_preset: ${{ inputs.build_preset }}
 
     - name: Run build and tests
@@ -88,3 +113,4 @@ runs:
         bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
         put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
         test_retry_count: ${{ inputs.test_retry_count }}
+        custom_branch_name: ${{ inputs.custom_branch_name }}

.github/actions/test_ya/action.yml

@@ -60,6 +60,11 @@ inputs:
     type: string
     default: ""
     description: "how many times to retry failed tests"
+  custom_branch_name:
+    description: "Custom branch name required when workflow branch != checkout branch"
+    type: string
+    required: false
+
 outputs:
   success:
     value: ${{ steps.build.outputs.status }}
@@ -271,8 +276,18 @@ runs:
           --cache-size 2TB --force-build-depends
         )
 
+        echo "inputs.custom_branch_name = ${{ inputs.custom_branch_name }}"
+        echo "GITHUB_REF_NAME = $GITHUB_REF_NAME"
+
+        if [ -z "${{ inputs.custom_branch_name }}" ]; then
+          BRANCH_NAME="${GITHUB_REF_NAME}"
+        else
+          BRANCH_NAME="${{ inputs.custom_branch_name }}"
+        fi
+        echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+        echo "BRANCH_NAME is set to $BRANCH_NAME"
 
-        TESTMO_BRANCH_TAG="$GITHUB_REF_NAME"
+        TESTMO_BRANCH_TAG="$BRANCH_NAME"
         TESTMO_ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
         TESTMO_PR_NUMBER=${{ github.event.number }}
 
@@ -453,6 +468,7 @@ runs:
               --public_dir "$PUBLIC_DIR" \
               --public_dir_url "$PUBLIC_DIR_URL" \
               --build_preset "$BUILD_PRESET" \
+              --branch "$BRANCH_NAME" \
               --status_report_file statusrep.txt \
               --is_retry $IS_RETRY \
               --is_last_retry $IS_LAST_RETRY \
@@ -470,7 +486,7 @@ runs:
 
           # upload tests results to YDB
           ydb_upload_run_name="${TESTMO_RUN_NAME// /"_"}"
-          result=`.github/scripts/analytics/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`
+          result=`.github/scripts/analytics/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch "${BRANCH_NAME}"`
 
           if [ ${{ inputs.testman_token }} ]; then
             # finish testme session
@@ -573,7 +589,7 @@ runs:
             echo file ${file_to_check} NOT changed
         else
             echo file ${file_to_check} changed
-            .github/scripts/tests/get_muted_tests.py --output_folder "$PUBLIC_DIR/mute_info/" get_mute_diff --base_sha $ORIGINAL_HEAD~1 --head_sha $ORIGINAL_HEAD --job-id "${{ github.run_id }}" --branch "${GITHUB_REF_NAME}"
+            .github/scripts/tests/get_muted_tests.py --output_folder "$PUBLIC_DIR/mute_info/" get_mute_diff --base_sha $ORIGINAL_HEAD~1 --head_sha $ORIGINAL_HEAD --job-id "${{ github.run_id }}" --branch "${BRANCH_NAME}"
             FILE_PATH=$PUBLIC_DIR/mute_info/2_new_muted_tests.txt
             SEPARATOR=""
             if [ -f "$FILE_PATH" ]; then
@@ -633,7 +649,7 @@ runs:
       run: |
         set -x
         export build_preset="${{ inputs.build_preset }}"
-        export branch_to_compare="$GITHUB_REF_NAME"
+        export branch_to_compare="$BRANCH_NAME"
         export yellow_treshold=102400
         export red_treshold=2097152
         export commit_git_sha="$(git rev-parse HEAD)"

.github/actions/update_changelog/update_changelog.py

@@ -57,7 +57,7 @@ def to_file(changelog_path, changelog):
             for category, items in changelog[UNRELEASED].items():
                 if(len(changelog[UNRELEASED][category]) == 0):
                     continue
-                file.write(f"{CATEGORY_PREFIX}{category}\n")
+                file.write(f"{CATEGORY_PREFIX}{category}\n\n")
                 for id, body in items.items():
                     file.write(f"{ITEM_PREFIX}{id}:{body.strip()}\n")
                 file.write("\n")
@@ -69,7 +69,7 @@ def to_file(changelog_path, changelog):
             for category, items in categories.items():
                 if(len(changelog[version][category]) == 0):
                     continue
-                file.write(f"{CATEGORY_PREFIX}{category}\n")
+                file.write(f"{CATEGORY_PREFIX}{category}\n\n")
                 for id, body in items.items():
                     file.write(f"{ITEM_PREFIX}{id}:{body.strip()}\n")
                 file.write("\n")
@@ -83,7 +83,7 @@ def extract_changelog_category(description):
     return None
 
 def extract_pr_number(changelog_entry):
-    match = re.search(r"#(\d+)", changelog_entry)
+    match = re.search(r"\* (\d+)", changelog_entry)
     if match:
         return int(match.group(1))
     return None
@@ -136,14 +136,17 @@ def update_changelog(changelog_path, pr_data):
     to_file(changelog_path, changelog)
 
 def run_command(command):
+    print(f"Executing command: {command}")
     try:
         result = subprocess.run(command, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output = result.stdout.decode().strip()
+        print(f"Command output: {output}")
+        return output
     except subprocess.CalledProcessError as e:
         print(f"::error::Command failed with exit code {e.returncode}: {e.stderr.decode()}")
         print(f"::error::Command: {e.cmd}")
         print(f"::error::Output: {e.stdout.decode()}")
         sys.exit(1)
-    return result.stdout.decode().strip()
 
 def branch_exists(branch_name):
     result = subprocess.run(["git", "ls-remote", "--heads", "origin", branch_name], capture_output=True, text=True)
@@ -169,7 +172,7 @@ def fetch_user_details(username):
     response.raise_for_status()
     return response.json()
 
-if __name__ == "__main__":
+def main():
     if len(sys.argv) != 5:
         print("Usage: update_changelog.py <pr_data_file> <changelog_path> <base_branch> <suffix>")
         sys.exit(1)
@@ -178,6 +181,13 @@ def fetch_user_details(username):
     changelog_path = sys.argv[2]
     base_branch = sys.argv[3]
     suffix = sys.argv[4]
+
+    # Fetch latest changes
+    run_command("git fetch origin")
+    # Check out existing branch
+    run_command(f"git checkout {base_branch}")
+    # Make sure it's up to date
+    run_command(f"git pull origin {base_branch}")
 
     try:
         with open(pr_data_file, 'r') as file:
@@ -187,6 +197,10 @@ def fetch_user_details(username):
         sys.exit(1)
 
     pr_data = []
+    if not pr_ids:
+        print("::notice::No PR IDs found in the provided file.")
+        return 
+
     for pr in pr_ids:
         try:
             pr_details = fetch_pr_details(pr["id"])
@@ -203,7 +217,7 @@ def fetch_user_details(username):
         except Exception as e:
             print(f"::error::Failed to fetch PR details for PR #{pr['id']}: {e}")
             sys.exit(1)
-
+   
     update_changelog(changelog_path, pr_data)
 
     base_branch_name = f"changelog/{base_branch}-{suffix}"
@@ -212,13 +226,18 @@ def fetch_user_details(username):
     while branch_exists(branch_name):
         branch_name = f"{base_branch_name}-{index}"
         index += 1
+
     run_command(f"git checkout -b {branch_name}")
     run_command(f"git add {changelog_path}")
     run_command(f"git commit -m \"Update CHANGELOG.md for {suffix}\"")
     run_command(f"git push origin {branch_name}")
 
-    pr_title = f"Update CHANGELOG.md for {suffix}"
+    pr_title = f"Update CHANGELOG.md for {base_branch}:{suffix}"
     pr_body = f"This PR updates the CHANGELOG.md file for {suffix}."
     pr_create_command = f"gh pr create --title \"{pr_title}\" --body \"{pr_body}\" --base {base_branch} --head {branch_name}"
     pr_url = run_command(pr_create_command)
     # run_command(f"gh pr edit {pr_url} --add-assignee galnat") # TODO: Make assignee customizable
+
+
+if __name__ == "__main__":
+    main()

.github/config/muted_ya.txt

@@ -16,7 +16,12 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadHuge
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.ReadSmall
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteHuge
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall
+ydb/core/kqp/ut/batch_operations KqpBatchUpdate.Large_3
+ydb/core/kqp/ut/batch_operations [*/*] chunk chunk
 ydb/core/kqp/ut/cost KqpCost.OlapWriteRow
+ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.ExecuteScriptWithLargeFile
+ydb/core/kqp/ut/federated_query/s3 KqpFederatedQuery.ExecuteScriptWithThinFile
+ydb/core/kqp/ut/federated_query/s3 sole chunk chunk
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
 ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
 ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesInBS
@@ -30,29 +35,21 @@ ydb/core/kqp/ut/olap KqpOlapWrite.TierDraftsGCWithRestart
 ydb/core/kqp/ut/olap [*/*] chunk chunk
 ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable+ColumnStore
 ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable-ColumnStore
-ydb/core/kqp/ut/query KqpLimits.TooBigColumn+useSink
 ydb/core/kqp/ut/query KqpStats.SysViewClientLost
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
 ydb/core/kqp/ut/scheme [*/*] chunk chunk
 ydb/core/kqp/ut/service [*/*] chunk chunk
 ydb/core/kqp/ut/tx KqpSinkTx.OlapInvalidateOnError
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictReadWriteOlap
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictReadWriteOltp
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictReadWriteOltpNoSink
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictWriteOlap
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictWriteOltp
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TConflictWriteOltpNoSink
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TReadOnlyOltp
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TReadOnlyOltpNoSink
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TSimpleOltp
-ydb/core/kqp/ut/tx KqpSnapshotIsolation.TSimpleOltpNoSink
 ydb/core/kqp/ut/yql KqpScripting.StreamExecuteYqlScriptScanOperationTmeoutBruteForce
 ydb/core/mind/hive/ut TStorageBalanceTest.TestScenario2
+ydb/core/persqueue/ut/ut_with_sdk TopicAutoscaling.PartitionSplit_DistributedTxCommit_CheckOffsetCommitForDifferentCases_SplitedTopic
+ydb/core/quoter/ut QuoterWithKesusTest.KesusRecreation
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/statistics/aggregator/ut AnalyzeColumnshard.AnalyzeRebootColumnShard
-ydb/core/tablet_flat/ut DataCleanup.CleanupDataWithFollowers
 ydb/core/tx/datashard/ut_incremental_backup IncrementalBackup.ComplexRestoreBackupCollection+WithIncremental
+ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.CancelShouldSucceedOnManyTables
+ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.ShouldSucceedOnManyTables
 ydb/core/tx/schemeshard/ut_login_large TSchemeShardLoginLargeTest.RemoveLogin_Many
 ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithData
 ydb/core/tx/schemeshard/ut_move_reboots TSchemeShardMoveRebootsTest.WithDataAndPersistentPartitionStats
@@ -62,7 +59,6 @@ ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll
 ydb/library/actors/interconnect/ut_huge_cluster sole chunk chunk
 ydb/library/yaml_config/ut_transform test_transform.py.TestYamlConfigTransformations.test_basic[args1-dump_ds_init]
 ydb/library/yql/dq/opt/ut sole chunk chunk
-ydb/public/sdk/cpp/src/client/topic/ut [*/*] chunk chunk
 ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync1
 ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync10
 ydb/services/ydb/sdk_sessions_ut YdbSdkSessions.TestSdkFreeSessionAfterBadSessionQueryService
@@ -84,8 +80,6 @@ ydb/tests/fq/yds test_select_limit_db_id.py.TestSelectLimitWithDbId.test_select_
 ydb/tests/fq/yds test_yds_bindings.py.TestBindings.test_yds_insert[v1]
 ydb/tests/functional/compatibility test_compatibility.py.TestCompatibility.test_simple
 ydb/tests/functional/compatibility test_followers.py.TestFollowersCompatibility.test_followers_compatability
-ydb/tests/functional/compatibility test_stress.py.TestStress.test_kv[last_stable-row]
-ydb/tests/functional/compatibility test_stress.py.TestStress.test_kv[mixed-row]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[last_stable-row]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_log[mixed-row]
 ydb/tests/functional/compatibility test_stress.py.TestStress.test_tpch1[current-row]
@@ -94,18 +88,11 @@ ydb/tests/functional/compatibility test_stress.py.TestStress.test_tpch1[mixed-ro
 ydb/tests/functional/hive test_drain.py.TestHive.test_drain_on_stop
 ydb/tests/functional/rename [test_rename.py */*] chunk chunk
 ydb/tests/functional/serializable sole chunk chunk
-ydb/tests/functional/serializable test.py.test_local
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--false]
 ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--true]
 ydb/tests/functional/suite_tests [test_sql_logic.py */*] chunk chunk
 ydb/tests/functional/suite_tests test_postgres.py.TestPGSQL.test_sql_suite[plan-jointest/join2.test]
-ydb/tests/functional/suite_tests test_postgres.py.TestPGSQL.test_sql_suite[plan-select.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[plan-select1-1.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[plan-select2-1.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[plan-select2-3.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[plan-select2-4.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[plan-select3-9.test]
-ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[results-select3-13.test]
+ydb/tests/functional/suite_tests test_sql_logic.py.TestSQLLogic.test_sql_suite[results-select2-1.test]
 ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_the_same_tenant2[enable_alter_database_create_hive_first--false]
 ydb/tests/functional/tenants test_dynamic_tenants.py.test_create_and_drop_the_same_tenant2[enable_alter_database_create_hive_first--true]
 ydb/tests/functional/tenants test_tenants.py.TestTenants.test_create_drop_create_table3[enable_alter_database_create_hive_first--false]
@@ -119,16 +106,17 @@ ydb/tests/functional/tpc/large sole chunk chunk
 ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[10]
 ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[11]
 ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[12]
-ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[36]
-ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[67]
-ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[86]
 ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[9]
-ydb/tests/functional/tpc/large test_tpch_spilling.py.TestTpchSpillingS10.test_tpch[7]
 ydb/tests/olap sole chunk chunk
+ydb/tests/olap test_log_scenario.py.TestLogScenario.test
+ydb/tests/olap test_log_scenario.py.TestLogScenario.test[1051200]
+ydb/tests/olap test_log_scenario.py.TestLogScenario.test[180]
 ydb/tests/olap test_quota_exhaustion.py.TestYdbWorkload.test_delete
-ydb/tests/olap/data_quotas test_quota_exhaustion.py.TestYdbWorkload.test_duplicates
 ydb/tests/olap/column_family/compression alter_compression.py.TestAlterCompression.test_all_supported_compression
 ydb/tests/olap/column_family/compression sole chunk chunk
+ydb/tests/olap/data_quotas [*/*] chunk chunk
+ydb/tests/olap/data_quotas test_quota_exhaustion.py.TestYdbWorkload.test
+ydb/tests/olap/data_quotas test_quota_exhaustion.py.TestYdbWorkload.test_duplicates
 ydb/tests/olap/oom overlapping_portions.py.TestOverlappingPortions.test
 ydb/tests/olap/scenario sole chunk chunk
 ydb/tests/olap/scenario test_alter_compression.py.TestAlterCompression.test[alter_compression]
@@ -139,7 +127,6 @@ ydb/tests/olap/ttl_tiering [data_migration_when_alter_ttl.py] chunk chunk
 ydb/tests/olap/ttl_tiering [ttl_delete_s3.py] chunk chunk
 ydb/tests/olap/ttl_tiering data_migration_when_alter_ttl.py.TestDataMigrationWhenAlterTtl.test
 ydb/tests/olap/ttl_tiering sole chunk chunk
-ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_data_unchanged_after_ttl_change
 ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_delete_s3_tiering
 ydb/tests/olap/ttl_tiering ttl_delete_s3.py.TestDeleteS3Ttl.test_ttl_delete
 ydb/tests/postgres_integrations/go-libpq [docker_wrapper_test.py] chunk chunk
@@ -225,4 +212,5 @@ ydb/tests/sql/large test_workload_manager.py.TestWorkloadManager.test_resource_p
 ydb/tests/sql/large test_workload_manager.py.TestWorkloadManager.test_resource_pool_queue_size_limit
 ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[column]
 ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[row]
+ydb/tests/stress/mixedpy test_mixed.py.TestYdbMixedWorkload.test[column]
 ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite