resolve conflict

Author: Mrart

.dlc.json

@@ -10,7 +10,7 @@
       "pattern": "^{"
     },
     {
-      "pattern": "^https://repo1.maven.org/maven2/org/apache/flink.*SNAPSHOT.*"
+      "pattern": "^https://repo1.maven.org/maven2/org/apache/flink.*"
     },
     {
       "pattern": "^https://mvnrepository.com"

.github/labeler.yml

@@ -39,10 +39,13 @@ e2e-tests:
   - flink-cdc-e2e-tests/**/*
 migration-tests:
   - flink-cdc-migration-tests/**/*
+add-ons:
+  - flink-cdc-pipeline-model/**/*
+  - flink-cdc-pipeline-udf-examples/**/*
 base:
   - flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/**/*
 debezium:
-  - flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-debezium/**/*
+  - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/**/*
 connector-test-util:
   - flink-cdc-connect/flink-cdc-source-connectors/flink-connector-test-util/**/*
 db2-cdc-connector:
@@ -86,3 +89,11 @@ starrocks-pipeline-connector:
   - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/**/*
 elasticsearch-pipeline-connector:
   - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch/**/*
+oceanbase-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase/**/*
+maxcompute-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute/**/*
+iceberg-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/**/*
+postgres-pipeline-connector:
+  - flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/**/*

.github/workflows/build_docs.yml

@@ -54,8 +54,8 @@ jobs:
       matrix:
         branch:
           - master
-          - release-3.2
-          - release-3.1
+          - release-3.4
+          - release-3.3
 
     steps:
       - uses: actions/checkout@v3
@@ -69,8 +69,8 @@ jobs:
           echo "flink_branch=${currentBranch}" >> ${GITHUB_ENV}
           
           if [ "${currentBranch}" = "master" ]; then
-          echo "flink_alias=release-3.3" >> ${GITHUB_ENV}
-          elif [ "${currentBranch}" = "release-3.2" ]; then
+          echo "flink_alias=release-3.5" >> ${GITHUB_ENV}
+          elif [ "${currentBranch}" = "release-3.4" ]; then
           echo "flink_alias=stable" >> ${GITHUB_ENV}
           fi
 
@@ -84,7 +84,7 @@ jobs:
           docker run --rm --volume "$PWD:/root/flink-cdc" chesnay/flink-ci:java_8_11_17_21_maven_386 bash -c "cd /root/flink-cdc && chmod +x ./.github/workflows/docs.sh && ./.github/workflows/docs.sh"
 
       - name: Upload documentation
-        uses: burnett01/rsync-deployments@5.2
+        uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
         with:
           switches: --archive --compress --delete
           path: docs/target/
@@ -96,7 +96,7 @@ jobs:
 
       - name: Upload documentation alias
         if: env.flink_alias != ''
-        uses: burnett01/rsync-deployments@5.2
+        uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
         with:
           switches: --archive --compress --delete
           path: docs/target/

.github/workflows/close_stale.yml

@@ -41,16 +41,19 @@ jobs:
           close-issue-message: >
             This issue has been closed because Flink CDC doesn't use GitHub issue trackers.
           # Stale PRs
-          days-before-pr-stale: 60
-          days-before-pr-close: 30
+          days-before-pr-stale: 120
+          days-before-pr-close: 60
           stale-pr-message: >
             This pull request has been automatically marked as stale because it has not had recent
-            activity for 60 days. It will be closed in 30 days if no further activity occurs.
+            activity for 120 days. It will be closed in 60 days if no further activity occurs.
           close-pr-message: >
             This pull request has been closed because it has not had recent activity. You could reopen it
             if you try to continue your work, and anyone who are interested in it are encouraged to continue
             work on this pull request.
           # Auto-remove stale has conflicts with existing approve label workflows
           # and will result in Stale label missing.
           remove-pr-stale-when-updated: false
-          remove-issue-stale-when-updated: false
+          remove-issue-stale-when-updated: false
+          # Allow pinning issues and PRs that is not meant to be closed automatically
+          exempt-issue-labels: "pinned"
+          exempt-pr-labels: "pinned"

.github/workflows/flink_cdc_base.yml

@@ -18,19 +18,25 @@ name: Flink CDC Base Workflow
 on:
   workflow_call:
     inputs:
-      java-version:
-        description: "Jdk version to test against."
+      java-versions:
+        description: "Jdk versions to test against."
         required: false
         type: string
-        default: "8"
-      flink-version:
-        description: "Flink version to test against."
+        default: "['8']"
+      flink-versions:
+        description: "Flink versions to test against."
         required: false
         type: string
-      module:
-        description: "Flink CDC module to test against."
+        default: "['generic']"
+      modules:
+        description: "Flink CDC modules to test against."
         required: true
         type: string
+      parallelism:
+        description: "Flink parallelism."
+        required: false
+        type: number
+        default: 4
       custom-maven-parameter:
         description: "Custom maven parameter."
         required: false
@@ -41,25 +47,39 @@ env:
   flink-cdc-cli,\
   flink-cdc-common,\
   flink-cdc-composer,\
-  flink-cdc-runtime"
+  flink-cdc-runtime,\
+  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
   MODULES_PIPELINE_CONNECTORS: "\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
-  MODULES_MYSQL: "\
+  MODULES_MYSQL_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
 
-  MODULES_POSTGRES: "\
+  MODULES_MYSQL_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
+
+  MODULES_POSTGRES_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
 
+  MODULES_POSTGRES_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres"
+
   MODULES_ORACLE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
@@ -76,10 +96,13 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
 
-  MODULES_OCEANBASE: "\
+  MODULES_OCEANBASE_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
 
+  MODULES_OCEANBASE_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
+
   MODULES_DB2: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
@@ -88,19 +111,48 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
 
+  MODULES_DORIS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
+
+  MODULES_STARROCKS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
+
+  MODULES_ICEBERG: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
+
+  MODULES_KAFKA: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
+
+  MODULES_PAIMON: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
+
+  MODULES_ELASTICSEARCH: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+
+  MODULES_MAXCOMPUTE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+
+  MODULES_FLUSS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss"
+
   MODULES_PIPELINE_E2E: "\
   flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
 
   MODULES_SOURCE_E2E: "\
   flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
 
 jobs:
-  compile_and_test:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        java-version: ${{ fromJSON(inputs.java-versions) }}
+        flink-version: ${{ fromJSON(inputs.flink-versions) }}
+        module: ${{ fromJSON(inputs.modules) }}
     steps:
-      - run: echo "Running CI pipeline for JDK version ${{ inputs.java-version }}"
-
+      - run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
       - name: Clean up disk space
         run: |
           set -euo pipefail
@@ -126,7 +178,7 @@ jobs:
       - name: Set JDK
         uses: actions/setup-java@v4
         with:
-          java-version: ${{ inputs.java-version }}
+          java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
           cache: 'maven'
 
@@ -136,22 +188,28 @@ jobs:
           maven-version: 3.8.6
 
       - name: Compile and test
-        timeout-minutes: 60
+        timeout-minutes: 90
         run: |
+          . .github/workflows/utils.sh
+          jvm_timezone=$(random_timezone)
+          echo "JVM timezone is set to $jvm_timezone"
           set -o pipefail
 
-          case ${{ inputs.module }} in
+          case ${{ matrix.module }} in
               ("core")
                modules=${{ env.MODULES_CORE }}
               ;;
-              ("pipeline_connectors")
-               modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
+              ("mysql-source")
+               modules=${{ env.MODULES_MYSQL_SOURCE }}
               ;;
-              ("mysql")
-               modules=${{ env.MODULES_MYSQL }}
+              ("mysql-pipeline")
+               modules=${{ env.MODULES_MYSQL_PIPELINE }}
               ;;
-              ("postgres")
-               modules=${{ env.MODULES_POSTGRES }}
+              ("postgres-source")
+               modules=${{ env.MODULES_POSTGRES_SOURCE }}
+              ;;
+              ("postgres-pipeline")
+               modules=${{ env.MODULES_POSTGRES_PIPELINE }}
               ;;
               ("oracle")
                modules=${{ env.MODULES_ORACLE }}
@@ -168,44 +226,71 @@ jobs:
               ("tidb")
                modules=${{ env.MODULES_TIDB }}
               ;;
-              ("oceanbase")
-               modules=${{ env.MODULES_OCEANBASE }}
+              ("oceanbase-source")
+               modules=${{ env.MODULES_OCEANBASE_SOURCE }}
+              ;;
+              ("oceanbase-pipeline")
+               modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
               ;;
               ("db2")
                modules=${{ env.MODULES_DB2 }}
               ;;
               ("vitess")
                modules=${{ env.MODULES_VITESS }}
               ;;
+              ("doris")
+               modules=${{ env.MODULES_DORIS }}
+              ;;
+              ("elasticsearch")
+               modules=${{ env.MODULES_ELASTICSEARCH }}
+              ;;
+              ("iceberg")
+               modules=${{ env.MODULES_ICEBERG }}
+              ;;
+              ("kafka")
+               modules=${{ env.MODULES_KAFKA }}
+              ;;
+              ("maxcompute")
+               modules=${{ env.MODULES_MAXCOMPUTE }}
+              ;;
+              ("paimon")
+               modules=${{ env.MODULES_PAIMON }}
+              ;;
+              ("starrocks")
+               modules=${{ env.MODULES_STARROCKS }}
+              ;;
+              ("fluss")
+               modules=${{ env.MODULES_FLUSS }}
+              ;;
               ("pipeline_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
                modules=${{ env.MODULES_PIPELINE_E2E }}
               ;;
               ("source_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
                modules=${{ env.MODULES_SOURCE_E2E }}
               ;;
           esac
           
-          if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
+          if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
             compile_modules=$modules
           fi
           
           build_maven_parameter=""
 
-          if [ ${{ inputs.module }} == "mongodb6" ]; then
+          if [ ${{ matrix.module }} == "mongodb6" ]; then
             build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
-          elif [ ${{ inputs.module }} == "mongodb7" ]; then
+          elif [ ${{ matrix.module }} == "mongodb7" ]; then
             build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
           fi
           
-          if [ ! -z "${{ inputs.flink-version }}" ]; then
-            build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ inputs.flink-version }}"
+          if [ ! -z "${{ matrix.flink-version }}" ]; then
+            build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ matrix.flink-version }}"
           fi
           
           build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
           
-          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules verify
+          mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} -Duser.timezone=$jvm_timezone verify
 
       - name: Print JVM thread dumps when cancelled
         if: ${{ failure() }}