From 1bcc470948eabd7c7e02982d25863fa6975925ed Mon Sep 17 00:00:00 2001 From: purelind Date: Mon, 20 Jan 2025 17:14:31 +0800 Subject: [PATCH 1/8] fix: checkout to minor version of tidb-test for hotfix Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index bd3a62759..61eacbf2b 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -1,6 +1,7 @@ // compute component branch from pr info. def computeBranchFromPR(String component, String prTargetBranch, String prTitle, String trunkBranch="master") { // pr title xxx | dep1=release-x.y + println("computeBranchFromPR component: ${component}, prTargetBranch: ${prTargetBranch}, prTitle: ${prTitle}, trunkBranch: ${trunkBranch}") final componentParamReg = /\b${component}\s*=\s*([^\s\\]+)(\s|\\|$)/ // - release-6.2 @@ -9,7 +10,7 @@ def computeBranchFromPR(String component, String prTargetBranch, String prTitle, // - feature_release-8.1-abcdefg final wipReleaseFeatureBranchReg = /^feature[\/_]release\-(\d+\.\d+)-.+/ // - release-6.2-20220801 - final oldHotfixBranchReg = /^release\-(\d+\.\d+)-.+/ + final oldHotfixBranchReg = /^release\-(\d+\.\d+)-(\d+)$/ // - release-6.1-20230101-v6.1.2 final newHotfixBranchReg = /^release\-\d+\.\d+\-\d+\-v((\d+\.\d+)\.\d+)/ @@ -130,25 +131,36 @@ def checkoutV2(gitUrl, component, prTargetBranch, prTitle, credentialsId="", tru def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, credentialsId="", trunkBranch="master", timeout=5) { def tidbTestRefs = [] // List of tidb-test refs PR:123, PR:456 boolean branchOrCommitSpecified = false // Flag to check if a branch or commit is specified in any PR title + // pr title xxx | dep1=release-x.y + final componentParamReg = /\b${component}\s*=\s*([^\s\\]+)(\s|\\|$)/ + def componentBranch = prTargetBranch // compute the branch. refs.pulls.each { pull -> - def componentBranch = computeBranchFromPR(component, prTargetBranch, pull.title, trunkBranch) + componentBranch = computeBranchFromPR(component, prTargetBranch, pull.title, trunkBranch) if (componentBranch.startsWith("pr/")) { tidbTestRefs.add("PR:${componentBranch}") // Add as PR reference } else { // some PR title contains a branch or commit - if ( prTargetBranch != componentBranch) { + if (prTitle =~ componentParamReg) { + // example PR tiltes: + // - feat: add new faeture | tidb=release-8.1 + // - feat: add new faeture | tidb= + componentBranch = (prTitle =~ componentParamReg)[0][1] branchOrCommitSpecified = true tidbTestRefs.add("Branch:${componentBranch}") // Add as branch reference specified in PR title + echo "current pr target branch ${prTargetBranch}, will checkout the specific branch ${componentBranch} of ${component} from PR title" + } else if (prTargetBranch != componentBranch) { + // current pr target branch a specifical branch, then need use the specific branch to checkout + echo "May be a hotfix branch or feature branch, current pr target branch ${prTargetBranch}, the component branch is ${componentBranch}" } } } // pre-merge for the PRs. if (tidbTestRefs.isEmpty()) { - echo "No tidb-test refs specified, defaulting to base branch ${prTargetBranch} of tidb-test." - checkoutSingle(gitUrl, prTargetBranch, prTargetBranch, credentialsId) + echo "No tidb-test refs specified in PR title, checkout the base branch ${componentBranch} of ${component}." + checkoutSingle(gitUrl, componentBranch, componentBranch, credentialsId) } else if (tidbTestRefs.size() == 1 && tidbTestRefs[0].startsWith("Branch:")) { // default branch or specific branch // Single PR with branch specified From 98897597478de7cf6f7afa94f2eea6419afe111f Mon Sep 17 00:00:00 2001 From: purelind Date: Mon, 20 Jan 2025 18:26:49 +0800 Subject: [PATCH 2/8] chore: make detecting branches of tidb-test simple Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 24 ++++++++++------------ 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index 61eacbf2b..a2b20220b 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -141,18 +141,15 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede if (componentBranch.startsWith("pr/")) { tidbTestRefs.add("PR:${componentBranch}") // Add as PR reference } else { - // some PR title contains a branch or commit - if (prTitle =~ componentParamReg) { - // example PR tiltes: - // - feat: add new faeture | tidb=release-8.1 - // - feat: add new faeture | tidb= - componentBranch = (prTitle =~ componentParamReg)[0][1] + // 1. some PR title contains a branch or commit + // 2. hotfix branch or feature branch + if (prTargetBranch != componentBranch) { + // TODO: may be a feature branch or hotfix branch, not specify the branch in PR title + // need more clearly handle the logic here. + // for hotfix branch batch merge, will encounter the error, incorrectly assumed that + // branches were specified in multiple PR titles. branchOrCommitSpecified = true - tidbTestRefs.add("Branch:${componentBranch}") // Add as branch reference specified in PR title - echo "current pr target branch ${prTargetBranch}, will checkout the specific branch ${componentBranch} of ${component} from PR title" - } else if (prTargetBranch != componentBranch) { - // current pr target branch a specifical branch, then need use the specific branch to checkout - echo "May be a hotfix branch or feature branch, current pr target branch ${prTargetBranch}, the component branch is ${componentBranch}" + tidbTestRefs.add("Branch:${componentBranch}") } } } @@ -162,10 +159,11 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede echo "No tidb-test refs specified in PR title, checkout the base branch ${componentBranch} of ${component}." checkoutSingle(gitUrl, componentBranch, componentBranch, credentialsId) } else if (tidbTestRefs.size() == 1 && tidbTestRefs[0].startsWith("Branch:")) { - // default branch or specific branch - // Single PR with branch specified + // 1. feature branch or hotfix branch + // 2. Single PR with branch or commit sha specified echo "Single PR with tidb-test branch specified: ${prTargetBranch}" def branch = tidbTestRefs[0].split(":")[1] + println("Checkout the branch: ${branch} of ${component}") checkoutSingle(gitUrl, prTargetBranch, branch, credentialsId) // if tidbTestRefs size > 1 and any of tidbTestRefs start with branch , then error and exit } else if (tidbTestRefs.size() > 1 && branchOrCommitSpecified) { From 47179b226c03bc9c04e5e15ac8da81984d9be678 Mon Sep 17 00:00:00 2001 From: purelind Date: Tue, 21 Jan 2025 11:09:23 +0800 Subject: [PATCH 3/8] feat: determine whether tidbRefs is valid in different situations Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 55 +++++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index a2b20220b..63fc54aa5 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -131,8 +131,7 @@ def checkoutV2(gitUrl, component, prTargetBranch, prTitle, credentialsId="", tru def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, credentialsId="", trunkBranch="master", timeout=5) { def tidbTestRefs = [] // List of tidb-test refs PR:123, PR:456 boolean branchOrCommitSpecified = false // Flag to check if a branch or commit is specified in any PR title - // pr title xxx | dep1=release-x.y - final componentParamReg = /\b${component}\s*=\s*([^\s\\]+)(\s|\\|$)/ + def componentBranch = prTargetBranch // compute the branch. @@ -144,15 +143,30 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede // 1. some PR title contains a branch or commit // 2. hotfix branch or feature branch if (prTargetBranch != componentBranch) { - // TODO: may be a feature branch or hotfix branch, not specify the branch in PR title - // need more clearly handle the logic here. - // for hotfix branch batch merge, will encounter the error, incorrectly assumed that - // branches were specified in multiple PR titles. - branchOrCommitSpecified = true tidbTestRefs.add("Branch:${componentBranch}") } } } + def (valid, filteredRefs) = validateAndFilterRefs(tidbTestRefs) + if (!valid) { + echo "Error: invalid ${component} refs in multiple PRs: ${filteredRefs}" + throw new Exception("Error: invalid ${component} refs in multiple PRs.") + } else { + if (filteredRefs.isEmpty()) { + echo "No tidb-test refs specified in PR title, checkout the base branch ${componentBranch} of ${component}." + checkoutSingle(gitUrl, componentBranch, componentBranch, credentialsId) + } else if (filteredRefs.size() == 1 && filteredRefs[0].startsWith("Branch:")) { + // 1. feature branch or hotfix branch + // 2. Single PR with branch or commit sha specified + def branch = filteredRefs[0].split(":")[1] + println("Checkout the branch: ${branch} of ${component}") + checkoutSingle(gitUrl, prTargetBranch, branch, credentialsId) + } else if (filteredRefs.size() > 1) { + // multi PR specified component PR (notice: for batch merge with specific branch is not supported) + // single PR with specified PR + checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs.collect { it.split(":")[1] } as List, credentialsId) + } + } // pre-merge for the PRs. if (tidbTestRefs.isEmpty()) { @@ -431,3 +445,30 @@ def ks3_download_fileserver(remote, local, credentialsId="ks3util-config"){ sh "ks3util -c \$KS3UTIL_CONF cp -f ks3://ee-fileserver/download/${remote} $local" } } + +def validateAndFilterRefs(tidbTestRefs) { + if (tidbTestRefs.isEmpty()) { + return [true, []] + } + + def prRefs = tidbTestRefs.findAll { it.startsWith("PR:") } + def branchRefs = tidbTestRefs.findAll { it.startsWith("Branch:") } + + // Check if all refs are PRs + if (prRefs.size() == tidbTestRefs.size()) { + return [true, prRefs.unique()] + } + + // Check if all refs are Branches and there's only one unique branch + if (branchRefs.size() == tidbTestRefs.size()) { + def uniqueBranches = branchRefs.unique() + if (uniqueBranches.size() > 1) { + // Multiple PR with different branches specified - not supported + return [false, uniqueBranches] + } + return [true, uniqueBranches] + } + + // Mixed refs - return false and the combined unique refs + return [false, (prRefs + branchRefs).unique()] +} From 53864794c5333233e157c260fbe73ff2da756fd0 Mon Sep 17 00:00:00 2001 From: purelind Date: Tue, 21 Jan 2025 11:15:07 +0800 Subject: [PATCH 4/8] feat: add more comments Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index 63fc54aa5..8ec4ab566 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -455,11 +455,15 @@ def validateAndFilterRefs(tidbTestRefs) { def branchRefs = tidbTestRefs.findAll { it.startsWith("Branch:") } // Check if all refs are PRs + // if all refs are PRs, need to merge the PRs with the same base branch if (prRefs.size() == tidbTestRefs.size()) { return [true, prRefs.unique()] } // Check if all refs are Branches and there's only one unique branch + // 1. for hotfix branch batch merge, valid + // 2. for feature branch batch merge, valid + // 3. for multi PR with different branches, invalid if (branchRefs.size() == tidbTestRefs.size()) { def uniqueBranches = branchRefs.unique() if (uniqueBranches.size() > 1) { @@ -469,6 +473,6 @@ def validateAndFilterRefs(tidbTestRefs) { return [true, uniqueBranches] } - // Mixed refs - return false and the combined unique refs + // Mixed refs is invalid - return false and the combined unique refs return [false, (prRefs + branchRefs).unique()] } From a7e25284c30992104b8050c0349f63bce6afb305 Mon Sep 17 00:00:00 2001 From: purelind Date: Wed, 22 Jan 2025 14:13:18 +0800 Subject: [PATCH 5/8] chore: rename tidbTestRefs Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index 8ec4ab566..f3ad16e06 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -446,17 +446,17 @@ def ks3_download_fileserver(remote, local, credentialsId="ks3util-config"){ } } -def validateAndFilterRefs(tidbTestRefs) { - if (tidbTestRefs.isEmpty()) { +def validateAndFilterRefs(componentRefs) { + if (componentRefs.isEmpty()) { return [true, []] } - def prRefs = tidbTestRefs.findAll { it.startsWith("PR:") } - def branchRefs = tidbTestRefs.findAll { it.startsWith("Branch:") } + def prRefs = componentRefs.findAll { it.startsWith("PR:") } + def branchRefs = componentRefs.findAll { it.startsWith("Branch:") } // Check if all refs are PRs // if all refs are PRs, need to merge the PRs with the same base branch - if (prRefs.size() == tidbTestRefs.size()) { + if (prRefs.size() == componentRefs.size()) { return [true, prRefs.unique()] } @@ -464,7 +464,7 @@ def validateAndFilterRefs(tidbTestRefs) { // 1. for hotfix branch batch merge, valid // 2. for feature branch batch merge, valid // 3. for multi PR with different branches, invalid - if (branchRefs.size() == tidbTestRefs.size()) { + if (branchRefs.size() == componentRefs.size()) { def uniqueBranches = branchRefs.unique() if (uniqueBranches.size() > 1) { // Multiple PR with different branches specified - not supported From c75edac244ae92cbfaf72bb2c61faaf915511a37 Mon Sep 17 00:00:00 2001 From: purelind Date: Wed, 22 Jan 2025 14:25:37 +0800 Subject: [PATCH 6/8] chore: remove Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index f3ad16e06..1f4783ea0 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -167,27 +167,6 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs.collect { it.split(":")[1] } as List, credentialsId) } } - - // pre-merge for the PRs. - if (tidbTestRefs.isEmpty()) { - echo "No tidb-test refs specified in PR title, checkout the base branch ${componentBranch} of ${component}." - checkoutSingle(gitUrl, componentBranch, componentBranch, credentialsId) - } else if (tidbTestRefs.size() == 1 && tidbTestRefs[0].startsWith("Branch:")) { - // 1. feature branch or hotfix branch - // 2. Single PR with branch or commit sha specified - echo "Single PR with tidb-test branch specified: ${prTargetBranch}" - def branch = tidbTestRefs[0].split(":")[1] - println("Checkout the branch: ${branch} of ${component}") - checkoutSingle(gitUrl, prTargetBranch, branch, credentialsId) - // if tidbTestRefs size > 1 and any of tidbTestRefs start with branch , then error and exit - } else if (tidbTestRefs.size() > 1 && branchOrCommitSpecified) { - echo "Error: Specifying a tidb-test branch is not supported for multiple tidb PRs." - throw new Exception("Error: Specifying a tidb-test branch is not supported for multiple tidb PRs batch.") - } else { - // multi PR specified PR (notice: for batch merge with specific branch is not supported) - // single PR with specified PR - checkoutPRWithPreMerge(gitUrl, prTargetBranch, tidbTestRefs.collect { it.split(":")[1] } as List, credentialsId) - } } def checkoutSingle(gitUrl, prTargetBranch, branchOrCommit, credentialsId, timeout=5) { From db7dc78137902cb4393457c72b8cb501e1bbd3aa Mon Sep 17 00:00:00 2001 From: purelind Date: Wed, 22 Jan 2025 16:10:58 +0800 Subject: [PATCH 7/8] chore: fix error Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index 1f4783ea0..2d29d7ada 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -153,7 +153,7 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede throw new Exception("Error: invalid ${component} refs in multiple PRs.") } else { if (filteredRefs.isEmpty()) { - echo "No tidb-test refs specified in PR title, checkout the base branch ${componentBranch} of ${component}." + echo "No tidb-test refs specified in PR title, checkout the default branch ${componentBranch} of ${component}." checkoutSingle(gitUrl, componentBranch, componentBranch, credentialsId) } else if (filteredRefs.size() == 1 && filteredRefs[0].startsWith("Branch:")) { // 1. feature branch or hotfix branch @@ -161,9 +161,14 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede def branch = filteredRefs[0].split(":")[1] println("Checkout the branch: ${branch} of ${component}") checkoutSingle(gitUrl, prTargetBranch, branch, credentialsId) - } else if (filteredRefs.size() > 1) { + } else if (filteredRefs.size() == 1 && filteredRefs[0].startsWith("PR:")) { + // 1. single PR with PR specified + // 2. multi PR with the same PR of tidb-test specified + def componentPr = filteredRefs[0].split(":")[1] + println("Checkout the PR: ${componentPr} of ${component}") + checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs, credentialsId) + } else { // multi PR specified component PR (notice: for batch merge with specific branch is not supported) - // single PR with specified PR checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs.collect { it.split(":")[1] } as List, credentialsId) } } From 4bb52102f6afc61b494762f8b74578b447c08cb0 Mon Sep 17 00:00:00 2001 From: purelind Date: Wed, 22 Jan 2025 16:30:08 +0800 Subject: [PATCH 8/8] chore: add some comments Signed-off-by: purelind --- libraries/tipipeline/vars/component.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/tipipeline/vars/component.groovy b/libraries/tipipeline/vars/component.groovy index 2d29d7ada..5eb74bdfd 100644 --- a/libraries/tipipeline/vars/component.groovy +++ b/libraries/tipipeline/vars/component.groovy @@ -166,9 +166,11 @@ def checkoutSupportBatch(gitUrl, component, prTargetBranch, prTitle, refs, crede // 2. multi PR with the same PR of tidb-test specified def componentPr = filteredRefs[0].split(":")[1] println("Checkout the PR: ${componentPr} of ${component}") + println("Note: When specifying tidb-test=pr/xxx in the PR title, the base branch of tidb-test pr must be the same as the base branch of tidb pr.If not, please specify tidb-test= or tidb-test=.") checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs, credentialsId) } else { // multi PR specified component PR (notice: for batch merge with specific branch is not supported) + println("Note: When specifying tidb-test=pr/xxx in the PR title, the base branch of tidb-test pr must be the same as the base branch of tidb pr.If not, please specify tidb-test= or tidb-test=.") checkoutPRWithPreMerge(gitUrl, prTargetBranch, filteredRefs.collect { it.split(":")[1] } as List, credentialsId) } }