From 346abd4104b2cda52ae1b5c397c4b4a15b60a781 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 6 Jun 2023 00:07:40 +0800 Subject: [PATCH] chore(pingcap/tidb): add a new ut canary job (#2119) Signed-off-by: wuhuizuo --- .../tidb/latest/canary_ghpr_unit_test.groovy | 38 ++++++ .../tidb/latest/canary-ghpr_unit_test.groovy | 48 ++++++++ .../latest/pod-canary-ghpr_unit_test.yaml | 110 ++++++++++++++++++ prow-jobs/pingcap-tidb-latest-presubmits.yaml | 8 ++ 4 files changed, 204 insertions(+) create mode 100644 jobs/pingcap/tidb/latest/canary_ghpr_unit_test.groovy create mode 100644 pipelines/pingcap/tidb/latest/canary-ghpr_unit_test.groovy create mode 100644 pipelines/pingcap/tidb/latest/pod-canary-ghpr_unit_test.yaml diff --git a/jobs/pingcap/tidb/latest/canary_ghpr_unit_test.groovy b/jobs/pingcap/tidb/latest/canary_ghpr_unit_test.groovy new file mode 100644 index 0000000000..e2c0ed0e58 --- /dev/null +++ b/jobs/pingcap/tidb/latest/canary_ghpr_unit_test.groovy @@ -0,0 +1,38 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('pingcap/tidb/canary_ghpr_unit_test') { + logRotator { + daysToKeep(30) + } + parameters { + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC", "", "Prow job spec struct data") + } + properties { + // priority(0) // 0 fast than 1 + githubProjectUrl("https://github.com/pingcap/tidb") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/pingcap/tidb/latest/canary-ghpr_unit_test.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} diff --git a/pipelines/pingcap/tidb/latest/canary-ghpr_unit_test.groovy b/pipelines/pingcap/tidb/latest/canary-ghpr_unit_test.groovy new file mode 100644 index 0000000000..a657dfc832 --- /dev/null +++ b/pipelines/pingcap/tidb/latest/canary-ghpr_unit_test.groovy @@ -0,0 +1,48 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-tidb" +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final GIT_FULL_REPO_NAME = 'pingcap/tidb' +final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/latest/pod-canary-ghpr_unit_test.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 90, unit: 'MINUTES') + } + stages { + stage('Checkout') { + steps { + dir(REFS.repo) { + cache(path: "./", filter: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + script { + git.setSshKey(GIT_CREDENTIALS_ID) + retry(2) { + prow.checkoutRefs(REFS, timeout = 5, credentialsId = '', gitBaseUrl = 'https://github.com', withSubmodule=true) + } + } + } + } + } + } + stage('Test') { + steps { + dir(REFS.repo) { + sh 'make bazel_coverage_test' + } + } + } + } +} diff --git a/pipelines/pingcap/tidb/latest/pod-canary-ghpr_unit_test.yaml b/pipelines/pingcap/tidb/latest/pod-canary-ghpr_unit_test.yaml new file mode 100644 index 0000000000..0629fc2e10 --- /dev/null +++ b/pipelines/pingcap/tidb/latest/pod-canary-ghpr_unit_test.yaml @@ -0,0 +1,110 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + # TODO(wuhuizuo): using standard bazel build image to shrink the image size + # and keep image simple,so no need to refresh image to update basic bazel out data. + image: hub.pingcap.net/ee/ci/base:v20230530-go1.20.4 + securityContext: + privileged: true + tty: true + resources: + limits: + memory: 128Gi + cpu: "32" + env: + - name: GOPATH + value: /share/.go + - name: GOCACHE + value: /share/.cache/go-build + volumeMounts: + - name: gocache + mountPath: /share/.cache/go-build + - name: gopathcache + mountPath: /share/.go + - name: bazel-rc + mountPath: /data/ + readOnly: true + - name: containerinfo + mountPath: /etc/containerinfo + lifecycle: + postStart: + exec: + command: + - /bin/sh + - /data/bazel-prepare-in-container.sh + - name: deno + image: "denoland/deno:1.31.2" + command: [sh] + tty: true + env: + - name: DENO_DIR + value: /cache/deno + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumeMounts: + - mountPath: /cache/deno + name: deno-dir + - name: net-tool + image: wbitt/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + - name: report + image: hub.pingcap.net/jenkins/python3-requests:latest + tty: true + resources: + limits: + memory: 256Mi + cpu: 100m + volumes: + - name: deno-dir + persistentVolumeClaim: + claimName: deno-dir + - name: gopathcache + persistentVolumeClaim: + claimName: gopathcache + - name: gocache + persistentVolumeClaim: + claimName: gocache + - name: bazel-rc + secret: + secretName: bazel + - name: containerinfo + downwardAPI: + items: + - path: cpu_limit + resourceFieldRef: + containerName: golang + resource: limits.cpu + - path: cpu_request + resourceFieldRef: + containerName: golang + resource: requests.cpu + - path: mem_limit + resourceFieldRef: + containerName: golang + resource: limits.memory + - path: mem_request + resourceFieldRef: + containerName: golang + resource: requests.memory + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/prow-jobs/pingcap-tidb-latest-presubmits.yaml b/prow-jobs/pingcap-tidb-latest-presubmits.yaml index 51d5d6c43f..9e7ecbe5a8 100644 --- a/prow-jobs/pingcap-tidb-latest-presubmits.yaml +++ b/prow-jobs/pingcap-tidb-latest-presubmits.yaml @@ -60,6 +60,14 @@ presubmits: rerun_command: "/test canary-scan-security" branches: - ^master$ + - name: pingcap/tidb/canary_ghpr_unit_test + agent: jenkins + decorate: false # need add this. + always_run: false + context: canary-unit-test + branches: + - ^master$ + - ^feature[_/].+ - name: pingcap/tidb/pull_integration_mysql_test agent: jenkins decorate: false # need add this.