diff --git a/concourse/pipeline/job_def.lib.yml b/concourse/pipeline/job_def.lib.yml index 0169761d..de7536c3 100644 --- a/concourse/pipeline/job_def.lib.yml +++ b/concourse/pipeline/job_def.lib.yml @@ -1,6 +1,15 @@ #@ load("base.lib.yml", "add_res_by_conf", "add_res_by_name") #@ load("@ytt:template", "template") + +#@ def inter_bin_name(base_name, release_build): +#@ if release_build: +#@ return base_name + "_rel" +#@ end +#@ +#@ return base_name +#@ end + #! Use bin_gpdb_postfix="" to use a release version of gpdb binary #! Job config for centos7 #@ def centos7_gpdb6_conf(release_build=False): @@ -8,7 +17,10 @@ res_test_image: centos7-gpdb6-image-test res_gpdb_bin: bin_gpdb6_centos7 res_gpdb_src: gpdb6_src res_plpython3_bin: bin_plpython3_gpdb6_rhel7 +res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_rhel7_intermediates", release_build) os: rhel7 +release_build: #@ release_build +release_bin: bin_greenplumpython_rhel7_release #@ end #! Job config for rhel8 @@ -17,7 +29,10 @@ res_test_image: rhel8-gpdb6-image-test res_gpdb_bin: bin_gpdb6_rhel8 res_gpdb_src: gpdb6_src res_plpython3_bin: bin_plpython3_gpdb6_rhel8 +res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_rhel8_intermediates", release_build) os: rhel8 +release_build: #@ release_build +release_bin: bin_greenplumpython_rhel8_release #@ end #! Job config for ubuntu18 @@ -26,7 +41,10 @@ res_test_image: ubuntu18-gpdb6-image-test res_gpdb_bin: bin_gpdb6_ubuntu18 res_gpdb_src: gpdb6_src res_plpython3_bin: bin_plpython3_gpdb6_ubuntu18 +res_intermediates_bin: #@ inter_bin_name("bin_greenplumpython_ubuntu18_intermediates", release_build) os: ubuntu1804 +release_build: #@ release_build +release_bin: bin_greenplumpython_ubuntu18_release #@ end #! The entry point of a pipeline. The job name must be 'entrance'. @@ -83,20 +101,22 @@ plan: #@ end #@ end -#@ def _test_task(conf): -task: #@ "test_" + conf["os"] +#@ def _test_and_build_task(conf): +task: #@ "test_and_build" + conf["os"] timeout: 2h config: platform: linux run: path: greenplumpython_src/concourse/scripts/entry.sh args: - - test + - test_and_build inputs: - name: bin_gpdb - name: gpdb_src - name: greenplumpython_src - name: bin_plpython3 + outputs: + - name: greenplumpython_artifacts image: image_test params: #@ end @@ -128,5 +148,47 @@ plan: resource: #@ conf["res_gpdb_src"] - get: bin_plpython3 resource: #@ conf["res_plpython3_bin"] -- #@ _test_task(conf) +- #@ _test_and_build_task(conf) +- put: #@ conf["res_intermediates_bin"] + params: + file: greenplumpython_artifacts/greenplumpython.tar.gz #@ end + +#! The final release job +#! 1. Push the artifacts to the release bucket +#! 2. Push the git tag +#@ def exit_release_job(param): +#@ trigger = param["trigger"] +#@ confs = param["confs"] +#@ passed_jobs = [] +#@ res_map = param["res_map"] +#@ for conf in confs: +#@ passed_jobs.append(build_test_job_name(conf)) +#@ add_res_by_name(res_map, conf["release_bin"]) +#@ end +name: exit_release +on_failure: #@ trigger["on_failure"] +on_error: #@ trigger["on_error"] +on_success: #@ trigger["on_success"] +plan: +#@ for to_get in trigger["to_get"]: +- passed: #@ passed_jobs + _: #@ template.replace(to_get) +#@ end +- in_parallel: + steps: +#@ for i, conf in enumerate(confs): + - do: + - get: #@ conf["res_intermediates_bin"] + passed: + - #@ passed_jobs[i] + params: + unpack: true + - put: #@ conf["release_bin"] + params: + file: #@ conf["res_intermediates_bin"] + "/greenplumpython_*_*.tar.gz" +#@ end +#@ for to_put in trigger["to_put_post"]: +- #@ to_put +#@ end +#@ end \ No newline at end of file diff --git a/concourse/pipeline/release.yml b/concourse/pipeline/release.yml index 165c520d..e8acc98e 100644 --- a/concourse/pipeline/release.yml +++ b/concourse/pipeline/release.yml @@ -2,7 +2,6 @@ #@ "entrance_job", #@ "build_test_job", #@ "exit_release_job", -#@ "centos6_gpdb6_conf", #@ "centos7_gpdb6_conf", #@ "rhel8_gpdb6_conf", #@ "ubuntu18_gpdb6_conf") @@ -15,7 +14,6 @@ #@ res_type_map = {} #@ trigger = release_trigger(res_map) #@ confs = [ -#@ centos6_gpdb6_conf(release_build=True), #@ centos7_gpdb6_conf(release_build=True), #@ rhel8_gpdb6_conf(release_build=True), #@ ubuntu18_gpdb6_conf(release_build=True) diff --git a/concourse/pipeline/res_def.yml b/concourse/pipeline/res_def.yml index 9980a384..941ce3d7 100644 --- a/concourse/pipeline/res_def.yml +++ b/concourse/pipeline/res_def.yml @@ -103,6 +103,72 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) regexp: server/published/gpdb6/server-rc-(.*)-ubuntu18.04_x86_64.tar.gz + +# For uploading to the dev bucket +# For uploading every build to gcs +# Dev +- name: bin_greenplumpython_rhel7_intermediates + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates/greenplumpython/greenplumpython_rhel7_gpdb6.tar.gz + +- name: bin_greenplumpython_rhel8_intermediates + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates/greenplumpython/greenplumpython_rhel8_gpdb6.tar.gz + +- name: bin_greenplumpython_ubuntu18_intermediates + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates/greenplumpython/greenplumpython_ubuntu18_gpdb6.tar.gz + +# Release +- name: bin_greenplumpython_rhel7_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/greenplumpython/greenplumpython_rhel7_gpdb6.tar.gz +- name: bin_greenplumpython_rhel8_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/greenplumpython/greenplumpython_rhel8_gpdb6.tar.gz +- name: bin_greenplumpython_ubuntu18_intermediates_rel + type: gcs + source: + bucket: gpdb-extensions-concourse-resources + json_key: ((extension/extensions-gcs-service-account-key)) + versioned_file: intermediates_release/greenplumpython/greenplumpython_ubuntu18.04_gpdb6.tar.gz + + +# For uploading to the release bucket +- name: bin_greenplumpython_rhel7_release + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: greenplumpython/released/greenplumpython-(.*).tar.gz +- name: bin_greenplumpython_rhel8_release + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: greenplumpython/released/greenplumpython-(.*).tar.gz +- name: bin_greenplumpython_ubuntu18_release + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: greenplumpython/released/greenplumpython-(.*).tar.gz + # plpython3 for gpdb6 - name: bin_plpython3_gpdb6_rhel7 type: gcs diff --git a/concourse/pipeline/trigger_def.lib.yml b/concourse/pipeline/trigger_def.lib.yml index 3a6211d7..47d37fe9 100644 --- a/concourse/pipeline/trigger_def.lib.yml +++ b/concourse/pipeline/trigger_def.lib.yml @@ -56,6 +56,27 @@ on_error: alert_type: errored #@ end +#! Release trigger. For dev pipelines. No webhook +#@ def release_trigger(res_map): +#@ add_res_by_name(res_map, "greenplumpython_commit") +#@ add_res_by_name(res_map, "greenplumpython_release") +auto_trigger: true +to_get: +- get: greenplumpython_src + resource: greenplumpython_commit +to_put_pre: #@ [] +to_put_post: +- put: greenplumpython_commit + params: + repository: greenplumpython_src + tag: greenplumpython_src/VERSION +#! To set the github commit status, https://github.com/Pix4D/cogito is a good choice. +#! Unfortunately it doesn't work with Concourse 5. +on_success: +on_failure: +on_error: +#@ end + #! Commit trigger. For dev pipelines. No webhook #@ def commit_dev_trigger(res_map): #@ add_res_by_name(res_map, "greenplumpython_commit_dev") diff --git a/concourse/scripts/entry.sh b/concourse/scripts/entry.sh index 0675abf2..b61e094d 100755 --- a/concourse/scripts/entry.sh +++ b/concourse/scripts/entry.sh @@ -161,13 +161,12 @@ setup_gpadmin_bashrc # Do the special setup with root permission for the each task, then run the real task script with # gpadmin. bashrc won't be read by 'su', it needs to be sourced explicitly. case "$1" in - test) - + test_and_build) # To make fly debug easier su gpadmin -c \ "source /home/gpadmin/.bashrc &&\ - /home/gpadmin/greenplumpython_src/concourse/scripts/test.sh" - ;; + /home/gpadmin/greenplumpython_src/concourse/scripts/test_and_build.sh" + ;; *) echo "Unknown target task $1" exit 1 diff --git a/concourse/scripts/test.sh b/concourse/scripts/test.sh old mode 100755 new mode 100644 diff --git a/concourse/scripts/test_and_build.sh b/concourse/scripts/test_and_build.sh new file mode 100755 index 00000000..a8a215c5 --- /dev/null +++ b/concourse/scripts/test_and_build.sh @@ -0,0 +1,21 @@ +#!/bin/bash -l + +set -exo pipefail + +function _main() { + # FIXME: The test db and extension creation should be handled by python code. + createdb gpadmin + psql gpadmin -c "CREATE LANGUAGE plpython3u;" + + # Run testing + pushd /home/gpadmin/greenplumpython_src + unset PYTHONPATH + unset PYTHONHOME + tox -e test_py39 + # build wheel + pip3 wheel . -w ../greenplumpython_artifacts + popd + tar -czf greenplumpython_artifacts/greenplumpython.tar.gz greenplumpython_artifacts/*.whl +} + +_main