Skip to content

Commit 41a18c5

Browse files
Jesus89Jesus89
authored
Several fixes in CI/CD (CartoDB#343)
1 parent 15d8485 commit 41a18c5

File tree

16 files changed

+110
-135
lines changed

16 files changed

+110
-135
lines changed

Diff for: ‎.github/workflows/bigquery.yml

+3-6
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ on:
1616
env:
1717
NODE_VERSION: 14
1818
PYTHON3_VERSION: 3.8.10
19+
GCLOUD_VERSION: 290.0.1
1920

2021
jobs:
2122

2223
test:
2324
runs-on: ubuntu-20.04
2425
timeout-minutes: 30
2526
env:
26-
GCLOUD_VERSION: 290.0.1
2727
BQ_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
2828
BQ_PROJECT: ${{ secrets.BQ_PROJECT_CI }}
2929
BQ_BUCKET: ${{ secrets.BQ_BUCKET_CI }}
@@ -68,7 +68,6 @@ jobs:
6868
runs-on: ubuntu-20.04
6969
timeout-minutes: 20
7070
env:
71-
GCLOUD_VERSION: 290.0.1
7271
BQ_PROJECT: ${{ secrets.BQ_PROJECT_CD }}
7372
BQ_BUCKET: ${{ secrets.BQ_BUCKET_CD }}
7473
BQ_REGION: ${{ secrets.BQ_REGION_CD }}
@@ -110,7 +109,6 @@ jobs:
110109
bucket: gs://spatialextension_os
111110
permissions: '[{"specialGroup":"allAuthenticatedUsers","role":"READER"}]'
112111
env:
113-
GCLOUD_VERSION: 290.0.1
114112
BQ_REGION: ${{ matrix.region }}
115113
BQ_PROJECT: ${{ matrix.project }}
116114
BQ_BUCKET: ${{ matrix.bucket }}
@@ -144,7 +142,6 @@ jobs:
144142
runs-on: ubuntu-20.04
145143
timeout-minutes: 10
146144
env:
147-
PACKAGE_NAME: carto-analytics-toolbox-core-bigquery
148145
PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/bigquery
149146
BQ_BUCKET: 'gs://@@BUCKET@@'
150147
steps:
@@ -154,7 +151,7 @@ jobs:
154151
uses: google-github-actions/auth@v0
155152
with:
156153
credentials_json: ${{ secrets.GCLOUD_PRODUCTION_RELEASE_SA }}
157-
project_id: ${{ env.GCLOUD_PRODUCTION_PROJECT }}
154+
project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
158155
create_credentials_file: true
159156
- name: Setup gcloud
160157
uses: google-github-actions/setup-gcloud@v0
@@ -169,5 +166,5 @@ jobs:
169166
run: |
170167
cd clouds/bigquery
171168
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
172-
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/${{ env.PACKAGE_NAME }}-latest.zip
169+
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip
173170
gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/

Diff for: ‎.github/workflows/databricks.yml

+49-76
Original file line numberDiff line numberDiff line change
@@ -2,74 +2,62 @@ name: Databricks CI/CD
22

33
on:
44
push:
5+
branches:
6+
- master
57
paths:
68
- "clouds/databricks/**"
7-
- "!clouds/databricks/**/*.md"
89
- ".github/workflows/databricks.yml"
10+
pull_request:
11+
paths:
12+
- "clouds/databricks/**"
13+
- ".github/workflows/databricks.yml"
14+
workflow_dispatch:
915

1016
env:
1117
PYTHON3_VERSION: 3.8.11
12-
GOOGLE_SECRET_MANAGER_PROJECT: cartodb-on-gcp-terraform-ci-cd
18+
GCLOUD_VERSION: 290.0.1
19+
GOOGLE_SECRET_MANAGER_PROJECT: ${{ secrets.GOOGLE_SECRET_MANAGER_PROJECT }}
1320

1421
jobs:
22+
1523
test:
1624
runs-on: ubuntu-20.04
1725
timeout-minutes: 30
1826
env:
1927
DB_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
2028
DB_CLUSTER_ID: ${{ secrets.DB_CLUSTER_ID_CI }}
21-
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME_CI }}
22-
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH_CI }}
29+
DB_HTTP_PATH: ${{ secrets.DB_HTTP_PATH_CI }}
30+
DB_HOST: ${{ secrets.DB_HOST_CI }}
2331
steps:
2432
- name: Checkout repo
2533
uses: actions/checkout@v3
26-
- id: 'auth'
27-
uses: 'google-github-actions/auth@v0'
34+
- name: Auth google
35+
uses: google-github-actions/auth@v0
2836
with:
2937
credentials_json: ${{ secrets.CARTODB_ON_GCP_TERRAFORM_CI_CD_SERVICE_ACCOUNT_KEY }}
30-
31-
- id: 'secrets'
32-
uses: 'google-github-actions/get-secretmanager-secrets@v0'
38+
- name: Get secrets
39+
id: secrets
40+
uses: google-github-actions/get-secretmanager-secrets@v0
3341
with:
3442
secrets: |-
3543
databricks-ci-token:projects/cartodb-on-gcp-terraform-ci-cd/secrets/terraform-databricks-aws-ciuser-token
3644
- name: Check diff
3745
uses: technote-space/get-diff-action@v4
38-
with:
39-
PATTERNS: '**/*.+(py|scala)'
40-
FILES: 'Makefile'
4146
- name: Setup scala
4247
uses: olafurpg/setup-scala@v11
4348
with:
4449
java-version: [email protected]
45-
- name: Setup node
46-
uses: actions/setup-node@v1
47-
with:
48-
node-version: ${{ env.NODE_VERSION }}
4950
- name: Setup python 3
5051
uses: actions/setup-python@v2
5152
with:
5253
python-version: ${{ env.PYTHON3_VERSION }}
5354
- name: Setup virtualenv
5455
run: pip install virtualenv
55-
- name: install-databricks-cli
56+
- name: Setup databricks cli
5657
uses: microsoft/[email protected]
57-
- name: Create databricks cfg file
58-
uses: "DamianReeves/[email protected]"
59-
env:
60-
DATABRICKS_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
61-
with:
62-
path: /home/runner/.databrickscfg
63-
write-mode: overwrite
64-
contents: |
65-
[DEFAULT]
66-
host = https://dbc-fbcbc113-2a61.cloud.databricks.com
67-
token = ${{ env.DATABRICKS_TOKEN }}
68-
jobs-api-version = 2.1
6958
- name: Run linter and tests
7059
env:
71-
DATABRICKS_CONFIG_FILE: /home/runner/.databrickscfg
72-
DATABRICKS_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
60+
DB_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
7361
run: |
7462
cd clouds/databricks
7563
make lint && \
@@ -83,98 +71,83 @@ jobs:
8371
runs-on: ubuntu-20.04
8472
timeout-minutes: 20
8573
env:
86-
DB_CLUSTER_ID: ${{ secrets.DB_CLUSTER_ID_CI }}
87-
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME_CI }}
88-
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH_CI }}
74+
DB_CLUSTER_ID: ${{ secrets.DB_CLUSTER_ID_CD }}
75+
DB_HTTP_PATH: ${{ secrets.DB_HTTP_PATH_CD }}
76+
DB_HOST: ${{ secrets.DB_HOST_CD }}
8977
steps:
9078
- name: Checkout repo
9179
uses: actions/checkout@v3
92-
- id: 'auth'
93-
uses: 'google-github-actions/auth@v0'
80+
- name: Auth google
81+
uses: google-github-actions/auth@v0
9482
with:
9583
credentials_json: ${{ secrets.CARTODB_ON_GCP_TERRAFORM_CI_CD_SERVICE_ACCOUNT_KEY }}
96-
- id: 'secrets'
97-
uses: 'google-github-actions/get-secretmanager-secrets@v0'
84+
- name: Get secrets
85+
id: secrets
86+
uses: google-github-actions/get-secretmanager-secrets@v0
9887
with:
9988
secrets: |-
10089
databricks-ci-token:projects/cartodb-on-gcp-terraform-ci-cd/secrets/terraform-databricks-aws-ciuser-token
10190
- name: Check diff
10291
uses: technote-space/get-diff-action@v4
103-
with:
104-
PATTERNS: '**/*.+(py|scala)'
105-
FILES: 'Makefile'
10692
- name: Setup scala
10793
uses: olafurpg/setup-scala@v11
10894
with:
10995
java-version: [email protected]
110-
- name: install-databricks-cli
111-
uses: microsoft/[email protected]
112-
- name: Create databricks cfg file
113-
uses: "DamianReeves/[email protected]"
114-
env:
115-
DATABRICKS_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
116-
with:
117-
path: /home/runner/.databrickscfg
118-
write-mode: overwrite
119-
contents: |
120-
[DEFAULT]
121-
host = https://dbc-fbcbc113-2a61.cloud.databricks.com
122-
token = ${{ env.DATABRICKS_TOKEN }}
123-
jobs-api-version = 2.1
12496
- name: Setup python 3
12597
uses: actions/setup-python@v2
12698
with:
12799
python-version: ${{ env.PYTHON3_VERSION }}
128100
- name: Setup virtualenv
129101
run: pip install virtualenv
102+
- name: Setup databricks cli
103+
uses: microsoft/[email protected]
130104
- name: Run deploy
131105
env:
132-
DATABRICKS_CONFIG_FILE: /home/runner/.databrickscfg
133-
DATABRICKS_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
106+
DB_TOKEN: ${{ steps.secrets.outputs.databricks-ci-token }}
134107
run: |
135108
cd clouds/databricks
136109
make deploy production=1
137110
138111
publish:
139-
name: Publish Artifacts
140112
needs: test
141113
if: github.ref_name == 'master'
142114
runs-on: ubuntu-20.04
115+
timeout-minutes: 10
143116
env:
144-
PACKAGE_NAME: carto-analytics-toolbox-core-databricks
145117
PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/databricks
146118
steps:
147-
- uses: actions/checkout@v2
148-
with:
149-
fetch-depth: 0
119+
- name: Checkout repo
120+
uses: actions/checkout@v3
150121
- name: Setup scala
151122
uses: olafurpg/setup-scala@v11
152-
- uses: google-github-actions/auth@v0
123+
with:
124+
java-version: [email protected]
125+
- name: Setup python 3
126+
uses: actions/setup-python@v2
127+
with:
128+
python-version: ${{ env.PYTHON3_VERSION }}
129+
- name: Setup virtualenv
130+
run: pip install virtualenv
131+
- name: Auth google
132+
uses: google-github-actions/auth@v0
153133
with:
154134
credentials_json: ${{ secrets.GCLOUD_PRODUCTION_RELEASE_SA }}
155-
project_id: ${{ env.GCLOUD_PRODUCTION_PROJECT }}
135+
project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
156136
create_credentials_file: true
157137
- name: Setup gcloud
158138
uses: google-github-actions/setup-gcloud@v0
159139
with:
160140
version: ${{ env.GCLOUD_VERSION }}
161141
project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
162-
- name: Setup python 3
163-
uses: actions/setup-python@v2
164-
with:
165-
python-version: ${{ env.PYTHON3_VERSION }}
166-
- name: Setup virtualenv
167-
run: pip install virtualenv
168142
- name: Release
169-
run: |
170-
cd clouds/databricks
171-
make ci-release-core production=1
172143
env:
173144
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
174145
PGP_SECRET: ${{ secrets.PGP_SECRET }}
175146
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
176147
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
177-
if: ${{ env.SONATYPE_PASSWORD != '' && env.SONATYPE_USERNAME != '' }}
148+
run: |
149+
cd clouds/databricks
150+
make ci-release-core production=1
178151
- name: Create package
179152
run: |
180153
cd clouds/databricks
@@ -183,6 +156,6 @@ jobs:
183156
run: |
184157
cd clouds/databricks
185158
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
186-
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/${{ env.PACKAGE_NAME }}-latest.zip
187-
gsutil -h "Content-Type:application/json" -m cp dist/metadata.json ${{ env.PACKAGE_BUCKET }}/
159+
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-databricks-latest.zip
160+
gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/
188161
gsutil -h "Content-Type:text/plain" -m cp dist/modules.sql ${{ env.PACKAGE_BUCKET }}/

Diff for: ‎.github/workflows/postgres.yml

+9-5
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ on:
1616
env:
1717
NODE_VERSION: 14
1818
PYTHON3_VERSION: 3.8.10
19+
GCLOUD_VERSION: 290.0.1
1920

2021
jobs:
2122

@@ -87,7 +88,6 @@ jobs:
8788
runs-on: ubuntu-20.04
8889
timeout-minutes: 10
8990
env:
90-
PACKAGE_NAME: carto-analytics-toolbox-core-postgres
9191
PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/postgres
9292
steps:
9393
- name: Checkout repo
@@ -96,19 +96,23 @@ jobs:
9696
uses: actions/setup-node@v1
9797
with:
9898
node-version: ${{ env.NODE_VERSION }}
99+
- name: Setup python 3
100+
uses: actions/setup-python@v2
101+
with:
102+
python-version: ${{ env.PYTHON3_VERSION }}
103+
- name: Setup virtualenv
104+
run: pip install virtualenv
99105
- name: Auth google
100106
uses: google-github-actions/auth@v0
101107
with:
102108
credentials_json: ${{ secrets.GCLOUD_PRODUCTION_RELEASE_SA }}
103-
project_id: ${{ env.GCLOUD_PRODUCTION_PROJECT }}
109+
project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
104110
create_credentials_file: true
105111
- name: Setup gcloud
106112
uses: google-github-actions/setup-gcloud@v0
107113
with:
108114
version: ${{ env.GCLOUD_VERSION }}
109115
project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
110-
- name: Setup virtualenv
111-
run: pip install virtualenv
112116
- name: Create package
113117
run: |
114118
cd clouds/postgres
@@ -117,5 +121,5 @@ jobs:
117121
run: |
118122
cd clouds/postgres
119123
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
120-
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/${{ env.PACKAGE_NAME }}-latest.zip
124+
gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-postgres-latest.zip
121125
gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/

0 commit comments

Comments
 (0)