Skip to content
This repository was archived by the owner on Dec 8, 2023. It is now read-only.

Commit 03a0745

Browse files
drewyangdevdrewyangdev
committed
codebook env image | ci/cd
1 parent 585582b commit 03a0745

File tree

11 files changed

+222
-0
lines changed

11 files changed

+222
-0
lines changed

‎.github/.staging_workflows/anchored_docker_images.yaml

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#
2+
3+
name: docker_images
4+
on:
5+
push:
6+
pull_request:
7+
8+
### For act testing locally
9+
anchor-dev-if: &dev-if
10+
if: |
11+
github.event_name == 'push' &&
12+
startsWith(github.ref, 'refs/tags') &&
13+
github.repository_owner == 'yambottle'
14+
anchor-dev-workflow: &dev-workflow
15+
uses: dj-sciops/djsciops-cicd/.github/workflows/u24_docker_images.yaml@main
16+
### For Github Action run in your own fork with release
17+
anchor-test-if: &test-if
18+
if: |
19+
github.event_name == 'push' &&
20+
startsWith(github.ref, 'refs/tags') &&
21+
github.repository_owner == 'yambottle'
22+
anchor-test-workflow: &test-workflow
23+
uses: dj-sciops/djsciops-cicd/.github/workflows/u24_docker_images.yaml@main
24+
### For Github Action run in datajoint
25+
anchor-prod-if: &prod-if
26+
if: |
27+
github.event_name == 'push' &&
28+
startsWith(github.ref, 'refs/tags') &&
29+
github.repository_owner == 'datajoint'
30+
anchor-prod-workflow: &prod-workflow
31+
uses: dj-sciops/djsciops-cicd/.github/workflows/u24_docker_images.yaml@main
32+
### Common anchor
33+
anchor-workflow-secrets: &workflow-secrets
34+
secrets:
35+
RAW_DEPLOY_KEY: ${{secrets.RAW_DEPLOY_KEY}}
36+
REGISTRY_USERNAME: ${{secrets.REGISTRY_USERNAME}}
37+
REGISTRY_PASSWORD: ${{secrets.REGISTRY_PASSWORD}}
38+
39+
jobs:
40+
get_meta:
41+
runs-on: ubuntu-latest
42+
steps:
43+
- uses: actions/checkout@v2
44+
- id: get_version
45+
name: Get build metadata
46+
run: |
47+
## Get workflow version as codebook env image version
48+
echo "::set-output name=workflow_version::$(cat workflow/version.py | tail -1 | awk -F\' '{print $2}')"
49+
- id: create_release
50+
name: Create release
51+
!!merge <<: *$STAGE-if
52+
uses: actions/create-release@v1
53+
env:
54+
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
55+
with:
56+
tag_name: ${{steps.get_version.outputs.workflow_version}}
57+
release_name: Release ${{steps.get_version.outputs.workflow_version}}
58+
body: Release ${{steps.get_version.outputs.workflow_version}}
59+
draft: false
60+
prerelease: false
61+
outputs:
62+
workflow_version: ${{steps.get_version.outputs.workflow_version}}
63+
release_upload_url: ${{steps.create_release.outputs.upload_url}}
64+
65+
call_u24_docker_image_py39:
66+
needs: [get_meta]
67+
!!merge <<: *$STAGE-workflow
68+
with:
69+
jhub_ver: 1.4.2
70+
py_ver: 3.9
71+
dist: debian
72+
workflow_version: ${{needs.get_meta.outputs.workflow_version}}
73+
release_upload_url: ${{needs.get_meta.outputs.release_upload_url}}
74+
!!merge <<: *workflow-secrets
75+
call_u24_docker_image_py38:
76+
needs: [get_meta]
77+
!!merge <<: *$STAGE-workflow
78+
with:
79+
jhub_ver: 1.4.2
80+
py_ver: 3.8
81+
dist: debian
82+
workflow_version: ${{needs.get_meta.outputs.workflow_version}}
83+
release_upload_url: ${{needs.get_meta.outputs.release_upload_url}}
84+
!!merge <<: *workflow-secrets
85+
86+
#

‎.github/.staging_workflows/normalize.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# For Github Action that doesn't support anchor yet...
3+
# https://github.com/actions/runner/issues/1182
4+
5+
STAGE=$1
6+
# .yaml in .staging_workflows has to be named using a prefix 'anchored_', this will be removed when normalizing
7+
PREFIX="anchored_"
8+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
9+
for source in $(ls $SCRIPT_DIR | grep yaml)
10+
do
11+
target=${source#$PREFIX}
12+
export STAGE
13+
envsubst '${STAGE}' < $SCRIPT_DIR/$source | yq e 'explode(.) | del(.anchor-*)' > $SCRIPT_DIR/../workflows/$target
14+
done

‎.github/.test/.secrets

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
RAW_DEPLOY_KEY=
2+
GITHUB_TOKEN=
3+
REGISTRY_USERNAME=
4+
REGISTRY_PASSWORD=

‎.github/.test/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
For local testing using act, you need to create few things:
2+
- make a `.test/artifacts` dir for `act --artifact-server-path ./.test/artifacts/`
3+
- make a `.test/.secrets` file similar as `.env` for `act --secret-file ./.test/.secrets`

‎.github/make-prod.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/bash
2+
bash ./.staging_workflows/normalize.sh prod

‎.github/make-test.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/bash
2+
bash ./.staging_workflows/normalize.sh test

‎.github/run-local.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
3+
bash $SCRIPT_DIR/.staging_workflows/normalize.sh dev
4+
cd ..
5+
act -P ubuntu-latest=drewyangdev/ubuntu:act-latest \
6+
--secret-file $SCRIPT_DIR/.test/.secrets \
7+
--artifact-server-path $SCRIPT_DIR/.test/artifacts/

‎.github/workflows/docker_images.yaml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#
2+
name: docker_images
3+
on:
4+
push:
5+
pull_request:
6+
jobs:
7+
get_meta:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: actions/checkout@v2
11+
- id: get_version
12+
name: Get build metadata
13+
run: |
14+
## Get workflow version as codebook env image version
15+
echo "::set-output name=workflow_version::$(cat workflow/version.py | tail -1 | awk -F\' '{print $2}')"
16+
- id: create_release
17+
name: Create release
18+
if: |
19+
github.event_name == 'push' &&
20+
startsWith(github.ref, 'refs/tags') &&
21+
github.repository_owner == 'datajoint'
22+
uses: actions/create-release@v1
23+
env:
24+
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
25+
with:
26+
tag_name: ${{steps.get_version.outputs.workflow_version}}
27+
release_name: Release ${{steps.get_version.outputs.workflow_version}}
28+
body: Release ${{steps.get_version.outputs.workflow_version}}
29+
draft: false
30+
prerelease: false
31+
outputs:
32+
workflow_version: ${{steps.get_version.outputs.workflow_version}}
33+
release_upload_url: ${{steps.create_release.outputs.upload_url}}
34+
call_u24_docker_image_py39:
35+
needs: [get_meta]
36+
uses: dj-sciops/djsciops-cicd/.github/workflows/u24_docker_images.yaml@main
37+
with:
38+
jhub_ver: 1.4.2
39+
py_ver: 3.9
40+
dist: debian
41+
workflow_version: ${{needs.get_meta.outputs.workflow_version}}
42+
release_upload_url: ${{needs.get_meta.outputs.release_upload_url}}
43+
secrets:
44+
RAW_DEPLOY_KEY: ${{secrets.RAW_DEPLOY_KEY}}
45+
REGISTRY_USERNAME: ${{secrets.REGISTRY_USERNAME}}
46+
REGISTRY_PASSWORD: ${{secrets.REGISTRY_PASSWORD}}
47+
call_u24_docker_image_py38:
48+
needs: [get_meta]
49+
uses: dj-sciops/djsciops-cicd/.github/workflows/u24_docker_images.yaml@main
50+
with:
51+
jhub_ver: 1.4.2
52+
py_ver: 3.8
53+
dist: debian
54+
workflow_version: ${{needs.get_meta.outputs.workflow_version}}
55+
release_upload_url: ${{needs.get_meta.outputs.release_upload_url}}
56+
secrets:
57+
RAW_DEPLOY_KEY: ${{secrets.RAW_DEPLOY_KEY}}
58+
REGISTRY_USERNAME: ${{secrets.REGISTRY_USERNAME}}
59+
REGISTRY_PASSWORD: ${{secrets.REGISTRY_PASSWORD}}
60+
61+
#

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ Diagram.ipynb
118118

119119
# docker
120120
.env
121+
*.pem
121122

122123
# vscode
123124
.vscode/settings.json

‎docker/codebook.Dockerfile

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
## docker build --build-arg JHUB_VER=1.4.2 --build-arg PY_VER=3.8 --build-arg DIST=debian --build-arg DEPLOY_KEY=workflow-array-ephys-deploy.pem --build-arg REPO_OWNER=datajoint --build-arg REPO_NAME=workflow-array-ephys -f codebook.Dockerfile -t registry.vathes.com/datajoint/codebook-workflow-array-ephys:v0.0.0 .
2+
3+
## Single Stage
4+
ARG JHUB_VER
5+
ARG PY_VER
6+
ARG DIST
7+
FROM datajoint/djlabhub:${JHUB_VER}-py${PY_VER}-${DIST}
8+
9+
USER root
10+
RUN apt-get -y update && apt-get install -y ssh git
11+
12+
ARG DEPLOY_KEY
13+
USER anaconda:anaconda
14+
COPY --chown=anaconda $DEPLOY_KEY $HOME/.ssh/datajoint_deploy.ssh
15+
16+
ARG REPO_OWNER
17+
ARG REPO_NAME
18+
WORKDIR /tmp
19+
RUN ssh-keyscan github.com >> $HOME/.ssh/known_hosts && \
20+
GIT_SSH_COMMAND="ssh -i $HOME/.ssh/datajoint_deploy.ssh" \
21+
git clone [email protected]:${REPO_OWNER}/${REPO_NAME}.git && \
22+
pip install ./${REPO_NAME} && \
23+
cp -r ./${REPO_NAME}/notebooks/ /home/ && \
24+
cp -r ./${REPO_NAME}/images/ /home/notebooks/ && \
25+
cp ./${REPO_NAME}/README.md /home/notebooks/ && \
26+
rm -rf /tmp/${REPO_NAME}
27+
WORKDIR /home/notebooks

‎docker/docker-compose-codebook_env.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# docker-compose up --build
2+
version: '2.4'
3+
services:
4+
codebook_env:
5+
build: # only necessary if rebuilding image
6+
context: .
7+
dockerfile: codebook.Dockerfile
8+
args:
9+
- JHUB_VER
10+
- PY_VER
11+
- DIST
12+
- DEPLOY_KEY
13+
- REPO_OWNER
14+
- REPO_NAME
15+
image: registry.vathes.com/datajoint/codebook-${REPO_NAME}:jhub${JHUB_VER}-py${PY_VER}-${DIST}-v${WORKFLOW_VERSION}

0 commit comments

Comments
 (0)