Skip to content

Commit d2f9024

Browse files
pingsutwpingsutw
committed
Merge branch 'main' of https://github.com/flyteorg/flyte into fix/connector-grpc-keepalive-retry
2 parents ddc8dde + e9b02d4 commit d2f9024

33 files changed

Lines changed: 12266 additions & 6441 deletions

‎.github/workflows/publish-helm.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Publish Helm Package
2+
3+
on:
4+
push:
5+
tags:
6+
- 'v2*'
7+
8+
env:
9+
DEPOT_PROJECT_ID: ${{ vars.DEPOT_PROJECT_ID }}
10+
11+
jobs:
12+
build-and-push-single-binary-image:
13+
runs-on: ubuntu-latest
14+
permissions:
15+
contents: read
16+
id-token: write # required for Depot OIDC auth (GHCR auth uses FLYTE_BOT_PAT)
17+
steps:
18+
- name: Checkout
19+
uses: actions/checkout@v4
20+
- name: Validate Depot project id
21+
run: |
22+
if [ -z "${DEPOT_PROJECT_ID}" ]; then
23+
echo "::error::DEPOT_PROJECT_ID repo variable is not set. Add it under Settings → Secrets and variables → Actions → Variables." >&2
24+
exit 1
25+
fi
26+
- name: Setup Golang caches
27+
uses: actions/cache@v4
28+
with:
29+
path: |
30+
/root/.cache/go-build
31+
/root/go/pkg/mod
32+
key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
33+
restore-keys: |
34+
${{ runner.os }}-golang-
35+
- name: Set versions
36+
id: set_version
37+
run: |
38+
# TODO: The console version should be set in config and send into Dockerfile in the future
39+
# echo "FLYTECONSOLE_VERSION=latest" >> $GITHUB_ENV
40+
echo "FLYTE_VERSION=${{ github.ref_name }}" >> $GITHUB_ENV
41+
- name: Prepare Image Names
42+
id: image-names
43+
uses: docker/metadata-action@v3
44+
with:
45+
images: |
46+
ghcr.io/${{ github.repository_owner }}/flyte-binary-v2
47+
tags: |
48+
type=ref,event=tag
49+
- name: Set up Depot
50+
uses: depot/setup-action@v1
51+
- name: Login to GitHub Container Registry
52+
uses: docker/login-action@v3
53+
with:
54+
registry: ghcr.io
55+
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
56+
password: "${{ secrets.FLYTE_BOT_PAT }}"
57+
- name: Build and push Image
58+
uses: depot/build-push-action@v1
59+
with:
60+
project: ${{ env.DEPOT_PROJECT_ID }}
61+
context: .
62+
# Native multi-arch on Depot — no QEMU. Re-uses the persistent
63+
# project layer cache from the per-arch builds above.
64+
platforms: linux/arm64,linux/amd64
65+
tags: ${{ steps.image-names.outputs.tags }}
66+
build-args: |
67+
FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
68+
FLYTE_VERSION=${{ env.FLYTE_VERSION }}
69+
file: Dockerfile
70+
push: true
71+
72+
helm-release:
73+
name: Flyte helm release
74+
runs-on: ubuntu-latest
75+
needs:
76+
- build-and-push-single-binary-image
77+
steps:
78+
- name: Checkout
79+
uses: actions/checkout@v4
80+
with:
81+
fetch-depth: "0"
82+
83+
- name: Install Helm
84+
uses: azure/setup-helm@v3
85+
with:
86+
version: 'latest' # default is latest (stable)
87+
token: ${{ secrets.GITHUB_TOKEN }} # only needed if version is 'latest'
88+
id: install
89+
- name: Configure Git
90+
run: |
91+
git config user.name "${{ github.actor }}"
92+
git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
93+
- name: Prepare Flyte Helm Release
94+
env:
95+
VERSION: ${{ github.ref_name }}
96+
REPOSITORY: "https://flyteorg.github.io/flyte"
97+
run: |
98+
make tag-helm
99+
- name: Run chart-releaser
100+
uses: stefanprodan/helm-gh-pages@v1.4.1
101+
with:
102+
token: "${{ secrets.FLYTE_BOT_PAT }}"
103+
linting: off

‎.github/workflows/v1-create_release.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
# This workflow is triggered manually and only releases the v1 flyte package.
2+
name: Create a flyte release
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
version:
7+
description: "version name. example v0.1.1, v1.16.0-b0"
8+
required: true
9+
10+
jobs:
11+
generate-tags:
12+
name: Generate git tags
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
with:
17+
fetch-depth: '0'
18+
- uses: actions/github-script@v6
19+
with:
20+
github-token: ${{ secrets.FLYTE_BOT_PAT }}
21+
script: |
22+
github.rest.git.createRef({
23+
owner: context.repo.owner,
24+
repo: context.repo.repo,
25+
ref: `refs/tags/${{ github.event.inputs.version }}`,
26+
sha: context.sha
27+
})
28+
const components = [
29+
"datacatalog",
30+
"flyteadmin",
31+
"flytecopilot",
32+
"flyteplugins",
33+
"flytepropeller",
34+
"flytestdlib",
35+
];
36+
for (const c of components) {
37+
github.rest.git.createRef({
38+
owner: context.repo.owner,
39+
repo: context.repo.repo,
40+
ref: `refs/tags/${c}/${{ github.event.inputs.version }}`,
41+
sha: context.sha
42+
})
43+
}
44+
45+
build-docker-images:
46+
needs:
47+
- generate-tags
48+
uses: ./.github/workflows/publish-images.yml
49+
with:
50+
version: ${{ github.event.inputs.version }}
51+
push: true
52+
secrets:
53+
FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
54+
FLYTE_BOT_USERNAME: ${{ secrets.FLYTE_BOT_USERNAME }}
55+
56+
publish-flyte-binary-image:
57+
name: Publish flyte binary image for the release version
58+
runs-on: ubuntu-latest
59+
needs:
60+
- generate-tags
61+
steps:
62+
- name: Checkout
63+
uses: actions/checkout@v4
64+
with:
65+
fetch-depth: "0"
66+
67+
- name: Login to GitHub Container Registry
68+
uses: docker/login-action@v1
69+
with:
70+
registry: ghcr.io
71+
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
72+
password: "${{ secrets.FLYTE_BOT_PAT }}"
73+
74+
- name: Tag image to release version
75+
run: |
76+
for release in latest ${{ github.event.inputs.version }}; do
77+
docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/flyte-binary-release:${release}" "ghcr.io/${{ github.repository_owner }}/flyte-binary:sha-${{ github.sha }}"
78+
done
79+
80+
publish-flyte-component-image:
81+
name: Publish flyte component image for the release version
82+
runs-on: ubuntu-latest
83+
needs:
84+
- build-docker-images
85+
strategy:
86+
matrix:
87+
component:
88+
[
89+
datacatalog,
90+
flyteadmin,
91+
flyteagent,
92+
flyteconsole,
93+
flytecopilot,
94+
flytepropeller,
95+
flytescheduler,
96+
]
97+
steps:
98+
- name: Checkout
99+
uses: actions/checkout@v4
100+
with:
101+
fetch-depth: "0"
102+
103+
- name: yq - portable yaml processor
104+
uses: mikefarah/yq@v4.15.1
105+
106+
- name: Get Latest Version of component
107+
id: set_version
108+
run: |
109+
if [ ${{ matrix.component }} = "flytecopilot" ]; then
110+
echo ::set-output name=version::$(yq eval '.configmap.copilot.plugins.k8s.co-pilot.image' charts/flyte-core/values.yaml | cut -d ":" -f 2 )
111+
elif [ ${{ matrix.component }} = "flyteagent" ]; then
112+
echo ::set-output name=version::$(yq eval '.image.tag' charts/flyteconnector/values.yaml)
113+
else
114+
echo ::set-output name=version::$(yq eval '.${{ matrix.component }}.image.tag' charts/flyte-core/values.yaml)
115+
fi
116+
shell: bash
117+
118+
- name: Login to GitHub Container Registry
119+
uses: docker/login-action@v1
120+
with:
121+
registry: ghcr.io
122+
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
123+
password: "${{ secrets.FLYTE_BOT_PAT }}"
124+
125+
- name: Tag Image to release version
126+
run: |
127+
for release in latest ${{ github.event.inputs.version }}; do
128+
docker buildx imagetools create --tag "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}-release:${release}" "ghcr.io/${{ github.repository_owner }}/${{ matrix.component }}:${{ steps.set_version.outputs.version }}"
129+
done
130+
131+
helm-release:
132+
name: Flyte helm release
133+
runs-on: ubuntu-latest
134+
needs:
135+
- build-docker-images
136+
steps:
137+
- name: Checkout
138+
uses: actions/checkout@v4
139+
with:
140+
fetch-depth: "0"
141+
142+
- name: Install Helm
143+
uses: azure/setup-helm@v3
144+
with:
145+
version: 'latest' # default is latest (stable)
146+
token: ${{ secrets.GITHUB_TOKEN }} # only needed if version is 'latest'
147+
id: install
148+
- name: Configure Git
149+
run: |
150+
git config user.name "${{ github.actor }}"
151+
git config user.email "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>"
152+
- name: Prepare Flyte Helm Release
153+
env:
154+
VERSION: ${{ github.event.inputs.version }}
155+
REPOSITORY: "https://flyteorg.github.io/flyte"
156+
run: |
157+
make prepare_artifacts
158+
- name: Run chart-releaser
159+
uses: stefanprodan/helm-gh-pages@v1.4.1
160+
with:
161+
token: "${{ secrets.FLYTE_BOT_PAT }}"
162+
linting: off
163+
164+
manifest-release:
165+
name: Flyte manifest release
166+
runs-on: ubuntu-latest
167+
needs:
168+
- build-docker-images
169+
steps:
170+
- name: Checkout
171+
uses: actions/checkout@v4
172+
with:
173+
fetch-depth: "0"
174+
175+
- name: Prepare Flyte Release
176+
env:
177+
VERSION: ${{ github.event.inputs.version }}
178+
run: |
179+
make prepare_artifacts
180+
git stash
181+
182+
- name: Run GoReleaser
183+
uses: goreleaser/goreleaser-action@v6.3.0
184+
with:
185+
version: latest
186+
args: release --release-notes=CHANGELOG/CHANGELOG-${{ github.event.inputs.version }}.md
187+
env:
188+
GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}

‎.github/workflows/v1-flyteidl-release.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# This workflow is triggered manually and only releases the v1 flyteidl package.
2+
name: Release flyteidl
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
version:
8+
description: "version. Do *not* use the `flyteidl/` prefix, e.g. `flyteidl/v1.2.3`, instead use only `v1.2.3` (including the `v`)"
9+
required: true
10+
11+
jobs:
12+
push-flyteidl-tag:
13+
name: Push git tag containing the `flyteidl/` prefix
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
fetch-depth: '0'
19+
- uses: actions/github-script@v6
20+
with:
21+
github-token: ${{ secrets.FLYTE_BOT_PAT }}
22+
script: |
23+
github.rest.git.createRef({
24+
owner: context.repo.owner,
25+
repo: context.repo.repo,
26+
ref: `refs/tags/flyteidl/${{ github.event.inputs.version }}`,
27+
sha: context.sha
28+
})
29+
deploy-to-pypi:
30+
needs:
31+
- push-flyteidl-tag
32+
runs-on: ubuntu-latest
33+
defaults:
34+
run:
35+
working-directory: flyteidl
36+
steps:
37+
- uses: actions/checkout@v4
38+
with:
39+
fetch-depth: '0'
40+
- name: Set up Python
41+
uses: actions/setup-python@v1
42+
with:
43+
python-version: "3.x"
44+
- name: Install dependencies
45+
run: |
46+
python -m pip install --upgrade pip
47+
pip install build twine
48+
- name: Build and publish
49+
env:
50+
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
51+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
52+
run: |
53+
python -m build
54+
twine upload dist/*
55+
deploy-to-npm:
56+
needs:
57+
- push-flyteidl-tag
58+
runs-on: ubuntu-latest
59+
defaults:
60+
run:
61+
working-directory: flyteidl
62+
steps:
63+
- uses: actions/checkout@v4
64+
- uses: actions/setup-node@v1
65+
with:
66+
node-version: "12.x"
67+
registry-url: "https://registry.npmjs.org"
68+
- name: Set version in npm package
69+
run: |
70+
# v1.2.3 get 1.2.3
71+
VERSION=$(echo ${{ inputs.version }} | sed 's#.*v##')
72+
VERSION=$VERSION make update_npmversion
73+
shell: bash
74+
- run: |
75+
npm install
76+
- run: |
77+
npm publish --access=public
78+
env:
79+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

0 commit comments

Comments
 (0)