Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 75 additions & 6 deletions .circleci/real_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1803,10 +1803,12 @@ jobs:
password: ${NGC_API_KEY}
- pre-package-and-push-system
- make-component:
dryrun: <<parameters.dryrun>>
component: master
target: release
no_output_timeout: "30m"
- make-component:
dryrun: <<parameters.dryrun>>
component: agent
target: release
no_output_timeout: "30m"
Expand All @@ -1815,6 +1817,10 @@ jobs:
path: /tmp/pkgs

package-and-push-system-release-ee:
parameters:
dryrun:
type: boolean
default: false
docker:
- image: <<pipeline.parameters.docker-image>>
environment:
Expand All @@ -1832,12 +1838,16 @@ jobs:
username: ${DOCKER_USER}
password: ${DOCKER_PASS}
- pre-package-and-push-system
- run:
no_output_timeout: 30m
command: make -C master release-ee
- run:
no_output_timeout: 30m
command: make -C agent release-ee
- make-component:
dryrun: <<parameters.dryrun>>
component: master
target: release-ee
no_output_timeout: "30m"
- make-component:
dryrun: <<parameters.dryrun>>
component: agent
target: release-ee
no_output_timeout: "30m"
- run: mkdir /tmp/pkgs && cp -v */dist/*.{rpm,deb,tar.gz} /tmp/pkgs

publish-helm-gh-rc:
Expand Down Expand Up @@ -5017,6 +5027,65 @@ workflows:
context: determined-production
filters: *release-filters

release-ee-dryrun:
jobs:
- build-helm:
filters: *release-dryrun

- build-proto:
filters: *release-dryrun

- build-react:
context: determined-production
matrix:
parameters:
ee: [true]
filters: *release-dryrun

- build-docs:
matrix:
parameters:
ee: [true]
context: determined-production
filters: *release-dryrun
requires:
- build-helm
- build-proto

- upload-docs-search-index:
requires:
- build-docs
context: determined-production
filters: *release-dryrun

- package-and-push-system-release-ee:
dryrun: true
requires:
- build-react
- build-docs
context: determined-production
filters: *release-dryrun

- publish-docs:
matrix:
parameters:
ee: [true]
dryrun: [true]
requires:
- build-docs
context: determined-production
filters: *release-dryrun

- publish-helm-gh:
matrix:
parameters:
ee: [true]
dryrun: [true]
requires:
- build-helm
context: determined-production
filters: *release-dryrun

release-ee:
jobs:
- build-helm:
Expand Down
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ go-version-check:

.PHONY: package
package:
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
$(MAKE) -C agent $@
$(MAKE) -C master $@

Expand All @@ -44,6 +45,7 @@ package-dryrun:

.PHONY: package-ee
package-ee:
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
$(MAKE) -C agent $@
$(MAKE) -C master $@

Expand Down
2 changes: 1 addition & 1 deletion agent/.goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ docker_manifests:
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.Env.VERSION}}"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.Env.VERSION_DOCKER}}"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
Expand Down
6 changes: 3 additions & 3 deletions agent/.goreleaser_ee.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,13 @@ nfpms:
release:
github:
owner: determined-ai
name: determined-ee
name: determined

# be sure to keep this in sync between agent/master/helm
# the "include" functionality is only in the pro version
header: |
## Release Notes
[{{ .Tag }}](https://github.com/determined-ai/determined-ee/blob/{{ .Tag }}/docs/release-notes.rst)
[{{ .Tag }}](https://github.com/determined-ai/determined/blob/{{ .Tag }}/docs/release-notes.rst)

dockers:
# amd64
Expand Down Expand Up @@ -134,7 +134,7 @@ docker_manifests:
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.Env.VERSION}}"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.Env.VERSION_DOCKER}}"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
Expand Down
144 changes: 144 additions & 0 deletions agent/.goreleaser_ee_dryrun.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
project_name: hpe-mlde-agent-dryrun

snapshot:
name_template: "{{ .Env.VERSION }}"

builds:
- main: ./cmd/determined-agent
id: determined-agent
binary: determined-agent
goos:
- linux
- darwin
goarch:
- amd64
- arm64

archives:
- wrap_in_directory: "true"
rlcp: true
files:
- src: "packaging/agent.yaml"
dst: "etc/determined/"
strip_parent: true
- src: "packaging/LICENSE"
strip_parent: true

#brews:
# - name: determined-agent
# tap:
# owner: determined-ai
# name: homebrew-determined
# url_template: "https://github.com/determined-ai/determined/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
# caveats: |
# Determined agent config is located at #{etc}/determined/agent.yaml
# homepage: "https://github.com/determined-ai/determined"
# license: "Apache-2.0"
# folder: Formula
# install: |
# bin.install "determined-agent"
#
# doc.install "LICENSE"
#
# (var/"log/determined").mkpath
#
# (etc/"determined").mkpath
# inreplace "etc/determined/agent.yaml" do |s|
# s.gsub! "# master_host: 0.0.0.0", "master_host: 127.0.0.1"
# s.gsub! "# master_port: 80", "master_port: 8080"
# end
#
# Pathname("etc/determined/agent.yaml").append_lines <<~EOS
# container_master_host: host.docker.internal
# EOS
#
# etc.install "etc/determined/agent.yaml" => "determined/agent.yaml"
# service: |
# run [opt_bin/"determined-agent", "--config-file", etc/"determined/agent.yaml"]
# keep_alive false
# error_log_path var/"log/determined/agent-stderr.log"
# log_path var/"log/determined/agent-stdout.log"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: maybe we should just cut this comment, since we're extremely unlikely to ever uncomment this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cut


nfpms:
- maintainer: "Determined AI <ai-open-source@hpe.com>"
formats:
- deb
- rpm
contents:
- src: "packaging/agent.yaml"
dst: "/etc/determined/agent.yaml"
type: config|noreplace
file_info:
mode: 0644
- src: "packaging/determined-agent.service"
dst: "/lib/systemd/system/determined-agent.service"

- src: "packaging/LICENSE"
dst: "/usr/share/licenses/determined-agent/LICENSE"
packager: rpm

- src: "packaging/LICENSE"
dst: "/usr/share/doc/determined-agent/copyright"
packager: deb

overrides:
deb:
scripts:
postinstall: packaging/debian/agent.postinst
preremove: packaging/debian/agent.prerm
postremove: packaging/debian/agent.postrm

release:
github:
owner: determined-ai
name: determined-dryrun

# be sure to keep this in sync between agent/master/helm
# the "include" functionality is only in the pro version
header: |
## Release Notes
[{{ .Tag }}](https://github.com/determined-ai/determined-dryrun/blob/{{ .Tag }}/docs/release-notes.rst)

dockers:
# amd64
- goos: linux
goarch: amd64
use: buildx
build_flag_templates:
- --platform=linux/amd64
- --builder=buildx-build
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
extra_files:
- packaging/entrypoint.sh
- packaging/LICENSE
# arm64
- goos: linux
goarch: arm64
use: buildx
build_flag_templates:
- --platform=linux/arm64
- --builder=buildx-build
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
extra_files:
- packaging/entrypoint.sh
- packaging/LICENSE

docker_manifests:
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.ShortCommit}}"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.Env.VERSION_DOCKER}}"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
- name_template: "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:latest"
image_templates:
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-amd64"
- "{{.Env.DOCKER_REPO}}/{{.ProjectName}}:{{.FullCommit}}-arm64"
9 changes: 8 additions & 1 deletion agent/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,17 @@ release-dryrun: packaging/LICENSE buildx

.PHONY: release-ee
release-ee: export GORELEASER_CURRENT_TAG := $(VERSION_TAG)-ee
release-ee: export GORELEASER_PREVIOUS_TAG := $(shell git tag --sort=-creatordate | grep -E '^v?[0-9.]+-ee$$' | grep "$(VERSION_TAG)-ee" -A1 | sed -n '2 p')
release-ee: export GORELEASER_PREVIOUS_TAG := $(shell git tag --sort=-creatordate | grep -E '^v?[0-9.]+-ee$$' | grep "$(GORELEASER_CURRENT_TAG)" -A1 | sed -n '2 p')
release-ee: packaging/LICENSE buildx
goreleaser --rm-dist -f ./.goreleaser_ee.yml

.PHONY: release-ee-dryrun
# VERSION_TAG v0.38.0+dryrun becomes v0.38.0-ee+dryrun
release-ee-dryrun: export GORELEASER_CURRENT_TAG := $(shell echo $(VERSION_TAG) | sed "s/+dryrun/-ee&/g")
release-ee-dryrun: export GORELEASER_PREVIOUS_TAG := $(shell git tag --sort=-creatordate | grep -E '^v?[0-9.]+-ee$$' | grep "$(GORELEASER_CURRENT_TAG)" -A1 | sed -n '2 p')
release-ee-dryrun: packaging/LICENSE buildx
goreleaser --rm-dist -f ./.goreleaser_ee_dryrun.yml

define manifest_publish
for image in $(2); do \
docker push $$image; \
Expand Down
1 change: 1 addition & 0 deletions harness/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ publish-ee: ;
publish-dryrun:
twine upload --verbose --non-interactive --repository testpypi dist/*

.PHONY: publish-ee-dryrun
publish-ee-dryrun: ;

.PHONY: fmt
Expand Down
2 changes: 1 addition & 1 deletion helm/.goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ release:
mode: keep-existing
extra_files:
- glob: build/determined-latest.tgz
name_template: "determined-helm-chart_{{ .Env.VERSION }}.tgz"
name_template: "determined-helm-chart_{{ .VERSION }}.tgz"
prerelease: auto

# be sure to keep this in sync between agent/master/helm
Expand Down
4 changes: 2 additions & 2 deletions helm/.goreleaser_dryrun.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
project_name: determined-helm
project_name: determined-helm-dryrun

build:
skip: true
Expand All @@ -10,7 +10,7 @@ release:
mode: keep-existing
extra_files:
- glob: build/determined-latest.tgz
name_template: "determined-helm-chart_{{ .Env.VERSION }}.tgz"
name_template: "determined-helm-chart_{{ .VERSION }}.tgz"

# be sure to keep this in sync between agent/master/helm
# the "include" functionality is only in the pro version
Expand Down
9 changes: 4 additions & 5 deletions helm/.goreleaser_ee.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
project_name: determined-helm
project_name: hpe-mlde-helm

build:
skip: true

release:
github:
owner: determined-ai
name: determined-ee
name: determined
mode: keep-existing
extra_files:
- glob: build/determined-latest.tgz
name_template: "determined-helm-chart_{{ .Env.VERSION }}.tgz"
prerelease: auto
name_template: "hpe-mlde-helm-chart_{{ .VERSION }}.tgz"

# be sure to keep this in sync between agent/master/helm
# the "include" functionality is only in the pro version
header: |
## Release Notes
[{{ .Tag }}](https://github.com/determined-ai/determined-ee/blob/{{ .Tag }}/docs/release-notes.rst)
[{{ .Tag }}](https://github.com/determined-ai/determined/blob/{{ .Tag }}/docs/release-notes.rst)
Loading