Publish Docker images to GHCR (#118)

allenai · Dec 4, 2024 · 10c4978 · 10c4978
1 parent d6981b3
commit 10c4978
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 10 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -16,11 +16,11 @@ on:
   #     - 'src/olmo_core/version.py'
   #     - 'src/Dockerfile'
   #     - '.github/workflows/docker.yml'
-  # push:
-  #   branches:
-  #     - main
-  #   tags:
-  #     - 'v*.*.*'
+  push:
+    # branches:
+    #   - main
+    tags:
+      - 'v*.*.*'
 
 jobs:
   beaker:
@@ -40,6 +40,7 @@ jobs:
       - name: Set env vars
         run: |
           echo "BEAKER_WORKSPACE=$(make get-beaker-workspace)" >> $GITHUB_ENV
+          rm -rf /opt/hostedtoolcache  # clear up some disk space
 
       - name: Build stable image
         run: |
@@ -49,20 +50,34 @@ jobs:
         run: |
           make nightly-image
 
-      - uses: allenai/setup-beaker@v2
+      - name: Authenticate with GHCR
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Authenticate with Beaker
+        uses: allenai/setup-beaker@v2
         if: env.BEAKER_TOKEN != ''
         with:
           token: ${{ env.BEAKER_TOKEN }}
           workspace: ${{ env.BEAKER_WORKSPACE }}
 
-      - name: Push stable image
+      - name: Push stable image to GHCR
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          make ghcr-image-stable
+
+      - name: Push nightly image to GHCR
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          make ghcr-image-nightly
+
+      - name: Push stable image to Beaker
         if: env.BEAKER_TOKEN != '' && startsWith(github.ref, 'refs/tags/')
         run: |
-          rm -rf /opt/hostedtoolcache  # clear up some disk space
           make beaker-image-stable
 
-      - name: Push nightly image
+      - name: Push nightly image to Beaker
         if: env.BEAKER_TOKEN != '' && startsWith(github.ref, 'refs/tags/')
         run: |
-          rm -rf /opt/hostedtoolcache  # clear up some disk space
           make beaker-image-nightly
diff --git a/Makefile b/Makefile
@@ -75,12 +75,30 @@ nightly-image :
 		-t $(IMAGE_BASENAME)-nightly .
 	echo "Built image '$(IMAGE_BASENAME)-nightly', size: $$(docker inspect -f '{{ .Size }}' $(IMAGE_BASENAME)-nightly | numfmt --to=si)"
 
+.PHONY : ghcr-image-stable
+ghcr-image-stable : stable-image
+	docker tag $(IMAGE_BASENAME) ghcr.io/allenai/$(IMAGE_BASENAME)
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)
+	docker tag $(IMAGE_BASENAME) ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION_SHORT)
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION_SHORT)
+	docker tag $(IMAGE_BASENAME) ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION)
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION)
+
 .PHONY : beaker-image-stable
 beaker-image-stable : stable-image
 	./src/scripts/beaker/create_beaker_image.sh $(IMAGE_BASENAME) $(IMAGE_BASENAME) $(BEAKER_WORKSPACE)
 	./src/scripts/beaker/create_beaker_image.sh $(IMAGE_BASENAME) $(IMAGE_BASENAME)-v$(VERSION_SHORT) $(BEAKER_WORKSPACE)
 	./src/scripts/beaker/create_beaker_image.sh $(IMAGE_BASENAME) $(IMAGE_BASENAME)-v$(VERSION) $(BEAKER_WORKSPACE)
 
+.PHONY : ghcr-image-nightly
+ghcr-image-nightly : nightly-image
+	docker tag $(IMAGE_BASENAME)-nightly ghcr.io/allenai/$(IMAGE_BASENAME)-nightly
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)-nightly
+	docker tag $(IMAGE_BASENAME)-nightly ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION_SHORT)-nightly
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION_SHORT)-nightly
+	docker tag $(IMAGE_BASENAME)-nightly ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION)-nightly
+	docker push ghcr.io/allenai/$(IMAGE_BASENAME)-v$(VERSION)-nightly
+
 .PHONY : beaker-image-nightly
 beaker-image-nightly : nightly-image
 	./src/scripts/beaker/create_beaker_image.sh $(IMAGE_BASENAME)-nightly $(IMAGE_BASENAME)-nightly $(BEAKER_WORKSPACE)

diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@
   <a href="https://github.com/allenai/OLMo-core/tree/main/src/examples">Examples</a> ||
   <a href="https://olmo-core.readthedocs.io/en/latest/">Docs</a> ||
   <a href="https://pypi.org/project/ai2-olmo-core/">PyPI</a> ||
+  <a href="https://github.com/orgs/allenai/packages?repo_name=OLMo-core">Docker Images</a> ||
   <a href="https://beaker.org/ws/ai2/OLMo-core/images">Beaker Images</a> ||
   <a href="https://github.com/allenai/OLMo-core/blob/main/LICENSE">License</a> ||
   <a href="https://github.com/allenai/OLMo-core/blob/main/CHANGELOG.md">Changelog</a>
@@ -24,6 +25,13 @@ There are a number of optional dependencies that must be installed to use certai
 - [torchao](https://github.com/pytorch/ao) for float8 training.
 - [megablocks](https://github.com/databricks/megablocks) for mixture-of-experts (MoE) models.
 
+The published [Docker images](https://github.com/orgs/allenai/packages?repo_name=OLMo-core) contain all core and optional dependencies, and are regularly tested on our in-house H100 clusters.
+But there are several things to keep in mind if you intend to use these images:
+- They do not come with the OLMo-core package installed, only its dependencies, to accommodate for regular code changes. Therefore the `*-nightly` tags indicate that they come with PyTorch nightly, not some nightly version of OLMo-core.
+- They may not work on your own cluster if you have different hardware or driver/CUDA versions.
+
+If the published images do not work for your use-case for any of the above reasons, you could adapt our [Dockerfile](https://github.com/allenai/OLMo-core/blob/main/src/Dockerfile) to build your own images.
+
 ## API stability
 
 Even though this library is under rapid development we are trying hard to adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) with every release except for features that are explicitly marked as beta features. Those features will be tagged like this in the [API docs](https://olmo-core.readthedocs.io/en/latest/):