diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000..d64fb43f --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,70 @@ +name: Build GFM artifacts + +# Builds the GFM library as artifacts that can be used by other workflows +# +# It checks for any existing artifacts with an expiry date at least 10 days after today +# If none are found, the library is built and uploaded. + +# This workflow can be run on demand, but is intended to be run on a weekly schedule +# This should ensure that there is always at least one artifact available but no more than 2 + +on: + # Schedule once a week + schedule: + - cron: '11 12 * * 0' + push: + paths: + - '**/build-artifacts.yml' # self + - 'pelican/build-cmark.sh' + workflow_dispatch: + +permissions: + content: read + +jobs: + build-artifact: + runs-on: ubuntu-latest + strategy: + # Allow for multiple versions to be maintained + matrix: + gfm_version: + - '0.28.3.gfm.12' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Fetch and build version ${{ matrix.gfm_version }} + id: build_gfm + env: + gfm_version: ${{ matrix.gfm_version }} + run: | + # N.B. This must agree with the definition in pelican/action.yml + export GFM_ARTIFACT_KEY=gfm-lib-${gfm_version} + echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV + + # Check if artifact is present (list all) + curl -sS https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json + # when does last one expire? + jq /tmp/max.txt + # is that more than 10 days away? (86400*10 seconds) + OUT=$(jq (now+864000|strftime("%FT%TZ")))') + if [[ -n $OUT ]] + then + echo "Found a valid artifact for ${gfm_version} (expires $OUT)" + exit 0 # No more to do + fi + + echo "Could not find a valid artifact for ${gfm_version}; building another" + # build GFM and set up LIBCMARKDIR + export LIBCMARKDIR=/tmp/gfm-${gfm_version} + mkdir -p ${LIBCMARKDIR} + bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${gfm_version} ${LIBCMARKDIR} + + # Tell the save step what to save + echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT + - name: Save the GFM build ${{ matrix.gfm_version }} + if: ${{ steps.build_gfm.outputs.created }} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.GFM_ARTIFACT_KEY }} + path: ${{ steps.build_gfm.outputs.created }} diff --git a/pelican/Dockerfile b/pelican/Dockerfile index 931b2f49..b895c592 100644 --- a/pelican/Dockerfile +++ b/pelican/Dockerfile @@ -1,11 +1,16 @@ # Settings # ======== # Use the Python version as installed on CI pelican builders (2023-06-02) -ARG PYTHON_VERSION=3.8.10 -ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below +ARG PYTHON_VERSION=3.8.10 # must agree with the copy below + +# Note: ARG scope ends at the FROM statement, so must be repeated if necessary # Build cmake-gfm FROM python:${PYTHON_VERSION}-slim-buster as pelican-asf +#======================================================= + +ARG PYTHON_VERSION=3.8.10 # must agree with the copy above +ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below RUN apt update && apt upgrade -y RUN apt install curl cmake build-essential -y @@ -15,11 +20,15 @@ WORKDIR /opt/pelican-asf # Copy only what we need to build cmark-gfm COPY build-cmark.sh bin/build-cmark.sh -# build it -RUN bash bin/build-cmark.sh ${GFM_VERSION} +# build GFM +# Must agree with definition below +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib +RUN mkdir -p ${LIBCMARKDIR} +RUN bash bin/build-cmark.sh ${GFM_VERSION} ${LIBCMARKDIR} # rebase the image to save on image size FROM python:${PYTHON_VERSION}-slim-buster +#======================================================= # Use the Pelican version as installed on CI pelican builders (2023-06-02) ARG PELICAN_VERSION=4.5.4 @@ -27,7 +36,8 @@ ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy above # Where we put GFM and the plugins ENV WORKDIR /opt/pelican-asf -ENV LIBCMARKDIR ${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib +# Must agree with definition above +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib RUN apt update && apt upgrade -y diff --git a/pelican/action.yml b/pelican/action.yml index 43b42aeb..d1832d4a 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -51,18 +51,46 @@ runs: # If the site uses Github Flavored Markdown, use this build branch - name: fetch and build libcmark-gfm.so if: ${{ inputs.gfm == 'true' }} + id: build_gfm shell: bash env: - WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script + # action_repository only works in the env context; empty for local action call + # it is always empty for local invocation, in which case use the current repo + GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }} + GH_TOKEN: ${{ github.token }} # needed by gh run: | + # The key needs to include the GFM version, but is otherwise arbitrary. + # It must agree with the definition in build-actions.yml + export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} + + if [[ -z $LIBCMARKDIR ]] # define LIBCMARKDIR if it is not already + then + # set up the GFM environment + export LIBCMARKDIR=/opt/pelican-asf/gfm-${GFM_VERSION} # arbitrary, but should contain version + mkdir -p $LIBCMARKDIR + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step + fi + # Does the GFM build already exist? - if [[ -n $LIBCMARKDIR && -d $LIBCMARKDIR ]] + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step fi + + # Is there a saved artifact for the GFM build? + echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] + then + echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" + exit 0 # nothing more to do in this step + fi + + # GFM binary not found, need to build it { + echo "Creating GFM binary in ${LIBCMARKDIR}" # disable stdout unless debug is on if [ "${{ inputs.debug }}" == 'true' ] then @@ -70,15 +98,8 @@ runs: else exec >/dev/null fi - # Don't pollute site checkout - mkdir -p $WORKDIR - pushd $WORKDIR - # build the code and define LIBCMARKDIR - bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ - source /tmp/libcmarkdir.$$ - popd - # ensure LIBCMARKDIR is defined for subsequent steps - echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV + # build the code and define LIBCMARKDIR under $WORKDIR + bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR } - name: Generate website from markdown diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index c9296dc8..76eac58d 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -1,15 +1,15 @@ #!/bin/bash # -# Build the cmark-gfm library and extensions within CURRENT DIRECTORY. +# Build the cmark-gfm library and extensions in a temporary directory # -# The binary output will be under: cmark-gfm-$VERSION/lib +# The binary output will be under: LIBCMARKDIR # # USAGE: -# $ build-cmark.sh [ VERSION [ TARDIR ] ] +# $ build-cmark.sh VERSION LIBCMARKDIR [TARFILE] # -# VERSION: defaults to 0.28.3.gfm.12 -# TARDIR: where to find a downloaded/cached tarball of the cmark -# code, or where to place a tarball +# VERSION: e.g. 0.28.3.gfm.12 +# LIBCMARKDIR: where to put the binary library files +# TARFILE: local copy of the tarfile; must be for the correct version! (optional) # # Echo all of our steps if DEBUG_STEPS is set @@ -17,33 +17,36 @@ test -n "$DEBUG_STEPS" && set -x set -e # early exit if any step fails -#VERSION=0.28.3.gfm.20 ### not yet -VERSION=0.28.3.gfm.12 -if [ "$1" != "" ]; then VERSION="$1"; fi +VERSION=${1:?version} +LIBCMARKDIR=${2:?library output} +TARFILE=$3 -# The tarball exists here, or will be downloaded here. -TARDIR="." -if [ "$2" != "" ]; then TARDIR="$2"; fi +if [[ -n $3 ]] +then + mkdir -p $3 + cd $3 +fi ARCHIVES="https://github.com/github/cmark-gfm/archive/refs/tags" -LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" +TARNAME="cmark-gfm.$VERSION.orig.tar.gz" +TARDIR="cmark-gfm-$VERSION" -# WARNING: this must agree with the parent directory in the tar file or the build will fail -EXTRACTED_AS="cmark-gfm-$VERSION" +# Work in a temporary directory +TEMP=$(mktemp -d) -# Follow redirects, and place the result into known name $LOCAL -if [ -f "$LOCAL" ]; then - echo "Using cached tarball: ${LOCAL}" >&2 +if [[ -f $TARFILE ]] +then + echo "Found tar!" + cp $TARFILE $TEMP # do this before cd to allow for relative paths + cd $TEMP else - echo "Fetching $VERSION from cmark archives" >&2 - curl -sSL --fail -o "$LOCAL" "$ARCHIVES/$VERSION.tar.gz" + cd $TEMP + echo "Fetching $VERSION from cmark archives" >&2 + curl -sSL --fail -o "$TARNAME" "$ARCHIVES/$VERSION.tar.gz" fi -# Clean anything old, then extract and build. -### somebody smart could peek into the .tgz. ... MEH -if [ -d "$EXTRACTED_AS" ]; then rm -r "$EXTRACTED_AS"; fi -tar xzf "$LOCAL" -pushd "$EXTRACTED_AS" >/dev/null +tar xzf "$TARNAME" +pushd "$TARDIR" >/dev/null mkdir build pushd build >/dev/null cmake --version >&2 @@ -53,14 +56,6 @@ pushd "$EXTRACTED_AS" >/dev/null } > build.log popd >/dev/null - mkdir lib - cp -Pp build/src/lib* lib/ - cp -Pp build/extensions/lib* lib/ + cp -Pp build/src/lib* ${LIBCMARKDIR}/ + cp -Pp build/extensions/lib* ${LIBCMARKDIR}/ popd >/dev/null - -# These files/dir may need a reference with LD_LIBRARY_PATH. -# gfm.py wants this lib/ in LIBCMARKDIR. -# ls -laF "$EXTRACTED_AS/lib/" - -# Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$(pwd)/$EXTRACTED_AS/lib'"