From a7637dd3ba2e27db45b4b3bac0bd9c8320064c8e Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 12 Jun 2024 00:07:01 +0100 Subject: [PATCH 01/23] Use artifacts to cache the build --- pelican/action.yml | 47 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index b312633b..3f82d01d 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -51,18 +51,53 @@ runs: # If the site uses Github Flavored Markdown, use this build branch - name: fetch and build libcmark-gfm.so if: ${{ inputs.gfm == 'true' }} + id: build_gfm shell: bash env: WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script + GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call + GH_TOKEN: ${{ github.token }} # needed by gh run: | + # Needs to be available to a subsequent step + echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV + if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists + then + # set up the GFM environment + export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib + echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >>$GITHUB_ENV + fi # Does the GFM build already exist? - if [[ -n $LIBCMARKDIR && -d $LIBCMARKDIR ]] + if [[ -d $LIBCMARKDIR ]] then echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step fi + # Is there a saved artifact for the GFM build? + + REPO=${{ github.repository }} # Check calling repo for artifact + echo "Check for GFM build artifact in current repo: $REPO" + gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $REPO || true + if [[ -d $LIBCMARKDIR ]] + then + echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" + exit 0 # nothing more to do in this step + fi + + if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact + then + echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" + gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $GITHUB_ACTION_REPO || true + if [[ -d $LIBCMARKDIR ]] + then + echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" + exit 0 # nothing more to do in this step + fi + fi + + # GFM binary not found, need to build it { + echo "Creating GFM binary in ${LIBCMARKDIR}" # disable stdout unless debug is on if [ "${{ inputs.debug }}" == 'true' ] then @@ -74,10 +109,16 @@ runs: mkdir -p $WORKDIR pushd $WORKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION + # Message to upload step + echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd - # ensure LIBCMARKDIR is defined for subsequent steps - echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >> $GITHUB_ENV } + - name: Save the GFM build + if: ${{ steps.build_gfm.outputs.created }} + uses: actions/upload-artifact@v4 + with: + name: cmark-gfm-${{ env.GFM_VERSION}} + path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown shell: bash From 8b3e1e67dd13019af8237af429c3d06b49c25bf5 Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 16:02:22 +0100 Subject: [PATCH 02/23] Extract artifact key name (#53) --- pelican/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index 3f82d01d..d3445279 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -59,13 +59,15 @@ runs: GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call GH_TOKEN: ${{ github.token }} # needed by gh run: | + export GFM_ARTIFACT_KEY=cmark-gfm-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV + echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib - echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >>$GITHUB_ENV + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV fi # Does the GFM build already exist? if [[ -d $LIBCMARKDIR ]] @@ -77,7 +79,7 @@ runs: REPO=${{ github.repository }} # Check calling repo for artifact echo "Check for GFM build artifact in current repo: $REPO" - gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $REPO || true + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $REPO || true if [[ -d $LIBCMARKDIR ]] then echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" @@ -87,7 +89,7 @@ runs: if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact then echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" - gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $GITHUB_ACTION_REPO || true + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true if [[ -d $LIBCMARKDIR ]] then echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" @@ -117,7 +119,7 @@ runs: if: ${{ steps.build_gfm.outputs.created }} uses: actions/upload-artifact@v4 with: - name: cmark-gfm-${{ env.GFM_VERSION}} + name: ${{ env.GFM_ARTIFACT_KEY }} path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown From 07645b708e541c63fb745e3b97787c1e4f22f10d Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:13:57 +0100 Subject: [PATCH 03/23] Distinguish the artifact key from directory name --- pelican/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/action.yml b/pelican/action.yml index d3445279..4063af3c 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -59,7 +59,8 @@ runs: GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call GH_TOKEN: ${{ github.token }} # needed by gh run: | - export GFM_ARTIFACT_KEY=cmark-gfm-${GFM_VERSION} + # The key needs to include the GFM version, but does not have to be the same as the directory name + export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV From 1a525a52e9469eff860d315631eaf4309ef7eab6 Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:45:12 +0100 Subject: [PATCH 04/23] Synch with main --- pelican/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pelican/action.yml b/pelican/action.yml index 4063af3c..d096b3b5 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -111,10 +111,14 @@ runs: # Don't pollute site checkout mkdir -p $WORKDIR pushd $WORKDIR - bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION + # build the code and define LIBCMARKDIR + bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ + source /tmp/libcmarkdir.$$ # Message to upload step echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd + # ensure LIBCMARKDIR is defined for subsequent steps + echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV } - name: Save the GFM build if: ${{ steps.build_gfm.outputs.created }} From f11fb702e0c6d235d90a202439c34ad6bde38e1a Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:53:09 +0100 Subject: [PATCH 05/23] Keep synching with main --- pelican/plugins/age_days_lt/__init__.py | 37 ---------- pelican/plugins/age_days_lt/agedayslt.py | 24 ------- pelican/plugins/extract_toc/__init__.py | 1 - pelican/plugins/extract_toc/extract_toc.py | 65 ----------------- pelican/plugins/jinja2content/__init__.py | 1 - .../plugins/jinja2content/jinja2content.py | 67 ------------------ .../plugins/md_inline_extension/__init__.py | 1 - pelican/plugins/md_inline_extension/inline.py | 69 ------------------- .../pelican_inline_markdown_extension.py | 69 ------------------- pelican/plugins/regex_replace/__init__.py | 17 ----- .../plugins/regex_replace/regex_replace.py | 34 --------- pelican/plugins/toc.py | 4 +- pelican/plugins/toc2.py | 2 +- 13 files changed, 3 insertions(+), 388 deletions(-) delete mode 100644 pelican/plugins/age_days_lt/__init__.py delete mode 100644 pelican/plugins/age_days_lt/agedayslt.py delete mode 100644 pelican/plugins/extract_toc/__init__.py delete mode 100644 pelican/plugins/extract_toc/extract_toc.py delete mode 100644 pelican/plugins/jinja2content/__init__.py delete mode 100644 pelican/plugins/jinja2content/jinja2content.py delete mode 100644 pelican/plugins/md_inline_extension/__init__.py delete mode 100644 pelican/plugins/md_inline_extension/inline.py delete mode 100644 pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py delete mode 100644 pelican/plugins/regex_replace/__init__.py delete mode 100644 pelican/plugins/regex_replace/regex_replace.py diff --git a/pelican/plugins/age_days_lt/__init__.py b/pelican/plugins/age_days_lt/__init__.py deleted file mode 100644 index dd32091a..00000000 --- a/pelican/plugins/age_days_lt/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -The age_in_days plugin adds a Jinja test, age_days_lt. - -It is intended to be used in Pelican templates like this to select articles newer than 90 days: - - {% for article in (articles | selectattr("date", "age_days_lt", 90) ) %} - ... - {% endif %} -""" -from pelican import signals -from . import agedayslt - -def add_test(pelican): - """Add age_days_lt test to Pelican.""" - pelican.env.tests.update({'age_days_lt': agedayslt.age_days_lt}) - - -def register(): - """Plugin registration.""" - signals.generator_init.connect(add_test) diff --git a/pelican/plugins/age_days_lt/agedayslt.py b/pelican/plugins/age_days_lt/agedayslt.py deleted file mode 100644 index c191df56..00000000 --- a/pelican/plugins/age_days_lt/agedayslt.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from datetime import datetime - -def age_days_lt(dt, days): - """Return true if a number of days since 'dt' < 'days'""" - now = datetime.now(dt.tzinfo) - delta = now - dt - return delta.days < days diff --git a/pelican/plugins/extract_toc/__init__.py b/pelican/plugins/extract_toc/__init__.py deleted file mode 100644 index 52c57781..00000000 --- a/pelican/plugins/extract_toc/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .extract_toc import * diff --git a/pelican/plugins/extract_toc/extract_toc.py b/pelican/plugins/extract_toc/extract_toc.py deleted file mode 100644 index 210bac84..00000000 --- a/pelican/plugins/extract_toc/extract_toc.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Extract Table of Content -======================== -A Pelican plugin to extract table of contents (ToC) from `article.content` and -place it in its own `article.toc` variable for use in templates. -""" - -from os import path -from bs4 import BeautifulSoup -from pelican import signals, readers, contents -import logging - -logger = logging.getLogger(__name__) - - -def extract_toc(content): - if isinstance(content, contents.Static): - return - - soup = BeautifulSoup(content._content, 'html.parser') # pylint: disable=protected-access - filename = content.source_path - extension = path.splitext(filename)[1][1:] - toc = None - - # default Markdown reader - if not toc and readers.MarkdownReader.enabled and extension in readers.MarkdownReader.file_extensions: - toc = soup.find('div', class_='toc') - if toc: - toc.extract() - if len(toc.find_next('ul').find_all('li')) == 0: - toc = None - - # default reStructuredText reader - if not toc and readers.RstReader.enabled and extension in readers.RstReader.file_extensions: - toc = soup.find('div', class_='contents topic') - if toc: - toc.extract() - tag = BeautifulSoup(str(toc), 'html.parser') - tag.div['class'] = 'toc' - tag.div['id'] = '' - p = tag.find('p', class_='topic-title first') - if p: - p.extract() - toc = tag - - # Pandoc reader (markdown and other formats) - if 'pandoc_reader' in content.settings['PLUGINS']: - try: - from pandoc_reader import PandocReader - except ImportError: - PandocReader = False - if not toc and PandocReader and PandocReader.enabled and extension in PandocReader.file_extensions: - toc = soup.find('nav', id='TOC') - - if toc: - toc.extract() - content._content = soup.decode() # pylint: disable=protected-access - content.toc = toc.decode() - if content.toc.startswith(''): - content.toc = content.toc[12:-14] - - -def register(): - signals.content_object_init.connect(extract_toc) diff --git a/pelican/plugins/jinja2content/__init__.py b/pelican/plugins/jinja2content/__init__.py deleted file mode 100644 index de025c4b..00000000 --- a/pelican/plugins/jinja2content/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .jinja2content import * diff --git a/pelican/plugins/jinja2content/jinja2content.py b/pelican/plugins/jinja2content/jinja2content.py deleted file mode 100644 index 44ff18ba..00000000 --- a/pelican/plugins/jinja2content/jinja2content.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -jinja2content.py ----------------- -Pelican plugin that processes Markdown files as jinja templates. -""" - -from jinja2 import Environment, FileSystemLoader, ChoiceLoader -import os -from pelican import signals -from pelican.readers import MarkdownReader, HTMLReader, RstReader -from pelican.utils import pelican_open -from tempfile import NamedTemporaryFile - -class JinjaContentMixin: - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # will look first in 'JINJA2CONTENT_TEMPLATES', by default the - # content root path, then in the theme's templates - local_dirs = self.settings.get('JINJA2CONTENT_TEMPLATES', ['.']) - local_dirs = [os.path.join(self.settings['PATH'], folder) - for folder in local_dirs] - theme_dir = os.path.join(self.settings['THEME'], 'templates') - - loaders = [FileSystemLoader(_dir) for _dir - in local_dirs + [theme_dir]] - if 'JINJA_ENVIRONMENT' in self.settings: # pelican 3.7 - jinja_environment = self.settings['JINJA_ENVIRONMENT'] - else: - jinja_environment = { - 'trim_blocks': True, - 'lstrip_blocks': True, - 'extensions': self.settings['JINJA_EXTENSIONS'] - } - self.env = Environment( - loader=ChoiceLoader(loaders), - **jinja_environment) - - - def read(self, source_path): - with pelican_open(source_path) as text: - text = self.env.from_string(text).render() - - with NamedTemporaryFile(delete=False) as f: - f.write(text.encode()) - f.close() - content, metadata = super().read(f.name) - os.unlink(f.name) - return content, metadata - - -class JinjaMarkdownReader(JinjaContentMixin, MarkdownReader): - pass - -class JinjaRstReader(JinjaContentMixin, RstReader): - pass - -class JinjaHTMLReader(JinjaContentMixin, HTMLReader): - pass - -def add_reader(readers): - for Reader in [JinjaMarkdownReader, JinjaRstReader, JinjaHTMLReader]: - for ext in Reader.file_extensions: - readers.reader_classes[ext] = Reader - -def register(): - signals.readers_init.connect(add_reader) diff --git a/pelican/plugins/md_inline_extension/__init__.py b/pelican/plugins/md_inline_extension/__init__.py deleted file mode 100644 index 2453fe9d..00000000 --- a/pelican/plugins/md_inline_extension/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .inline import * diff --git a/pelican/plugins/md_inline_extension/inline.py b/pelican/plugins/md_inline_extension/inline.py deleted file mode 100644 index aab0736e..00000000 --- a/pelican/plugins/md_inline_extension/inline.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Markdown Inline Extension For Pelican -===================================== -Extends Pelican's Markdown module -and allows for customized inline HTML -""" - -# mypy: ignore-errors - -import sys - -from pelican import signals - -try: - from . pelican_inline_markdown_extension import PelicanInlineMarkdownExtension -except ImportError as e: - PelicanInlineMarkdownExtension = None - print("\nMarkdown is not installed - inline Markdown extension disabled\n") - -def process_settings(pelicanobj): - """Sets user specified settings (see README for more details)""" - - # Default settings - inline_settings = {} - inline_settings['config'] = {'[]':('', 'pelican-inline')} - - # Get the user specified settings - try: - settings = pelicanobj.settings['MD_INLINE'] - except Exception: # TODO: narrow further to expected Exceptions - settings = None - - # If settings have been specified, add them to the config - if isinstance(settings, dict): - inline_settings['config'].update(settings) - - return inline_settings - -def inline_markdown_extension(pelicanobj, config): - """Instantiates a customized Markdown extension""" - - # Instantiate Markdown extension and append it to the current extensions - try: - if isinstance(pelicanobj.settings.get('MD_EXTENSIONS'), list): # pelican 3.6.3 and earlier - pelicanobj.settings['MD_EXTENSIONS'].append(PelicanInlineMarkdownExtension(config)) - else: - pelicanobj.settings['MARKDOWN'].setdefault('extensions', []).append(PelicanInlineMarkdownExtension(config)) - except Exception: # TODO: narrow further to expected Exceptions - sys.excepthook(*sys.exc_info()) - sys.stderr.write("\nError - the pelican Markdown extension failed to configure. Inline Markdown extension is non-functional.\n") - sys.stderr.flush() - -def pelican_init(pelicanobj): - """Loads settings and instantiates the Python Markdown extension""" - - # If there was an error loading Markdown, then do not process any further - if not PelicanInlineMarkdownExtension: - return - - # Process settings - config = process_settings(pelicanobj) - - # Configure Markdown Extension - inline_markdown_extension(pelicanobj, config) - -def register(): - """Plugin registration""" - signals.initialized.connect(pelican_init) diff --git a/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py b/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py deleted file mode 100644 index 7ef526b5..00000000 --- a/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Pelican Inline Markdown Extension -================================== -An extension for the Python Markdown module that enables -the Pelican Python static site generator to add inline patterns. -""" - -# pylint: skip-file - -import markdown -import re - -class PelicanInlineMarkdownExtensionPattern(markdown.inlinepatterns.Pattern): - """Inline Markdown processing""" - - def __init__(self, pelican_markdown_extension, tag, pattern): - super(PelicanInlineMarkdownExtensionPattern,self).__init__(pattern) - self.tag = tag - self.config = pelican_markdown_extension.getConfig('config') - - def handleMatch(self, m): - node = markdown.util.etree.Element(self.tag) - tag_attributes = self.config.get(m.group('prefix'), ('', 'pelican-inline')) - tag_class = 'pelican-inline' # default class - tag_style = '' # default is for no styling - - if isinstance(tag_attributes, tuple): - tag_style = tag_attributes[0] - tag_class = tag_attributes[1] if len(tag_attributes) > 1 else '' - elif isinstance(tag_attributes, str): - tag_class = tag_attributes - - if tag_class != '': - node.set('class', tag_class) - if tag_style!= '': - node.set('style', tag_style) - - node.text = markdown.util.AtomicString(m.group('text')) - - return node - -class PelicanInlineMarkdownExtension(markdown.Extension): - """A Markdown extension enabling processing in Markdown for Pelican""" - def __init__(self, config): - - try: - # Needed for Markdown versions >= 2.5 - self.config['config'] = ['{}', 'config for markdown extension'] - super(PelicanInlineMarkdownExtension,self).__init__(**config) - except AttributeError: - # Markdown versions < 2.5 - config['config'] = [config['config'], 'config for markdown extension'] - super(PelicanInlineMarkdownExtension, self).__init__(config) - - # N.B. md_globals parameter was dropped: https://github.com/Python-Markdown/markdown/pull/697 - def extendMarkdown(self, md): - # Regex to detect mathjax - config = self.getConfig('config') - patterns = [] - - # The following mathjax settings can be set via the settings dictionary - for key in config: - patterns.append(re.escape(key)) - - inline_regex = r'(?P%s)(?P.+?)\2' % ('|'.join(patterns)) - - # Process after escapes - md.inlinePatterns.add('texthighlight_inlined', PelicanInlineMarkdownExtensionPattern(self, 'span', inline_regex), '>emphasis2') diff --git a/pelican/plugins/regex_replace/__init__.py b/pelican/plugins/regex_replace/__init__.py deleted file mode 100644 index f97c8a3e..00000000 --- a/pelican/plugins/regex_replace/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from .regex_replace import * diff --git a/pelican/plugins/regex_replace/regex_replace.py b/pelican/plugins/regex_replace/regex_replace.py deleted file mode 100644 index bbe544b5..00000000 --- a/pelican/plugins/regex_replace/regex_replace.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Markdown regex_replace filter for pelican -""" -from pelican import signals -import re - -# Custom filter method -def regex_replace(s, find, replace): - return re.sub(find, replace, s) - -def add_filter(pelican): - """Add filter to Pelican.""" - pelican.env.filters.update({'regex_replace': regex_replace}) - -def register(): - """Plugin registration.""" - signals.generator_init.connect(add_filter) diff --git a/pelican/plugins/toc.py b/pelican/plugins/toc.py index 81157e4e..be13485f 100644 --- a/pelican/plugins/toc.py +++ b/pelican/plugins/toc.py @@ -127,7 +127,7 @@ def generate_toc(content): raise e # Find TOC tag - tocTag = soup.find('p', text = '[TOC]') + tocTag = soup.find('p', text='[TOC]') if tocTag: for header in tocTag.findAllNext(header_re): settoc = True @@ -139,7 +139,7 @@ def generate_toc(content): tree_string = '{}'.format(tree) tree_soup = BeautifulSoup(tree_string, 'html.parser') content.toc = tree_soup.decode(formatter='html') - itoc = soup.find('p', text = '[TOC]') + itoc = soup.find('p', text='[TOC]') if itoc: itoc.replaceWith(tree_soup) diff --git a/pelican/plugins/toc2.py b/pelican/plugins/toc2.py index d3287470..0779bda0 100644 --- a/pelican/plugins/toc2.py +++ b/pelican/plugins/toc2.py @@ -138,7 +138,7 @@ def generate_toc(content): if settoc: print("Generating ToC for %s" % content.slug) tree_string = '{}'.format(tree) - print("ToC: %s" % tree_string) + # print("ToC: %s" % tree_string) tree_soup = BeautifulSoup(tree_string, 'html.parser') content.toc = tree_soup.decode(formatter='html') itoc = soup.find('p', text='[TOC]') From 4d99cd3d77a7e9a15c8759e261c700cc24d6f55e Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 23:16:17 +0100 Subject: [PATCH 06/23] Don't assume library name (#55) --- pelican/action.yml | 3 ++- pelican/build-cmark.sh | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index d096b3b5..eea95f68 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -67,7 +67,8 @@ runs: if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment - export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib + NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ + source /tmp/libcmarkdir1.$$ echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV fi # Does the GFM build already exist? diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index c9296dc8..8d8a9001 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -30,6 +30,13 @@ LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" # WARNING: this must agree with the parent directory in the tar file or the build will fail EXTRACTED_AS="cmark-gfm-$VERSION" +LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" + +# Allow caller to find out library directory without needing to build first +if [[ -n $NOBUILD ]] +then + echo "export LIBCMARKDIR='$LIBCMARKDIR'" +fi # Follow redirects, and place the result into known name $LOCAL if [ -f "$LOCAL" ]; then @@ -63,4 +70,4 @@ popd >/dev/null # ls -laF "$EXTRACTED_AS/lib/" # Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$(pwd)/$EXTRACTED_AS/lib'" +echo "export LIBCMARKDIR='$LIBCMARKDIR'" From fdb938f3ad083466bbe2f8de6b9c9498968be241 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 01:23:25 +0100 Subject: [PATCH 07/23] Workflow to create GFM artifact(s) --- .github/workflows/build-artifacts.yml | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/build-artifacts.yml diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml new file mode 100644 index 00000000..996952e6 --- /dev/null +++ b/.github/workflows/build-artifacts.yml @@ -0,0 +1,64 @@ +name: Build GFM artifacts + +# Builds the GFM library as artifacts that can be used by other workflows +# +# It checks for any existing artifacts with an expiry date at least 10 days after today +# If none are found, the library is built and uploaded. + +# This workflow can be run on demand, but is intended to be run on a weekly schedule +# This should ensure that there is always at least one artifact available but no more than 2 + +on: + # Schedule once a week + schedule: + - cron: '11 12 * * 0' + push: + paths: + - '**/build-artifact.yml' + - 'pelican/build-cmark.sh' + workflow_dispatch: + +env: + WORKDIR: /opt/pelican-asf # where to build GFM +jobs: + build-artifact: + runs-on: ubuntu-latest + strategy: + # Allow for multiple versions to be maintained + matrix: + gfm_version: ['0.28.3.gfm.12'] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Fetch and build version ${{ matrix.gfm_version }} + id: build_gfm + run: | + # N.B. This must agree with the definition in pelican/action.yml + export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }} + echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV + # Check if artifact is present (list all) + curl -sS https://api.github.com/repos/sebbASF/infra-actions/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json + # when does last one expire? + jq /tmp/max.txt + # is that more than 10 days away? (86400*10 seconds) + OUT=$(jq (now+864000|strftime("%FT%TZ")))') + if [[ -n $OUT ]] + then + echo "Found a valid artifact for ${{ matrix.gfm_version }} (expires $OUT)" + exit 0 # No more to do + fi + echo "Could not find a valid artifact for ${{ matrix.gfm_version }}; building another" + mkdir -p $WORKDIR + cd $WORKDIR + # build GFM and set up LIBCMARKDIR + bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ + source /tmp/libcmarkdir.$$ + # Tell the save step what to save + echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT + - name: Save the GFM build ${{ matrix.gfm_version}} + if: ${{ steps.build_gfm.outputs.created }} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.GFM_ARTIFACT_KEY}} + path: ${{ steps.build_gfm.outputs.created }} From a6bafc0e6197b806d1efcd4e5e451d531eafe419 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 13:54:42 +0100 Subject: [PATCH 08/23] Opps - actually skip the build (#57) --- pelican/build-cmark.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index 8d8a9001..737991c7 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -36,6 +36,7 @@ LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" if [[ -n $NOBUILD ]] then echo "export LIBCMARKDIR='$LIBCMARKDIR'" + exit # skip the build fi # Follow redirects, and place the result into known name $LOCAL From 3ebd89e274a309221330add7f157c74e3809846c Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 14:28:16 +0100 Subject: [PATCH 09/23] Don't cache artifacts in user repositories (#58) Use artifact from action repo only --- pelican/action.yml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index eea95f68..ebf3e335 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -56,10 +56,13 @@ runs: env: WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script - GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call + # action_repository only works in the env context; empty for local action call + # it is always empty for local invocation, in which case use the current repo + GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }} GH_TOKEN: ${{ github.token }} # needed by gh run: | # The key needs to include the GFM version, but does not have to be the same as the directory name + # must agree with the definition in build-actions.yml export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV @@ -77,28 +80,16 @@ runs: echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step fi - # Is there a saved artifact for the GFM build? - REPO=${{ github.repository }} # Check calling repo for artifact - echo "Check for GFM build artifact in current repo: $REPO" - gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $REPO || true + # Is there a saved artifact for the GFM build? + echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true if [[ -d $LIBCMARKDIR ]] then - echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" + echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" exit 0 # nothing more to do in this step fi - if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact - then - echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" - gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - if [[ -d $LIBCMARKDIR ]] - then - echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" - exit 0 # nothing more to do in this step - fi - fi - # GFM binary not found, need to build it { echo "Creating GFM binary in ${LIBCMARKDIR}" @@ -115,18 +106,10 @@ runs: # build the code and define LIBCMARKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ source /tmp/libcmarkdir.$$ - # Message to upload step - echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd # ensure LIBCMARKDIR is defined for subsequent steps echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV } - - name: Save the GFM build - if: ${{ steps.build_gfm.outputs.created }} - uses: actions/upload-artifact@v4 - with: - name: ${{ env.GFM_ARTIFACT_KEY }} - path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown shell: bash From d81f735a418df4a59630290f293b90e80cfc198b Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 14:40:57 +0100 Subject: [PATCH 10/23] Typo in workflow name --- .github/workflows/build-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 996952e6..379ea595 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -14,7 +14,7 @@ on: - cron: '11 12 * * 0' push: paths: - - '**/build-artifact.yml' + - '**/build-artifacts.yml' # self - 'pelican/build-cmark.sh' workflow_dispatch: From 82760d9a77298b6f6f0b5039e9c4dfb1e92daf37 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 15:11:45 +0100 Subject: [PATCH 11/23] Download the artifact to the work dir (#59) Ensure LIBCMARKDIR uses WORKDIR --- pelican/action.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index ebf3e335..0c19ec07 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -70,9 +70,13 @@ runs: if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment - NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ - source /tmp/libcmarkdir1.$$ - echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV + # Don't pollute site checkout + mkdir -p $WORKDIR + pushd $WORKDIR >/dev/null # ensure LIBCMARKDIR is under WORKDIR + NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ + source /tmp/libcmarkdir1.$$ + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV + popd >/dev/null fi # Does the GFM build already exist? if [[ -d $LIBCMARKDIR ]] From 27bd12b034e353abebe34d977d885da0920dd485 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 22:47:44 +0100 Subject: [PATCH 12/23] Simplify build command Move WORKDIR pushd/popd to build script --- .github/workflows/build-artifacts.yml | 10 ++++++---- pelican/build-cmark.sh | 22 +++++++++++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 379ea595..d27b9ab5 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -37,6 +37,7 @@ jobs: # N.B. This must agree with the definition in pelican/action.yml export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }} echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV + # Check if artifact is present (list all) curl -sS https://api.github.com/repos/sebbASF/infra-actions/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json # when does last one expire? @@ -48,12 +49,13 @@ jobs: echo "Found a valid artifact for ${{ matrix.gfm_version }} (expires $OUT)" exit 0 # No more to do fi + echo "Could not find a valid artifact for ${{ matrix.gfm_version }}; building another" - mkdir -p $WORKDIR - cd $WORKDIR - # build GFM and set up LIBCMARKDIR - bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ + # build GFM and set up LIBCMARKDIR in $WORDIR + bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} $WORKDIR $WORKDIR |\ + grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ source /tmp/libcmarkdir.$$ + # Tell the save step what to save echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT - name: Save the GFM build ${{ matrix.gfm_version}} diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index c9296dc8..27a974da 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -5,11 +5,13 @@ # The binary output will be under: cmark-gfm-$VERSION/lib # # USAGE: -# $ build-cmark.sh [ VERSION [ TARDIR ] ] +# $ build-cmark.sh [ VERSION [ TARDIR [WORKDIR] ] ] # # VERSION: defaults to 0.28.3.gfm.12 # TARDIR: where to find a downloaded/cached tarball of the cmark -# code, or where to place a tarball +# code, or where to place a tarball (defaults to .) +# WORKDIR: where to extract the source and build the code (defaults to .) +# If specified, the directory will be created if necessary. # # Echo all of our steps if DEBUG_STEPS is set @@ -25,11 +27,25 @@ if [ "$1" != "" ]; then VERSION="$1"; fi TARDIR="." if [ "$2" != "" ]; then TARDIR="$2"; fi +if [[ -n $3 ]] +then + mkdir -p $3 + cd $3 +fi + ARCHIVES="https://github.com/github/cmark-gfm/archive/refs/tags" LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" # WARNING: this must agree with the parent directory in the tar file or the build will fail EXTRACTED_AS="cmark-gfm-$VERSION" +LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" + +# Allow caller to find out library directory without needing to build first +if [[ -n $NOBUILD ]] +then + echo "export LIBCMARKDIR='$LIBCMARKDIR'" + exit # skip the build +fi # Follow redirects, and place the result into known name $LOCAL if [ -f "$LOCAL" ]; then @@ -63,4 +79,4 @@ popd >/dev/null # ls -laF "$EXTRACTED_AS/lib/" # Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$(pwd)/$EXTRACTED_AS/lib'" +echo "export LIBCMARKDIR='$LIBCMARKDIR'" From f5246ff0fb425677b5109c7540d2276d34fcd5e2 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 00:56:54 +0100 Subject: [PATCH 13/23] Artifact caching (#60) --- pelican/Dockerfile | 20 +++++++++---- pelican/action.yml | 39 +++++++++--------------- pelican/build-cmark.sh | 67 +++++++++++++++--------------------------- 3 files changed, 53 insertions(+), 73 deletions(-) diff --git a/pelican/Dockerfile b/pelican/Dockerfile index 931b2f49..b895c592 100644 --- a/pelican/Dockerfile +++ b/pelican/Dockerfile @@ -1,11 +1,16 @@ # Settings # ======== # Use the Python version as installed on CI pelican builders (2023-06-02) -ARG PYTHON_VERSION=3.8.10 -ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below +ARG PYTHON_VERSION=3.8.10 # must agree with the copy below + +# Note: ARG scope ends at the FROM statement, so must be repeated if necessary # Build cmake-gfm FROM python:${PYTHON_VERSION}-slim-buster as pelican-asf +#======================================================= + +ARG PYTHON_VERSION=3.8.10 # must agree with the copy above +ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below RUN apt update && apt upgrade -y RUN apt install curl cmake build-essential -y @@ -15,11 +20,15 @@ WORKDIR /opt/pelican-asf # Copy only what we need to build cmark-gfm COPY build-cmark.sh bin/build-cmark.sh -# build it -RUN bash bin/build-cmark.sh ${GFM_VERSION} +# build GFM +# Must agree with definition below +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib +RUN mkdir -p ${LIBCMARKDIR} +RUN bash bin/build-cmark.sh ${GFM_VERSION} ${LIBCMARKDIR} # rebase the image to save on image size FROM python:${PYTHON_VERSION}-slim-buster +#======================================================= # Use the Pelican version as installed on CI pelican builders (2023-06-02) ARG PELICAN_VERSION=4.5.4 @@ -27,7 +36,8 @@ ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy above # Where we put GFM and the plugins ENV WORKDIR /opt/pelican-asf -ENV LIBCMARKDIR ${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib +# Must agree with definition above +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib RUN apt update && apt upgrade -y diff --git a/pelican/action.yml b/pelican/action.yml index 0c19ec07..934b4f9e 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -54,32 +54,26 @@ runs: id: build_gfm shell: bash env: - WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script # action_repository only works in the env context; empty for local action call # it is always empty for local invocation, in which case use the current repo GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }} GH_TOKEN: ${{ github.token }} # needed by gh run: | - # The key needs to include the GFM version, but does not have to be the same as the directory name - # must agree with the definition in build-actions.yml + # The key needs to include the GFM version, but is otherwise arbitrary. + # It must agree with the definition in build-actions.yml export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} - # Needs to be available to a subsequent step - echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV - echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV - if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists + + if [[ -z $LIBCMARKDIR ]] # define LIBCMARKDIR if it is not already then # set up the GFM environment - # Don't pollute site checkout - mkdir -p $WORKDIR - pushd $WORKDIR >/dev/null # ensure LIBCMARKDIR is under WORKDIR - NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ - source /tmp/libcmarkdir1.$$ - echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV - popd >/dev/null + export LIBCMARKDIR=/opt/pelican-asf/gfm-${GFM_VERSION} # arbitrary, but should contain version + mkdir -p $LIBCMARKDIR + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step fi + ls -l $LIBCMARKDIR # Does the GFM build already exist? - if [[ -d $LIBCMARKDIR ]] + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step @@ -88,7 +82,8 @@ runs: # Is there a saved artifact for the GFM build? echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - if [[ -d $LIBCMARKDIR ]] + ls -l $LIBCMARKDIR + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" exit 0 # nothing more to do in this step @@ -104,16 +99,10 @@ runs: else exec >/dev/null fi - # Don't pollute site checkout - mkdir -p $WORKDIR - pushd $WORKDIR - # build the code and define LIBCMARKDIR - bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ - source /tmp/libcmarkdir.$$ - popd - # ensure LIBCMARKDIR is defined for subsequent steps - echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV + # build the code and define LIBCMARKDIR under $WORKDIR + bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR } + ls -l $LIBCMARKDIR - name: Generate website from markdown shell: bash diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index 737991c7..aa50f086 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -1,15 +1,15 @@ #!/bin/bash # -# Build the cmark-gfm library and extensions within CURRENT DIRECTORY. +# Build the cmark-gfm library and extensions in a temporary directory # -# The binary output will be under: cmark-gfm-$VERSION/lib +# The binary output will be under: LIBCMARKDIR # # USAGE: -# $ build-cmark.sh [ VERSION [ TARDIR ] ] +# $ build-cmark.sh VERSION LIBCMARKDIR [TARFILE] # -# VERSION: defaults to 0.28.3.gfm.12 -# TARDIR: where to find a downloaded/cached tarball of the cmark -# code, or where to place a tarball +# VERSION: e.g. 0.28.3.gfm.12 +# LIBCMARKDIR: where to put the binary library files +# TARFILE: local copy of the tarfile; must be for the correct version! (optional) # # Echo all of our steps if DEBUG_STEPS is set @@ -17,41 +17,30 @@ test -n "$DEBUG_STEPS" && set -x set -e # early exit if any step fails -#VERSION=0.28.3.gfm.20 ### not yet -VERSION=0.28.3.gfm.12 -if [ "$1" != "" ]; then VERSION="$1"; fi - -# The tarball exists here, or will be downloaded here. -TARDIR="." -if [ "$2" != "" ]; then TARDIR="$2"; fi +VERSION=${1:?version} +LIBCMARKDIR=${2:?library output} +TARFILE=$3 ARCHIVES="https://github.com/github/cmark-gfm/archive/refs/tags" -LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" +TARNAME="cmark-gfm.$VERSION.orig.tar.gz" +TARDIR="cmark-gfm-$VERSION" -# WARNING: this must agree with the parent directory in the tar file or the build will fail -EXTRACTED_AS="cmark-gfm-$VERSION" -LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" +# Work in a temporary directory +TEMP=$(mktemp -d) -# Allow caller to find out library directory without needing to build first -if [[ -n $NOBUILD ]] +if [[ -f $TARFILE ]] then - echo "export LIBCMARKDIR='$LIBCMARKDIR'" - exit # skip the build -fi - -# Follow redirects, and place the result into known name $LOCAL -if [ -f "$LOCAL" ]; then - echo "Using cached tarball: ${LOCAL}" >&2 + echo "Found tar!" + cp $TARFILE $TEMP # do this before cd to allow for relative paths + cd $TEMP else - echo "Fetching $VERSION from cmark archives" >&2 - curl -sSL --fail -o "$LOCAL" "$ARCHIVES/$VERSION.tar.gz" + cd $TEMP + echo "Fetching $VERSION from cmark archives" >&2 + curl -sSL --fail -o "$TARNAME" "$ARCHIVES/$VERSION.tar.gz" fi -# Clean anything old, then extract and build. -### somebody smart could peek into the .tgz. ... MEH -if [ -d "$EXTRACTED_AS" ]; then rm -r "$EXTRACTED_AS"; fi -tar xzf "$LOCAL" -pushd "$EXTRACTED_AS" >/dev/null +tar xzf "$TARNAME" +pushd "$TARDIR" >/dev/null mkdir build pushd build >/dev/null cmake --version >&2 @@ -61,14 +50,6 @@ pushd "$EXTRACTED_AS" >/dev/null } > build.log popd >/dev/null - mkdir lib - cp -Pp build/src/lib* lib/ - cp -Pp build/extensions/lib* lib/ + cp -Pp build/src/lib* ${LIBCMARKDIR}/ + cp -Pp build/extensions/lib* ${LIBCMARKDIR}/ popd >/dev/null - -# These files/dir may need a reference with LD_LIBRARY_PATH. -# gfm.py wants this lib/ in LIBCMARKDIR. -# ls -laF "$EXTRACTED_AS/lib/" - -# Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$LIBCMARKDIR'" From 79a054353328d361b3d028becd216ba4f350e77e Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 01:01:30 +0100 Subject: [PATCH 14/23] Drop debug (#61) --- pelican/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index 934b4f9e..d1832d4a 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -71,7 +71,7 @@ runs: mkdir -p $LIBCMARKDIR echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step fi - ls -l $LIBCMARKDIR + # Does the GFM build already exist? if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then @@ -82,7 +82,6 @@ runs: # Is there a saved artifact for the GFM build? echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - ls -l $LIBCMARKDIR if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" @@ -102,7 +101,6 @@ runs: # build the code and define LIBCMARKDIR under $WORKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR } - ls -l $LIBCMARKDIR - name: Generate website from markdown shell: bash From 4fd637e91f23b70d2ca98005e7004f46d55246a2 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 01:09:26 +0100 Subject: [PATCH 15/23] Rework for updated build-cmark script --- .github/workflows/build-artifacts.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index d27b9ab5..d403ed84 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -51,10 +51,10 @@ jobs: fi echo "Could not find a valid artifact for ${{ matrix.gfm_version }}; building another" - # build GFM and set up LIBCMARKDIR in $WORDIR - bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} $WORKDIR $WORKDIR |\ - grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ - source /tmp/libcmarkdir.$$ + # build GFM and set up LIBCMARKDIR + export LIBCMARKDIR=/tmp/gfm-${{ matrix.gfm_version }} + mkdir -p ${LIBCMARKDIR} + bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} ${LIBCMARKDIR} # Tell the save step what to save echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT From 7250700e30733b1cbfba01a0ec69c5b88be2a761 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 01:13:26 +0100 Subject: [PATCH 16/23] TEST - force rebuild --- .github/workflows/build-artifacts.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index d403ed84..b0b283ac 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -18,8 +18,6 @@ on: - 'pelican/build-cmark.sh' workflow_dispatch: -env: - WORKDIR: /opt/pelican-asf # where to build GFM jobs: build-artifact: runs-on: ubuntu-latest @@ -35,7 +33,7 @@ jobs: id: build_gfm run: | # N.B. This must agree with the definition in pelican/action.yml - export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }} + export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }}-HACK echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV # Check if artifact is present (list all) From b291ad17dfc86f81bda040e0de2e7a15d71f05f1 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 01:14:24 +0100 Subject: [PATCH 17/23] Revert test --- .github/workflows/build-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index b0b283ac..7c65becf 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -33,7 +33,7 @@ jobs: id: build_gfm run: | # N.B. This must agree with the definition in pelican/action.yml - export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }}-HACK + export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }} echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV # Check if artifact is present (list all) From c4a18f796e9c6d63dd8dc1c48684772caa484293 Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:38:21 +0000 Subject: [PATCH 18/23] Update .github/workflows/build-artifacts.yml Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Signed-off-by: Sebb --- .github/workflows/build-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 7c65becf..09b3a99c 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -60,5 +60,5 @@ jobs: if: ${{ steps.build_gfm.outputs.created }} uses: actions/upload-artifact@v4 with: - name: ${{ env.GFM_ARTIFACT_KEY}} + name: ${{ env.GFM_ARTIFACT_KEY }} path: ${{ steps.build_gfm.outputs.created }} From 03b50bbdf85cad28b7b1956ece01a5323a6704c2 Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:38:41 +0000 Subject: [PATCH 19/23] Update .github/workflows/build-artifacts.yml Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Signed-off-by: Sebb --- .github/workflows/build-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 09b3a99c..cbf5cf03 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -56,7 +56,7 @@ jobs: # Tell the save step what to save echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT - - name: Save the GFM build ${{ matrix.gfm_version}} + - name: Save the GFM build ${{ matrix.gfm_version }} if: ${{ steps.build_gfm.outputs.created }} uses: actions/upload-artifact@v4 with: From f71bf333627442c8d83367a3aaad88034de8ad6d Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:38:55 +0000 Subject: [PATCH 20/23] Update .github/workflows/build-artifacts.yml Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Signed-off-by: Sebb --- .github/workflows/build-artifacts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index cbf5cf03..7db5c888 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -24,7 +24,8 @@ jobs: strategy: # Allow for multiple versions to be maintained matrix: - gfm_version: ['0.28.3.gfm.12'] + gfm_version: + - '0.28.3.gfm.12' steps: - uses: actions/checkout@v4 with: From c7426f7918a0eeb5203ea006679e586c29f5e8b4 Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:39:13 +0000 Subject: [PATCH 21/23] Update .github/workflows/build-artifacts.yml Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Signed-off-by: Sebb --- .github/workflows/build-artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 7db5c888..f83c2daf 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -38,7 +38,7 @@ jobs: echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV # Check if artifact is present (list all) - curl -sS https://api.github.com/repos/sebbASF/infra-actions/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json + curl -sS https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json # when does last one expire? jq /tmp/max.txt # is that more than 10 days away? (86400*10 seconds) From 3d03a549ce28b587dc8fa0f6de92194354fee505 Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:44:06 +0000 Subject: [PATCH 22/23] Don't use bare variables from matrix in run blocks --- .github/workflows/build-artifacts.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index f83c2daf..c6725aff 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -32,9 +32,11 @@ jobs: persist-credentials: false - name: Fetch and build version ${{ matrix.gfm_version }} id: build_gfm + env: + gfm_version: ${{ matrix.gfm_version }} run: | # N.B. This must agree with the definition in pelican/action.yml - export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }} + export GFM_ARTIFACT_KEY=gfm-lib-${gfm_version} echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV # Check if artifact is present (list all) @@ -45,15 +47,15 @@ jobs: OUT=$(jq (now+864000|strftime("%FT%TZ")))') if [[ -n $OUT ]] then - echo "Found a valid artifact for ${{ matrix.gfm_version }} (expires $OUT)" + echo "Found a valid artifact for ${gfm_version} (expires $OUT)" exit 0 # No more to do fi - echo "Could not find a valid artifact for ${{ matrix.gfm_version }}; building another" + echo "Could not find a valid artifact for ${gfm_version}; building another" # build GFM and set up LIBCMARKDIR - export LIBCMARKDIR=/tmp/gfm-${{ matrix.gfm_version }} + export LIBCMARKDIR=/tmp/gfm-${gfm_version} mkdir -p ${LIBCMARKDIR} - bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} ${LIBCMARKDIR} + bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${gfm_version} ${LIBCMARKDIR} # Tell the save step what to save echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT From 852cdd5c9e688934cf83593a84583a0de60fce1b Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 5 Feb 2025 21:51:29 +0000 Subject: [PATCH 23/23] Add restrictive permission --- .github/workflows/build-artifacts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index c6725aff..d64fb43f 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -18,6 +18,9 @@ on: - 'pelican/build-cmark.sh' workflow_dispatch: +permissions: + content: read + jobs: build-artifact: runs-on: ubuntu-latest