Workflow to create GFM artifact(s)

.github/workflows/build-artifacts.yml

@@ -0,0 +1,65 @@
+name: Build GFM artifacts
+
+# Builds the GFM library as artifacts that can be used by other workflows
+#
+# It checks for any existing artifacts with an expiry date at least 10 days after today
+# If none are found, the library is built and uploaded.
+
+# This workflow can be run on demand, but is intended to be run on a weekly schedule
+# This should ensure that there is always at least one artifact available but no more than 2
+
+on:
+  # Schedule once a week
+  schedule:
+    - cron: '11 12 * * 0'
+  push:
+    paths:
+      - '**/build-artifacts.yml' # self
+      - 'pelican/build-cmark.sh'
+  workflow_dispatch:
+
+jobs:
+  build-artifact:
+    runs-on: ubuntu-latest
+    strategy:
+      # Allow for multiple versions to be maintained
+      matrix:
+        gfm_version:
+        - '0.28.3.gfm.12'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Fetch and build version ${{ matrix.gfm_version }}
+      id: build_gfm
+      run: |
+        # N.B. This must agree with the definition in pelican/action.yml
+        export GFM_ARTIFACT_KEY=gfm-lib-${{ matrix.gfm_version }}
+        echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >> $GITHUB_ENV
+
+        # Check if artifact is present (list all)
+        curl -sS https://api.github.com/repos/$GITHUB_REPOSITORY/actions/artifacts?name=${GFM_ARTIFACT_KEY} >/tmp/artifact.json
+        # when does last one expire?
+        jq </tmp/artifact.json '[.artifacts[]|select(.expired==false)|.expires_at]|max' >/tmp/max.txt
+        # is that more than 10 days away? (86400*10 seconds)
+        OUT=$(jq </tmp/max.txt 'select(.>(now+864000|strftime("%FT%TZ")))')
+        if [[ -n $OUT ]]
+        then
+          echo "Found a valid artifact for ${{ matrix.gfm_version }} (expires $OUT)"
+          exit 0 # No more to do
+        fi
+
+        echo "Could not find a valid artifact for ${{ matrix.gfm_version }}; building another"
+        # build GFM and set up LIBCMARKDIR
+        export LIBCMARKDIR=/tmp/gfm-${{ matrix.gfm_version }}
+        mkdir -p ${LIBCMARKDIR}
+        bash $GITHUB_WORKSPACE/pelican/build-cmark.sh ${{ matrix.gfm_version }} ${LIBCMARKDIR}
+
+        # Tell the save step what to save
+        echo "created=${LIBCMARKDIR}" | tee -a $GITHUB_OUTPUT
+    - name: Save the GFM build ${{ matrix.gfm_version }}
+      if: ${{ steps.build_gfm.outputs.created }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.GFM_ARTIFACT_KEY }}
+        path: ${{ steps.build_gfm.outputs.created }}

pelican/Dockerfile

@@ -1,11 +1,16 @@
 # Settings
 # ========
 # Use the Python version as installed on CI pelican builders (2023-06-02)
-ARG PYTHON_VERSION=3.8.10
-ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below
+ARG PYTHON_VERSION=3.8.10 # must agree with the copy below
+
+# Note: ARG scope ends at the FROM statement, so must be repeated if necessary
 
 # Build cmake-gfm
 FROM python:${PYTHON_VERSION}-slim-buster as pelican-asf
+#=======================================================
+
+ARG PYTHON_VERSION=3.8.10 # must agree with the copy above
+ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below
 
 RUN apt update && apt upgrade -y
 RUN apt install curl cmake build-essential -y
@@ -15,19 +20,24 @@ WORKDIR /opt/pelican-asf
 # Copy only what we need to build cmark-gfm
 COPY build-cmark.sh bin/build-cmark.sh
 
-# build it
-RUN bash bin/build-cmark.sh ${GFM_VERSION}
+# build GFM
+# Must agree with definition below
+ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib
+RUN mkdir -p ${LIBCMARKDIR}
+RUN bash bin/build-cmark.sh ${GFM_VERSION} ${LIBCMARKDIR}
 
 # rebase the image to save on image size
 FROM python:${PYTHON_VERSION}-slim-buster
+#=======================================================
 
 # Use the Pelican version as installed on CI pelican builders (2023-06-02)
 ARG PELICAN_VERSION=4.5.4
 ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy above
 
 # Where we put GFM and the plugins
 ENV WORKDIR /opt/pelican-asf
-ENV LIBCMARKDIR ${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib
+# Must agree with definition above
+ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib
 
 RUN apt update && apt upgrade -y
 

pelican/action.yml

@@ -51,34 +51,55 @@ runs:
     # If the site uses Github Flavored Markdown, use this build branch
     - name: fetch and build libcmark-gfm.so
       if: ${{ inputs.gfm == 'true' }}
+      id: build_gfm
       shell: bash
       env:
-        WORKDIR: /opt/pelican-asf # where to build GFM
         GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script
+        # action_repository only works in the env context; empty for local action call
+        # it is always empty for local invocation, in which case use the current repo
+        GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }}
+        GH_TOKEN: ${{ github.token }} # needed by gh
       run: |
+        # The key needs to include the GFM version, but is otherwise arbitrary.
+        # It must agree with the definition in build-actions.yml
+        export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION}
+
+        if [[ -z $LIBCMARKDIR ]] # define LIBCMARKDIR if it is not already
+        then
+          # set up the GFM environment
+          export LIBCMARKDIR=/opt/pelican-asf/gfm-${GFM_VERSION} # arbitrary, but should contain version
+          mkdir -p $LIBCMARKDIR
+          echo  "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step
+        fi
+
         # Does the GFM build already exist?
-        if [[ -n $LIBCMARKDIR && -d $LIBCMARKDIR ]]
+        if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]]
         then
           echo "Already have GFM binary at $LIBCMARKDIR, skipping build"
           exit 0 # nothing more to do in this step
         fi
+
+        # Is there a saved artifact for the GFM build?
+        echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO"
+        gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true
+        if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]]
+        then
+          echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!"
+          exit 0 # nothing more to do in this step
+        fi
+
+        # GFM binary not found, need to build it
         {
+          echo "Creating GFM binary in ${LIBCMARKDIR}"
           # disable stdout unless debug is on
           if [ "${{ inputs.debug }}" == 'true' ]
           then
             DEBUG_STEPS=1; export DEBUG_STEPS
           else
             exec >/dev/null
           fi
-          # Don't pollute site checkout
-          mkdir -p $WORKDIR
-          pushd $WORKDIR
-            # build the code and define LIBCMARKDIR
-            bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$
-            source /tmp/libcmarkdir.$$
-          popd
-          # ensure LIBCMARKDIR is defined for subsequent steps
-          echo "LIBCMARKDIR=${LIBCMARKDIR}"  >> $GITHUB_ENV
+          # build the code and define LIBCMARKDIR under $WORKDIR
+          bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR
         }
 
     - name: Generate website from markdown

pelican/build-cmark.sh

@@ -1,49 +1,52 @@
 #!/bin/bash
 #
-# Build the cmark-gfm library and extensions within CURRENT DIRECTORY.
+# Build the cmark-gfm library and extensions in a temporary directory
 # 
-# The binary output will be under: cmark-gfm-$VERSION/lib
+# The binary output will be under: LIBCMARKDIR
 # 
 # USAGE:
-#   $ build-cmark.sh [ VERSION [ TARDIR ] ]
+#   $ build-cmark.sh VERSION LIBCMARKDIR [TARFILE]
 #
-#   VERSION: defaults to 0.28.3.gfm.12
-#   TARDIR: where to find a downloaded/cached tarball of the cmark
-#           code, or where to place a tarball
+#   VERSION: e.g. 0.28.3.gfm.12
+#   LIBCMARKDIR: where to put the binary library files
+#   TARFILE: local copy of the tarfile; must be for the correct version! (optional)
 #
 
 # Echo all of our steps if DEBUG_STEPS is set
 test -n "$DEBUG_STEPS" && set -x
 
 set -e # early exit if any step fails
 
-#VERSION=0.28.3.gfm.20  ### not yet
-VERSION=0.28.3.gfm.12
-if [ "$1" != "" ]; then VERSION="$1"; fi
+VERSION=${1:?version}
+LIBCMARKDIR=${2:?library output}
+TARFILE=$3
 
-# The tarball exists here, or will be downloaded here.
-TARDIR="."
-if [ "$2" != "" ]; then TARDIR="$2"; fi
+if [[ -n $3 ]]
+then 
+  mkdir -p $3
+  cd $3
+fi
 
 ARCHIVES="https://github.com/github/cmark-gfm/archive/refs/tags"
-LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz"
+TARNAME="cmark-gfm.$VERSION.orig.tar.gz"
+TARDIR="cmark-gfm-$VERSION"
 
-# WARNING: this must agree with the parent directory in the tar file or the build will fail
-EXTRACTED_AS="cmark-gfm-$VERSION"
+# Work in a temporary directory
+TEMP=$(mktemp -d)
 
-# Follow redirects, and place the result into known name $LOCAL
-if [ -f "$LOCAL" ]; then
-    echo "Using cached tarball: ${LOCAL}" >&2
+if [[ -f $TARFILE ]]
+then
+  echo "Found tar!"
+  cp $TARFILE $TEMP # do this before cd to allow for relative paths
+  cd $TEMP
 else
-    echo "Fetching $VERSION from cmark archives" >&2
-    curl -sSL --fail -o "$LOCAL" "$ARCHIVES/$VERSION.tar.gz"
+  cd $TEMP
+  echo "Fetching $VERSION from cmark archives" >&2
+  curl -sSL --fail -o "$TARNAME" "$ARCHIVES/$VERSION.tar.gz"
 fi
 
-# Clean anything old, then extract and build.
-### somebody smart could peek into the .tgz. ... MEH
-if [ -d "$EXTRACTED_AS" ]; then rm -r "$EXTRACTED_AS"; fi
-tar xzf "$LOCAL"
-pushd "$EXTRACTED_AS" >/dev/null
+tar xzf "$TARNAME"
+pushd "$TARDIR" >/dev/null
   mkdir build
   pushd build >/dev/null
     cmake --version >&2
@@ -53,14 +56,6 @@ pushd "$EXTRACTED_AS" >/dev/null
     } > build.log
   popd >/dev/null
 
-  mkdir lib
-  cp -Pp build/src/lib* lib/
-  cp -Pp build/extensions/lib* lib/
+  cp -Pp build/src/lib* ${LIBCMARKDIR}/
+  cp -Pp build/extensions/lib* ${LIBCMARKDIR}/
 popd >/dev/null
-
-# These files/dir may need a reference with LD_LIBRARY_PATH.
-# gfm.py wants this lib/ in LIBCMARKDIR.
-# ls -laF "$EXTRACTED_AS/lib/"
-
-# Provide a handy line for copy/paste.
-echo "export LIBCMARKDIR='$(pwd)/$EXTRACTED_AS/lib'"