Unify workflow yamls with pre-processing step

.github/scripts/expand-workflow.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+print_help() {
+    cat <<'EOF'
+expand-includes.sh — simple textual include expander
+
+USAGE:
+  expand-includes.sh <input-file>
+  expand-includes.sh -h | --help
+
+DESCRIPTION:
+  Reads <input-file> and writes the expanded result to stdout.
+
+  Lines matching the following form are replaced by the contents
+  of the referenced file:
+
+      include: "path/to/file.yml.in" # optional comment
+
+  Rules:
+    - The filepath MUST be in double quotes.
+    - Leading whitespace (spaces and/or tabs) is preserved and applied
+      to every line of the included file.
+    - Trailing comments starting with '#' are ignored.
+    - Whitespace around 'include:' and the filename is ignored.
+    - Included files may themselves contain include: directives.
+    - Expansion is purely textual; no YAML parsing is performed.
+
+EOF
+}
+
+expand_file() {
+    local file="$1"
+
+    while IFS= read -r line || [[ -n "$line" ]]; do
+        # Remove trailing comments for matching (keep original for output)
+        local stripped="${line%%#*}"
+
+        # Match YAML include line:
+        #
+        #   <indent>include: "path"
+        #
+        if [[ "$stripped" =~ ^([[:space:]]*)include[[:space:]]*:[[:space:]]*\"([^\"]+)\"[[:space:]]*$ ]]; then
+            local indent="${BASH_REMATCH[1]}"
+            local include_file="${BASH_REMATCH[2]}"
+
+            if [[ ! -f "$include_file" ]]; then
+                echo "ERROR: included file not found: $include_file" >&2
+                return 1
+            fi
+
+            # Recursively expand included file (no subshells)
+            if ! mapfile -t inc_lines < <(expand_file "$include_file"); then
+                return 1
+            fi
+
+            for inc_line in "${inc_lines[@]}"; do
+                printf "%s%s\n" "$indent" "$inc_line"
+            done
+
+        else
+            echo "$line"
+        fi
+    done < "$file"
+}
+
+
+# ---- main ----
+
+if [[ $# -ne 1 ]]; then
+    print_help >&2
+    exit 1
+fi
+
+case "$1" in
+    -h|--help)
+        print_help
+        exit 0
+        ;;
+esac
+
+printf "# Warning: generated file, do not edit. Generated by '%s %s'\n" "$0" "$*"
+
+expand_file "$1"

.github/workflow-src/Makefile

@@ -0,0 +1,21 @@
+SHELL := /usr/bin/env bash
+
+SRC_DIR := .
+OUT_DIR := ../workflows
+EXPAND  := ../scripts/expand-workflow.sh
+
+# All input files (recursively)
+INPUTS := $(shell find $(SRC_DIR) -type f)
+
+# All top-level workflow inputs (one-to-one with outputs)
+WORKFLOW_SRCS := $(wildcard $(SRC_DIR)/*.yml.in)
+WORKFLOWS := $(patsubst $(SRC_DIR)/%.yml.in, $(OUT_DIR)/%.yml, $(WORKFLOW_SRCS))
+
+.PHONY: all
+all: $(WORKFLOWS)
+
+# Rule: output depends on *all* inputs and the expand script
+$(OUT_DIR)/%.yml: $(SRC_DIR)/%.yml.in $(INPUTS) $(EXPAND)
+	@echo "Generating $@ from $<"
+	@mkdir -p $(OUT_DIR)
+	@$(EXPAND) $< > $@

.github/workflow-src/cjose.yml.in

@@ -0,0 +1,46 @@
+name: cjose Tests
+include: "./common/wp-trigger.yml.in"
+
+jobs:
+  include: "./common/wp-build.yml.in"
+
+  test_cjose:
+    include: "./common/wp-testenv.yml.in"
+
+    strategy:
+      matrix:
+        cjose_ref: [ 'v0.6.2.1' ]
+        force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ]
+        include: "./common/wp-matrix.yml.in"
+
+    steps:
+      include: "./common/wp-install.yml.in"
+
+      - name: Install cjose dependencies
+        run: |
+          apt-get update
+          apt-get install -y git build-essential autoconf automake \
+            libtool pkg-config libjansson-dev check ca-certificates dpkg-dev
+
+      - name: Download cjose
+        uses: actions/checkout@v4
+        with:
+          repository: OpenIDC/cjose
+          ref: ${{ matrix.cjose_ref }}
+          path: cjose
+          fetch-depth: 1
+
+      - name: Build cjose
+        working-directory: cjose
+        run: |
+          ./configure CFLAGS="-Wno-error=deprecated-declarations"
+          make
+
+      - name: Run cjose tests
+        working-directory: cjose
+        run: |
+          export ${{ matrix.force_fail }}
+          make test 2>&1 | tee cjose-test.log
+          TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0")
+          echo "TEST_RESULT = $TEST_RESULT"
+          $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} cjose

.github/workflow-src/common/wp-build.yml.in

@@ -0,0 +1,10 @@
+build_wolfprovider:
+  uses: ./.github/workflows/build-wolfprovider.yml
+  with:
+    wolfssl_ref: ${{ matrix.wolfssl_ref }}
+    openssl_ref: ${{ matrix.openssl_ref }}
+    fips_ref: ${{ matrix.fips_ref }}
+    replace_default: ${{ matrix.replace_default }}
+  strategy:
+    matrix:
+      include: "./common/wp-matrix.yml.in"

.github/workflow-src/common/wp-install.yml.in

@@ -0,0 +1,38 @@
+- name: Setup env vars
+  run: |
+    # These paths must match the ones in build-wolfprovider.yml
+    PACKAGES_PATH=/tmp
+    echo "PACKAGES_PATH=$PACKAGES_PATH" >> "$GITHUB_ENV"
+    echo "WOLFSSL_PACKAGES_PATH=\${PACKAGES_PATH}/wolfssl-packages"   >> "$GITHUB_ENV"
+    echo "OPENSSL_PACKAGES_PATH=\${PACKAGES_PATH}/openssl-packages"   >> "$GITHUB_ENV"
+    echo "WOLFPROV_PACKAGES_PATH=\${PACKAGES_PATH}/wolfprov-packages" >> "$GITHUB_ENV"
+
+- name: Checkout wolfProvider
+  uses: actions/checkout@v4
+  with:
+    fetch-depth: 1
+
+- name: Download packages from build job
+  uses: actions/download-artifact@v4
+  with:
+    name: debian-packages-${{ matrix.fips_ref }}${{ matrix.replace_default && '-replace-default' || '' }}-${{ matrix.wolfssl_ref }}-${{ matrix.openssl_ref }}
+    path: ${{ env.PACKAGES_PATH }}
+
+- name: Install wolfSSL/OpenSSL/wolfprov packages
+  run: |
+    apt install --reinstall -y \
+      ${{ env.WOLFSSL_PACKAGES_PATH }}/libwolfssl_*.deb
+
+    apt install --reinstall -y --allow-downgrades --allow-change-held-packages \
+      ${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \
+      ${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \
+      ${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb
+
+    apt install --reinstall -y \
+      ${{ env.WOLFPROV_PACKAGES_PATH }}/libwolfprov_*.deb
+
+- name: Verify wolfProvider is properly installed
+  run: |
+    $GITHUB_WORKSPACE/scripts/verify-install.sh \
+      ${{ matrix.replace_default && '--replace-default' || '' }} \
+      ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }}

.github/workflow-src/common/wp-matrix.yml.in

@@ -0,0 +1,4 @@
+wolfssl_ref: [ 'v5.8.4-stable' ]
+openssl_ref: [ 'openssl-3.5.4' ]
+fips_ref: [ 'FIPS', 'non-FIPS' ]
+replace_default: [ true ]

.github/workflow-src/common/wp-testenv.yml.in

@@ -0,0 +1,10 @@
+runs-on: ubuntu-22.04
+needs: build_wolfprovider
+# Run inside Debian Bookworm to match packaging environment
+container:
+  image: debian:bookworm
+  env:
+    DEBIAN_FRONTEND: noninteractive
+
+# This should be a safe limit for the tests to run.
+timeout-minutes: 20

.github/workflow-src/common/wp-trigger.yml.in

@@ -0,0 +1,9 @@
+on:
+  push:
+    branches: [ 'master', 'main', 'release/**' ]
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true

.github/workflow-src/curl.yml.in

@@ -0,0 +1,57 @@
+name: Curl Tests
+
+include: "./common/wp-trigger.yml.in"
+
+jobs:
+  include: "./common/wp-build.yml.in"
+
+  test_curl:
+    include: "./common/wp-testenv.yml.in"
+
+    strategy:
+      matrix:
+        curl_ref: [ 'curl-8_4_0', 'curl-7_88_1' ]
+        force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ]
+        include: "./common/wp-matrix.yml.in"
+    steps:
+      include: "./common/wp-install.yml.in"
+
+      - name: Install curl dependencies
+        run: |
+          apt-get update
+          apt-get install -y nghttp2 libpsl5 libpsl-dev python3-impacket \
+            build-essential autoconf automake libtool
+
+      - name: Build curl
+        uses: wolfSSL/actions-build-autotools-project@v1
+        with:
+          repository: curl/curl
+          path: curl
+          ref: ${{ matrix.curl_ref }}
+          configure: --with-openssl
+          check: false
+
+      - name: Generate certificates for curl master force-fail tests
+        run: |
+          if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ] &&
+             [ "${{ matrix.curl_ref }}" = "master" ]; then
+            cd curl/tests/certs
+            make test-ca.cacert
+            cd ../..
+          fi
+      - name: Test curl with wolfProvider
+        working-directory: curl
+        shell: bash
+        run: |
+          set +o pipefail # ignore errors from make check
+          export ${{ matrix.force_fail }}
+          export CURL_REF=${{ matrix.curl_ref }}
+
+          # Tests rely on $USER being set
+          export USER=testuser
+
+          # Run tests and save output to test.log
+          make -j$(nproc) test-ci 2>&1 | tee curl-test.log
+          # Capture the test result using PIPESTATUS (Bash only)
+          TEST_RESULT=${PIPESTATUS[0]}
+          $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} curl

.github/workflow-src/readme.md

@@ -0,0 +1,18 @@
+# Github workflow pre-processing
+
+## TLDR
+Run `make` from this directory to regenerate the yamls in the `.github/workflows` directory. Then commit those files without further modification.
+
+## Details
+In order to unify large chunks of code, we pre-process the workflow yamls to insert common blocks of code. 
+
+The files `*.yml.in` are the pre-processed workflow files. Manually edit these, not the generated ones in `.github/workflows`. Run `make` here to regenerate any outdated workflows.
+
+The workflow `generated-workflows.yml` ensures that the pre-processing has been done.
+
+### Syntax
+Designed to be as lightweight and yaml-like as possible, insert the following line structure into a `.yml.in` file.
+
+`  include: "path/to/file.yml.in"`
+
+Any leading whitespace indentation is preserved.

.github/workflows/build-wolfprovider.yml

@@ -15,7 +15,6 @@ on:
       replace_default:
         required: false
         type: boolean
-        default: false
 
 jobs:
   build_wolfprovider_common:
@@ -42,6 +41,7 @@ jobs:
       OPENSSL_PACKAGES_PATH: /tmp/openssl-packages
       WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages
       DEBS_PATH: debs
+      PACKAGE_NAME: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }} 
     steps:
       # Check if artifact already exists from another job in the same workflow run
       # When multiple matrix jobs run in parallel, the first one to finish uploads the artifact
@@ -51,7 +51,7 @@ jobs:
         continue-on-error: true
         uses: actions/download-artifact@v4
         with:
-          name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }}
+          name: ${{ env.PACKAGE_NAME }}
 
       # Download pre-built packages from debs branch
       - name: Checkout debs branch
@@ -199,7 +199,7 @@ jobs:
         if: steps.check_artifact.outcome != 'success'
         uses: actions/upload-artifact@v4
         with:
-          name: debian-packages-${{ inputs.fips_ref }}${{ inputs.replace_default && '-replace-default' || '' }}-${{ inputs.wolfssl_ref }}-${{ inputs.openssl_ref }}
+          name: ${{ env.PACKAGE_NAME }}
           path: |
             ${{ env.WOLFSSL_PACKAGES_PATH }}
             ${{ env.OPENSSL_PACKAGES_PATH }}