spc-download #407
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: spc-download | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| force_full: | |
| description: "Force a full rebuild of all PHP versions regardless of detected changes" | |
| type: boolean | |
| required: false | |
| default: false | |
| permissions: | |
| contents: read | |
| actions: write | |
| jobs: | |
| download: | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| actions: write | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| PHP_MINORS: "8.2 8.3 8.4 8.5 8.6" | |
| steps: | |
| - name: Install PHP and composer | |
| run: | | |
| sudo curl -L https://files.henderkes.com/x86_64-linux/php -o /usr/local/bin/php | |
| sudo chmod +x /usr/local/bin/php | |
| sudo curl -sS https://raw.githubusercontent.com/composer/getcomposer.org/f3108f64b4e1c1ce6eb462b159956461592b3e3e/web/installer | php -- --quiet | |
| sudo mv composer.phar /usr/local/bin/composer | |
| - name: Checkout code | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| persist-credentials: false | |
| - name: Composer install | |
| run: composer install | |
| - name: Restore previous state | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .download-state | |
| key: spc-downloads-lock | |
| - name: Seed empty state if missing | |
| run: | | |
| set -euo pipefail | |
| mkdir -p .download-state | |
| [[ -f .download-state/lock.prev.json ]] || echo '{}' > .download-state/lock.prev.json | |
| [[ -f .download-state/meta.prev.json ]] || echo '{}' > .download-state/meta.prev.json | |
| - name: Download extensions | |
| run: | | |
| set -uo pipefail | |
| EXTS=amqp,apcu,ast,bcmath,brotli,bz2,calendar,clickhouse,ctype,curl,dba,decimal,deepclone,dio,dom,ds,ev,event,excimer,exif,fastchart,fastjson,ffi,fileinfo,filter,ftp,gd,gearman,gettext,gmp,gmssl,grpc,iconv,igbinary,imagick,inotify,intl,ldap,libxml,lz4,maxminddb,mbregex,mbstring,memcache,memcached,mongodb,msgpack,mysqli,mysqlnd,mysqlnd_parsec,mysqlnd_ed25519,odbc,opcache,openssl,opentelemetry,parallel,password-argon2,pcov,pcntl,pdo,pdo_mysql,pdo_odbc,pdo_pgsql,pdo_sqlite,pdo_sqlsrv,pgsql,phar,posix,protobuf,rar,rdkafka,readline,redis,session,shmop,simdjson,simplexml,snappy,soap,sockets,sodium,spx,sqlite3,sqlsrv,ssh2,swoole,sysvmsg,sysvsem,sysvshm,tidy,tokenizer,trader,uuid,uv,xdebug,xhprof,xlswriter,xml,xmlreader,xmlwriter,xsl,xz,yac,yaml,zip,zlib,zstd | |
| # confluentinc (librdkafka's GitHub org) has an IP allow list enabled. | |
| # It rejects AUTHENTICATED api.github.com requests coming from GitHub | |
| # Actions runner IPs with HTTP 403 ("...IP address is not permitted..."), | |
| # while ANONYMOUS requests to the same public endpoints return 200. spc | |
| # always attaches GITHUB_TOKEN to GitHub sources, so its librdkafka fetch | |
| # 403s. Fetch librdkafka first WITHOUT a token; the main authenticated | |
| # pass below then finds it cached and skips the blocked API call. | |
| # (curl --retry does not retry 403, so this must be fixed, not retried.) | |
| if ! env -u GITHUB_TOKEN -u GH_TOKEN php vendor/bin/spc download librdkafka --retry=5; then | |
| echo "::error::failed to fetch librdkafka anonymously" | |
| exit 1 | |
| fi | |
| # spc's per-request retry uses `curl --retry`, which does not retry HTTP | |
| # 403. Wrap the main download in an outer retry-with-backoff loop; spc | |
| # skips already-cached sources, so each retry only re-fetches the artifacts | |
| # that failed and gives transient rate-limit windows time to reset. | |
| attempt=1 | |
| max=6 | |
| until php vendor/bin/spc download --shallow-clone --retry=5 -e "$EXTS" --for-packages=frankenphp; do | |
| rc=$? | |
| if [ "$attempt" -ge "$max" ]; then | |
| echo "::error::spc download failed after ${attempt} attempts (last exit ${rc})" | |
| exit "$rc" | |
| fi | |
| wait=$((attempt * 30)) | |
| echo "spc download attempt ${attempt} failed (exit ${rc}); retrying in ${wait}s (cached sources are skipped) ..." | |
| sleep "$wait" | |
| attempt=$((attempt + 1)) | |
| done | |
| echo "spc download completed on attempt ${attempt}" | |
| - name: Probe upstream releases | |
| id: probe-upstream | |
| run: | | |
| set -euo pipefail | |
| new='{}' | |
| for v in $PHP_MINORS; do | |
| # php.net answers an unreleased minor with HTTP 200 and {"error":"Unknown | |
| # version"}, so `keys[0]` yields the literal string "error". Only accept a | |
| # key that actually looks like a version. | |
| latest=$(curl -fsSL "https://www.php.net/releases/index.php?json=&max=1&version=${v}" \ | |
| | jq -r 'to_entries | map(select(.key | test("^[0-9]+\\.[0-9]+\\.[0-9]+"))) | .[0].key // empty') | |
| if [[ -z "$latest" ]]; then | |
| # pre-GA minor: php.net has no entry yet, so mirror what spc downloads | |
| # (StaticPHP\Artifact\Downloader\Type\PhpRelease resolves the newest tag). | |
| # Rank with version_compare, not `sort -V`: byte order puts the uppercase | |
| # "RC" tags *before* lowercase "alpha"/"beta", so a freshly tagged RC1 | |
| # would lose to beta1 and never trigger a rebuild. | |
| latest=$(curl -fsSL \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/php/php-src/git/matching-refs/tags/php-${v}." \ | |
| | jq -r '.[].ref | sub("^refs/tags/php-"; "")' \ | |
| | grep -E "^${v}\.[0-9]+((alpha|beta|RC)[0-9]+)?$" \ | |
| | php -r '$t = array_values(array_filter(array_map("trim", file("php://stdin")))); usort($t, "version_compare"); echo end($t) ?: "";' || true) | |
| fi | |
| if [[ -z "$latest" ]]; then | |
| echo "::error::Failed to fetch latest PHP version for ${v}" | |
| exit 1 | |
| fi | |
| echo "PHP ${v} latest: ${latest}" | |
| new=$(jq --arg v "$v" --arg l "$latest" '. + {($v): $l}' <<< "$new") | |
| done | |
| pie_latest=$(curl -fsSL \ | |
| -H "Authorization: Bearer $GITHUB_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| "https://api.github.com/repos/php/pie/releases/latest" \ | |
| | jq -r '.tag_name // empty') | |
| if [[ -z "$pie_latest" ]]; then | |
| echo "::error::Failed to fetch latest pie release" | |
| exit 1 | |
| fi | |
| echo "pie latest: ${pie_latest}" | |
| mkdir -p downloads | |
| jq --argjson v "$new" --arg pie "$pie_latest" -n \ | |
| '{php_versions: $v, pie_version: $pie}' > downloads/.spc-meta.json | |
| - name: Diff state and compute triggers | |
| id: diff | |
| env: | |
| FORCE_FULL: ${{ inputs.force_full }} | |
| run: | | |
| set -euo pipefail | |
| # spc v3 writes downloads/.cache.json, keyed by artifact/package name, | |
| # with a source's hash at .<key>.source.hash. Only a subset of these keys | |
| # are valid `--packages` values for bin/spp: PHP extensions (keyed "ext-<name>", | |
| # emitted without the prefix) and the frankenphp SAPI. Every other key is a | |
| # bare library/toolchain source (libaom, openssl, zlib, …), php-src, or | |
| # php-micro — none of which are craft packages. Passing those to bin/spp | |
| # produces "Package X not found in configuration" warnings and wasteful | |
| # partial rebuilds, so they are dropped here. php-src bumps are handled | |
| # separately via full_versions below; a library-only bump intentionally | |
| # triggers nothing and gets folded into the tarball on the next real trigger. | |
| changed_pkgs=$(jq -nc \ | |
| --slurpfile prev .download-state/lock.prev.json \ | |
| --slurpfile cur downloads/.cache.json ' | |
| ($prev[0] // {}) as $p | ($cur[0] // {}) as $c | | |
| [ ($c | keys[]) as $k | |
| | select(($p[$k].source.hash // "") != ($c[$k].source.hash // "")) | |
| | select(($k | startswith("ext-")) or $k == "frankenphp") | |
| | ($k | sub("^ext-"; "")) ] | |
| | unique | |
| ') | |
| echo "Changed packages: $(jq -c . <<< "$changed_pkgs")" | |
| # pie is downloaded outside SPC (src/package/pie.php pulls from | |
| # github.com/php/pie releases), so it never appears in .cache.json. | |
| # Detect bumps via the version stashed in .spc-meta.json by probe-upstream. | |
| prev_pie=$(jq -r '.pie_version // ""' .download-state/meta.prev.json) | |
| cur_pie=$(jq -r '.pie_version // ""' downloads/.spc-meta.json) | |
| if [[ "$prev_pie" != "$cur_pie" ]]; then | |
| echo "pie version bump: ${prev_pie:-<none>} -> ${cur_pie}" | |
| changed_pkgs=$(jq -c '. + ["pie"] | unique' <<< "$changed_pkgs") | |
| fi | |
| pkgs=$(jq -r 'join(",")' <<< "$changed_pkgs") | |
| echo "Changed packages: ${pkgs:-<none>}" | |
| full_versions=$(jq -nrc \ | |
| --slurpfile prev .download-state/meta.prev.json \ | |
| --slurpfile cur downloads/.spc-meta.json ' | |
| (($prev[0].php_versions) // {}) as $p | | |
| (($cur[0].php_versions) // {}) as $c | | |
| [ ($c | keys[]) as $k | |
| | select($c[$k] != ($p[$k] // "")) | |
| | $k ] | |
| ') | |
| echo "PHP minors with bumped php-src: $(jq -c . <<< "$full_versions")" | |
| all_minors=$(jq -nc --arg s "$PHP_MINORS" '$s | split(" ")') | |
| if [[ "$FORCE_FULL" == "true" ]]; then | |
| echo "force_full input set -> rebuilding all PHP versions" | |
| full_versions="$all_minors" | |
| fi | |
| full_csv=$(jq -r 'join(",")' <<< "$full_versions") | |
| partial_csv="" | |
| if [[ -n "$pkgs" ]]; then | |
| partial_csv=$(jq -nrc \ | |
| --argjson all "$all_minors" \ | |
| --argjson full "$full_versions" ' | |
| $all - $full | join(",") | |
| ') | |
| fi | |
| any="false" | |
| if [[ -n "$pkgs" || -n "$full_csv" ]]; then | |
| any="true" | |
| fi | |
| echo "full_php=$full_csv" | tee -a $GITHUB_OUTPUT | |
| echo "partial_php=$partial_csv" | tee -a $GITHUB_OUTPUT | |
| echo "changed_packages=$pkgs" | tee -a $GITHUB_OUTPUT | |
| echo "any_change=$any" | tee -a $GITHUB_OUTPUT | |
| { | |
| echo "## spc-download summary" | |
| echo "- Full rebuild PHP versions: ${full_csv:-<none>}" | |
| echo "- Partial rebuild PHP versions: ${partial_csv:-<none>}" | |
| echo "- Changed packages: ${pkgs:-<none>}" | |
| echo "- Any change: $any" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Persist current state for next run | |
| if: hashFiles('downloads/.cache.json') != '' | |
| run: | | |
| set -euo pipefail | |
| cp downloads/.cache.json .download-state/lock.prev.json | |
| cp downloads/.spc-meta.json .download-state/meta.prev.json | |
| - name: Delete previous state cache | |
| if: hashFiles('downloads/.cache.json') != '' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| gh api -X DELETE \ | |
| "/repos/$GITHUB_REPOSITORY/actions/caches?key=spc-downloads-lock" \ | |
| 2>/dev/null || true | |
| - name: Save state cache | |
| if: hashFiles('downloads/.cache.json') != '' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: .download-state | |
| key: spc-downloads-lock | |
| - name: Create tarball (keep permissions) | |
| if: steps.diff.outputs.any_change == 'true' || github.event_name == 'workflow_dispatch' | |
| run: | | |
| tar -czf downloads.tar.gz -C downloads . | |
| - name: Upload downloads directory | |
| if: steps.diff.outputs.any_change == 'true' || github.event_name == 'workflow_dispatch' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: downloads-tarball | |
| path: downloads.tar.gz | |
| retention-days: 14 | |
| - name: Trigger downstream workflows | |
| if: steps.diff.outputs.any_change == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_PAT }} | |
| FULL_PHP: ${{ steps.diff.outputs.full_php }} | |
| PARTIAL_PHP: ${{ steps.diff.outputs.partial_php }} | |
| CHANGED_PACKAGES: ${{ steps.diff.outputs.changed_packages }} | |
| run: | | |
| set -euo pipefail | |
| workflows=( | |
| build-rpm-modular-packages.yml | |
| build-deb-forgejo.yml | |
| build-apk-forgejo.yml | |
| ) | |
| for wf in "${workflows[@]}"; do | |
| if [[ -n "$FULL_PHP" ]]; then | |
| echo ">> $wf : full rebuild for $FULL_PHP" | |
| gh workflow run "$wf" -f php_versions="$FULL_PHP" | |
| fi | |
| if [[ -n "$CHANGED_PACKAGES" && -n "$PARTIAL_PHP" ]]; then | |
| echo ">> $wf : partial rebuild for $PARTIAL_PHP, packages=$CHANGED_PACKAGES" | |
| gh workflow run "$wf" \ | |
| -f php_versions="$PARTIAL_PHP" \ | |
| -f packages="$CHANGED_PACKAGES" | |
| fi | |
| done |