Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions .github/workflows/cache-reset.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
name: Reset CI Cache

# Manual recovery for poisoned BuildKit/ccache state: if a cache scope
# is producing compile-time-only failures that warm builds reproduce
# but cold builds do not, run this workflow with the scope substring
# to delete all matching cache entries and force a cold rebuild.
#
# Usage: GitHub UI -> Actions -> Reset CI Cache -> Run workflow
# Enter scope substring, e.g.:
# - "main" -> deletes master-branch cache
# - "pr-master" -> deletes shared PR-against-master cache
# - "pr-release-1.0" -> deletes a specific release-branch PR cache

on:
workflow_dispatch:
inputs:
scope_substring:
description: 'Substring of cache key to delete (e.g., main, pr-master)'
required: true
type: string

permissions:
actions: write # required to delete caches

jobs:
reset:
runs-on: ubuntu-latest
steps:
- name: Delete matching caches
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Pass user input via env (not direct ${{ }} interpolation in the
# run: block) to avoid workflow-injection if the substring contains
# shell metacharacters like single quotes.
SCOPE_PATTERN: ${{ inputs.scope_substring }}
run: |
set -uo pipefail
echo "Listing caches matching: $SCOPE_PATTERN"
# gh cache list returns id<TAB>key<TAB>... ; jq filters by key
# substring. --limit 1000 is the gh CLI page max. If we hit the
# limit, the listing may be incomplete and the operator should
# rerun after the first batch is deleted.
LIMIT=1000
listing=$(gh cache list --repo "$GITHUB_REPOSITORY" --limit "$LIMIT" --json id,key)
total=$(echo "$listing" | jq 'length')
if [ "$total" -eq "$LIMIT" ]; then
echo "WARN: hit --limit $LIMIT; listing may be truncated. Rerun after this batch."
fi
# Print id<TAB>key for each match so the operator can audit which
# entries are about to be deleted (substring matches can be broader
# than intended — e.g. 'main' matches 'domain-foo').
matches=$(echo "$listing" \
| jq -r --arg pat "$SCOPE_PATTERN" '.[] | select(.key | contains($pat)) | "\(.id)\t\(.key)"')
if [ -z "$matches" ]; then
echo "No caches matched substring: $SCOPE_PATTERN"
exit 0
fi
echo "Will delete the following caches (id<TAB>key):"
echo "$matches"
# Best-effort deletion: a failure on one cache (race with auto-
# eviction, transient 5xx) should not abort the whole loop.
# Track failures and exit non-zero at the end if any occurred.
failures=0
while IFS=$'\t' read -r id key; do
if ! gh cache delete "$id" --repo "$GITHUB_REPOSITORY"; then
echo "WARN: failed to delete cache id=$id key=$key"
failures=$((failures + 1))
fi
done <<< "$matches"
if [ "$failures" -gt 0 ]; then
echo "Done with $failures failure(s)."
exit 1
fi
echo "Done."
77 changes: 77 additions & 0 deletions .github/workflows/docker-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,112 @@ jobs:
with:
submodules: recursive

- uses: docker/setup-buildx-action@v3
id: setup-buildx

- uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

# Cache-mount key hashes BOTH Dockerfiles so the testnet and
# production jobs (which install identical packages) compute the
# same key prefix — letting build_prod restore what build_testnet
# just saved.
- name: Restore BuildKit cache mounts from Actions Cache
uses: actions/cache@v4
id: cache-mounts
with:
path: cache-mount
key: viz-cache-mount-${{ runner.os }}-main-${{ hashFiles('share/vizd/docker/Dockerfile-production', 'share/vizd/docker/Dockerfile-testnet') }}-${{ github.sha }}
restore-keys: |
viz-cache-mount-${{ runner.os }}-main-${{ hashFiles('share/vizd/docker/Dockerfile-production', 'share/vizd/docker/Dockerfile-testnet') }}-
viz-cache-mount-${{ runner.os }}-main-

- name: Inject cache mounts into BuildKit
uses: reproducible-containers/buildkit-cache-dance@v3
with:
builder: ${{ steps.setup-buildx.outputs.name }}
cache-map: |
{
"cache-mount/var-cache-apt": "/var/cache/apt",
"cache-mount/var-lib-apt-lists": "/var/lib/apt/lists",
"cache-mount/ccache": {
"target": "/root/.ccache",
"id": "viz-ccache"
}
}
skip-extraction: ${{ steps.cache-mounts.outputs.cache-hit }}

# Both jobs share scope=main so build_prod (which runs after
# build_testnet via needs:) can reuse the ccache populated by the
# testnet build. The two images differ only in CMake flags so most
# compiled object files have identical ccache hashes.
- uses: docker/build-push-action@v6
with:
context: .
file: ./share/vizd/docker/Dockerfile-testnet
push: true
tags: vizblockchain/vizd:testnet
cache-from: type=gha,scope=main
cache-to: type=gha,scope=main,mode=max

build_prod:
name: Build production image
runs-on: ubuntu-latest
# needs: build_testnet ensures build_prod runs after build_testnet so
# it can pull the warm cache exported by build_testnet's cache-to step.
# Side-effect: a build_testnet failure now blocks build_prod, which is
# arguably an improvement (don't ship prod if testnet is broken).
needs: build_testnet

steps:
- uses: actions/checkout@v4
with:
submodules: recursive

- uses: docker/setup-buildx-action@v3
id: setup-buildx

- uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

# Same cache-mount key prefix as build_testnet so this job restores
# the cache build_testnet just saved (same SHA, same Dockerfile
# hashes). Apt content fully reusable; ccache reuse depends on how
# -DBUILD_TESTNET propagates through the build (measured per run).
- name: Restore BuildKit cache mounts from Actions Cache
uses: actions/cache@v4
id: cache-mounts
with:
path: cache-mount
key: viz-cache-mount-${{ runner.os }}-main-${{ hashFiles('share/vizd/docker/Dockerfile-production', 'share/vizd/docker/Dockerfile-testnet') }}-${{ github.sha }}
restore-keys: |
viz-cache-mount-${{ runner.os }}-main-${{ hashFiles('share/vizd/docker/Dockerfile-production', 'share/vizd/docker/Dockerfile-testnet') }}-
viz-cache-mount-${{ runner.os }}-main-

- name: Inject cache mounts into BuildKit
uses: reproducible-containers/buildkit-cache-dance@v3
with:
builder: ${{ steps.setup-buildx.outputs.name }}
cache-map: |
{
"cache-mount/var-cache-apt": "/var/cache/apt",
"cache-mount/var-lib-apt-lists": "/var/lib/apt/lists",
"cache-mount/ccache": {
"target": "/root/.ccache",
"id": "viz-ccache"
}
}
skip-extraction: ${{ steps.cache-mounts.outputs.cache-hit }}

- uses: docker/build-push-action@v6
with:
context: .
file: ./share/vizd/docker/Dockerfile-production
push: true
tags: vizblockchain/vizd:latest
cache-from: type=gha,scope=main
cache-to: type=gha,scope=main,mode=max
57 changes: 56 additions & 1 deletion .github/workflows/docker-pr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,69 @@ jobs:
with:
submodules: recursive

# setup-buildx-action is required for advanced cache features
# (cache-from / cache-to with type=gha and mode=max). The id: lets
# buildkit-cache-dance below pick up the builder name.
- uses: docker/setup-buildx-action@v3
id: setup-buildx

- uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

# Docker tags cannot contain '/', but git branch names can (e.g.
# 'ci/build-time-reduction'). Sanitize via bash parameter expansion
# in an env-var (not direct ${{ }} interpolation) to avoid workflow
# injection if a branch name ever contains shell metacharacters.
- name: Compute docker tag from branch name
id: tag
env:
BRANCH_NAME: ${{ github.head_ref }}
run: |
set -euo pipefail
echo "value=${BRANCH_NAME//\//-}" >> "$GITHUB_OUTPUT"

# cache-from/cache-to type=gha exports Docker LAYERS but not the
# contents of --mount=type=cache directories (ccache, apt). On
# ephemeral GHA runners the BuildKit daemon is fresh each run, so
# named cache mounts always start empty without the dance below.
# actions/cache restores a host directory across runs;
# buildkit-cache-dance binds it into the build's cache mounts and
# extracts the post-build state for the next save.
- name: Restore BuildKit cache mounts from Actions Cache
uses: actions/cache@v4
id: cache-mounts
with:
path: cache-mount
key: viz-cache-mount-${{ runner.os }}-pr-${{ github.base_ref }}-${{ hashFiles('share/vizd/docker/Dockerfile-production') }}-${{ github.sha }}
restore-keys: |
viz-cache-mount-${{ runner.os }}-pr-${{ github.base_ref }}-${{ hashFiles('share/vizd/docker/Dockerfile-production') }}-
viz-cache-mount-${{ runner.os }}-pr-${{ github.base_ref }}-

- name: Inject cache mounts into BuildKit
uses: reproducible-containers/buildkit-cache-dance@v3
with:
builder: ${{ steps.setup-buildx.outputs.name }}
cache-map: |
{
"cache-mount/var-cache-apt": "/var/cache/apt",
"cache-mount/var-lib-apt-lists": "/var/lib/apt/lists",
"cache-mount/ccache": {
"target": "/root/.ccache",
"id": "viz-ccache"
}
}
skip-extraction: ${{ steps.cache-mounts.outputs.cache-hit }}

# Layer cache scope is keyed on the PR's base branch so all PRs
# targeting master share one warm-cache pool, while PRs against
# release branches cannot poison master's cache.
- uses: docker/build-push-action@v6
with:
context: .
file: ./share/vizd/docker/Dockerfile-production
push: true
tags: vizblockchain/vizd:${{ github.head_ref }}
tags: vizblockchain/vizd:${{ steps.tag.outputs.value }}
cache-from: type=gha,scope=pr-${{ github.base_ref }}
cache-to: type=gha,scope=pr-${{ github.base_ref }},mode=max
8 changes: 7 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,10 @@ documentation/doxygen/latex
*.pyc
*.pyo
*.egg-info/
build
build

# Local design specs and implementation plans (not part of repo)
docs/superpowers/

# Claude Code local session/config files
.claude/
55 changes: 48 additions & 7 deletions share/vizd/docker/Dockerfile-production
Original file line number Diff line number Diff line change
@@ -1,10 +1,42 @@
# syntax=docker/dockerfile:1.4
FROM phusion/baseimage:noble-1.0.3 AS builder

ENV LANG=en_US.UTF-8
ENV APPDIR=/usr/local/src/viz/
ENV HOME=/var/lib/vizd

RUN \
# ccache configuration (cache directory is provided as a BuildKit cache mount below).
# CCACHE_MAXSIZE=2G keeps ccache's contribution to the GHA Actions Cache bounded.
# With Docker layer cache (~3-4 GB) plus ccache (<=2 GB), total stays comfortably
# under the 10 GB GHA per-repo cache quota. Do not raise beyond 4G without also
# verifying the layer cache footprint.
# CCACHE_COMPRESS=1 is redundant with ccache >=4.0 defaults but kept explicit for
# older-ccache compatibility (local builds on older distros).
ENV CCACHE_DIR=/root/.ccache
ENV CCACHE_MAXSIZE=2G
ENV CCACHE_COMPRESS=1

# Install build dependencies.
#
# IMPORTANT: the `rm -f /etc/apt/apt.conf.d/docker-clean` line is required.
# Ubuntu/Debian base images ship an apt hook that auto-deletes
# /var/cache/apt/archives and /var/lib/apt/lists after every apt run, which
# silently defeats both BuildKit cache mounts below. Do not remove this rm.
#
# Equally important: the Keep-Downloaded-Packages line. Even with
# docker-clean removed, apt's default is to delete .deb archives after
# install (Binary::apt::APT::Keep-Downloaded-Packages "false"). Without
# explicitly flipping it to "true", buildkit-cache-dance extracts an
# empty /var/cache/apt/archives/ from the cache mount.
#
# We also intentionally do NOT run `apt-get clean` or `rm -rf /var/lib/apt/lists/*`
# at the end, because those paths are mounted as BuildKit caches that we want
# to persist across CI runs. The builder stage is discarded in the final image
# regardless, so leftover cache contents have no effect on the production image.
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
rm -f /etc/apt/apt.conf.d/docker-clean && \
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
for i in 1 2 3; do \
apt-get update && \
apt-get install -y --no-install-recommends \
Expand Down Expand Up @@ -40,9 +72,7 @@ RUN \
pkg-config \
zlib1g-dev && \
break || sleep 10; \
done && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
done

# Explicitly copy needed files to avoid re-building when changing other files
COPY .git $APPDIR/.git
Expand All @@ -52,11 +82,18 @@ COPY thirdparty $APPDIR/thirdparty
COPY plugins $APPDIR/plugins
COPY libraries $APPDIR/libraries

RUN \
# Build vizd. The named cache mount `viz-ccache` is shared across all VIZ
# Dockerfiles so that production and testnet builds in the same workflow
# benefit from each other's compiled-object cache. `make -j$(nproc) vizd`
# (instead of unscoped `make`) skips cli_wallet, util programs, and tests
# that are not installed into the production image. `sharing=locked` is
# required because ccache is not concurrent-safe; `shared` would risk cache
# corruption under any future matrix-build setup.
RUN --mount=type=cache,target=/root/.ccache,id=viz-ccache,sharing=locked \
cd $APPDIR && \
git submodule update --init --recursive -f && \
sed -i '/add_subdirectory(tests)/d' thirdparty/fc/CMakeLists.txt && \
mkdir build && \
mkdir -p build && \
cd build && \
cmake \
-DCMAKE_BUILD_TYPE=Release \
Expand All @@ -66,13 +103,17 @@ RUN \
-DENABLE_MONGO_PLUGIN=FALSE \
.. \
&& \
make -j$(nproc)
make -j$(nproc) vizd && \
ccache -s

RUN set -xe ;\
cd $APPDIR/build ;\
make install ;\
rm -rf $APPDIR

# Final cleanup of the builder stage. apt cache mounts above are not affected
# by these rm/clean calls because cache mounts are unmounted by the time this
# RUN executes; this only removes anything that landed in real layers.
RUN \
apt-get clean && \
apt-get autoremove -y && \
Expand Down
Loading
Loading