Repository navigation
feat: back up and restore external databases on Kubernetes #172
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
8df6a24
c2bcaeb
03ba244
fb213e2
c560786
e200950
e0cab74
3881bb0
2242579
23be87e
3584a53
f24c0ed
0a6eb71
d632fff
79f278b
da1b884
25941e9
bc6321b
97b2af0
d4bb6b5
72eb2d3
241faa4
8981548
887fa50
263ad9a
a170ef0
ea1d73b
48f6e95
b5e87ad
e6abd08
90a088c
97461b9
d1b3c2f
3fe7b95
2a506b0
9ba780d
16c4a08
afa67f1
1bb78fe
2ebc56e
5707324
ca4992e
2bfed31
a5a2ee5
d30545f
76d674f
395e062
43eace0
9bd34bc
4737a16
3f9abb2
ad45562
7090010
304756b
c11c057
742add6
e9fd83b
8f96843
01a308c
91eafee
69f368b
b9e9b23
8af0bbd
9a61410
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -55,7 +55,7 @@ jobs: | |
| - name: golangci-lint | ||
| uses: golangci/golangci-lint-action@v9 | ||
| with: | ||
| version: v2.7.2 | ||
| version: v2.13.2 | ||
|
|
||
| test: | ||
| if: needs.files-changed.outputs.go == 'true' | ||
|
|
@@ -130,12 +130,20 @@ jobs: | |
| PYTEST_MARKERS: ${{ matrix.edition == 'enterprise' && 'docker' || 'docker and not enterprise' }} | ||
| run: exec tini -s -g -- uv run pytest -v -m "$PYTEST_MARKERS" tests/e2e | ||
|
|
||
| # Python changes run it too: the suites themselves are Python, and a change to them alone | ||
| # must still be exercised. `test` is skipped when no Go changed, and a skipped need skips | ||
| # this job unless the condition says otherwise, so it runs unless a need failed. | ||
| e2e-tests-k8s: | ||
| if: needs.files-changed.outputs.go == 'true' | ||
| needs: ["files-changed", "test"] | ||
| if: | | ||
| !cancelled() && !contains(needs.*.result, 'failure') && | ||
| (needs.files-changed.outputs.go == 'true' || needs.files-changed.outputs.python == 'true') | ||
| # go-lint is listed so that its failure stops this job; `test` only inherits it as a skip. | ||
| needs: ["files-changed", "go-lint", "test"] | ||
| runs-on: | ||
| group: huge-runners | ||
| timeout-minutes: 30 | ||
| # The enterprise leg also runs the two external-database sessions, each with its own | ||
| # vcluster and Infrahub deployment. | ||
| timeout-minutes: 60 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
|
|
@@ -173,6 +181,91 @@ jobs: | |
| - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) | ||
| run: make build | ||
|
|
||
| # The external-database cases run the database tools from this image (ADR 0017) and | ||
| # pass it with --external-db-image, because a development build has no published image | ||
| # to default to. The vcluster is created by pytest, so the `external_db_image` fixture | ||
| # loads the image into the cluster's nodes; it is tagged with the commit rather than | ||
| # `latest` so that the workload takes it from the node instead of pulling it. | ||
| - name: Build the infrahub-backup image under test | ||
| run: | | ||
| docker build -t "infrahub-backup:e2e-${GITHUB_SHA}" . | ||
| echo "INFRAHUB_TESTING_EXTERNAL_DB_IMAGE=infrahub-backup:e2e-${GITHUB_SHA}" >> "$GITHUB_ENV" | ||
|
|
||
| # The image's Java runtime and Neo4j jars are trimmed to what the tools load. This runner is | ||
| # amd64, so this is where the trimmed image is checked on that architecture. | ||
| - name: Check the image carries working database tools | ||
| run: SKIP_BUILD=1 ./scripts/check-image-tools.sh "infrahub-backup:e2e-${GITHUB_SHA}" | ||
|
|
||
| # The external-database suites (tasks.md T072) need Neo4j Enterprise, PostgreSQL and an | ||
| # object store outside the cluster under test. They run as containers on one Docker | ||
| # network, so Neo4j can fetch a restore's seed from MinIO by name. Pods in the vcluster | ||
| # and the tool on the runner both reach them through the Docker host's bridge address, | ||
| # where the ports are published. Only the enterprise leg declares the topology, so the | ||
| # suites run once. | ||
| - name: Start the external databases | ||
| if: matrix.edition == 'enterprise' | ||
| run: | | ||
| docker rm -f ext-minio ext-neo4j ext-postgres 2>/dev/null || true | ||
| docker network rm ext-db 2>/dev/null || true | ||
| docker network create ext-db | ||
| # The tool verifies certificates by default (FR-029) and always encrypts PostgreSQL, | ||
| # so both servers present one self-signed certificate and the suites opt out of | ||
| # verifying it. Neo4j keeps plain Bolt available for Infrahub's own connections. | ||
| certs="${RUNNER_TEMP}/ext-db-certs" | ||
| rm -rf "${certs}" && mkdir -p "${certs}/bolt/trusted" "${certs}/bolt/revoked" | ||
| openssl req -x509 -newkey rsa:2048 -nodes -days 2 -subj "/CN=ext-db" \ | ||
| -keyout "${certs}/bolt/private.key" -out "${certs}/bolt/public.crt" | ||
| cp "${certs}/bolt/public.crt" "${certs}/bolt/trusted/" | ||
| chmod -R a+rX "${certs}" | ||
| docker run -d --name ext-minio --network ext-db -p 19000:9000 \ | ||
| -e MINIO_ROOT_USER=extaccess -e MINIO_ROOT_PASSWORD=extsecretkey \ | ||
| --entrypoint sh pgsty/minio:RELEASE.2026-08-04T00-00-00Z \ | ||
| -c 'mkdir -p /data/infrahub-seed && minio server /data' | ||
| docker run -d --name ext-neo4j --network ext-db -p 17687:7687 -p 6362:6362 \ | ||
| -v "${certs}:/ssl:ro" \ | ||
| -e NEO4J_dbms_ssl_policy_bolt_enabled=true \ | ||
| -e NEO4J_dbms_ssl_policy_bolt_base__directory=/ssl/bolt \ | ||
| -e NEO4J_dbms_ssl_policy_bolt_client__auth=NONE \ | ||
| -e NEO4J_server_bolt_tls__level=OPTIONAL \ | ||
| -e NEO4J_ACCEPT_LICENSE_AGREEMENT=yes -e NEO4J_AUTH=neo4j/admin12345 \ | ||
| -e NEO4J_server_backup_enabled=true \ | ||
| -e NEO4J_server_backup_listen__address=0.0.0.0:6362 \ | ||
| -e AWS_REGION=us-east-1 -e AWS_ACCESS_KEY_ID=extaccess -e AWS_SECRET_ACCESS_KEY=extsecretkey \ | ||
| -e AWS_ENDPOINT_URL_S3=http://ext-minio:9000 \ | ||
| neo4j:2025.10.1-enterprise | ||
| docker run -d --name ext-postgres --network ext-db -p 15432:5432 \ | ||
| -v "${certs}:/certs:ro" \ | ||
| -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=prefect -e POSTGRES_DB=prefect \ | ||
| --entrypoint sh postgres:18-alpine -c ' | ||
| mkdir -p /etc/pgssl && cp /certs/bolt/public.crt /certs/bolt/private.key /etc/pgssl/ && | ||
| chown -R postgres /etc/pgssl && chmod 600 /etc/pgssl/private.key && | ||
| exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/pgssl/public.crt -c ssl_key_file=/etc/pgssl/private.key' | ||
| for i in $(seq 1 90); do | ||
| docker exec ext-neo4j cypher-shell -u neo4j -p admin12345 'RETURN 1' && break | ||
| sleep 2 | ||
| done | ||
| for i in $(seq 1 30); do | ||
| docker exec ext-postgres pg_isready -U postgres && break | ||
| sleep 1 | ||
| done | ||
| host=$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}') | ||
| { | ||
| echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_ADDRESS=${host}:17687" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_USERNAME=neo4j" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_PASSWORD=admin12345" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_EDITION=enterprise" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_ADDRESS=${host}:15432" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_USERNAME=postgres" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_PASSWORD=prefect" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_SEED_BUCKET=infrahub-seed" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_SEED_ENDPOINT=http://${host}:19000" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_SEED_ACCESS_KEY=extaccess" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_SEED_SECRET_KEY=extsecretkey" | ||
| echo "INFRAHUB_TESTING_EXTERNAL_INSECURE_TLS=true" | ||
| # Neo4j 2025.x reports no store size, so a capture needs one (tasks.md T158). | ||
| echo "INFRAHUB_TESTING_EXTERNAL_SCRATCH_SIZE=2Gi" | ||
| } >> "$GITHUB_ENV" | ||
|
|
||
| - name: Install Python dependencies | ||
| run: uv sync --all-packages | ||
|
|
||
|
|
@@ -184,6 +277,101 @@ jobs: | |
| - name: Run e2e tests (Kubernetes) | ||
| run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e | ||
|
|
||
| # Each external topology is its own pytest session, because the deployment under test | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Custom agent: Flag AI Slop and Fabricated Changes The preceding all- Prompt for AI agents
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| # is created once per session from INFRAHUB_TESTING_EXTERNAL_DB. | ||
| - name: Run e2e tests (Kubernetes, external Neo4j) | ||
|
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
|
||
| if: matrix.edition == 'enterprise' | ||
| env: | ||
| INFRAHUB_TESTING_EXTERNAL_DB: database | ||
| run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e/test_k8s_external_neo4j.py tests/e2e/test_k8s_external_in_namespace.py | ||
|
|
||
| - name: Run e2e tests (Kubernetes, external Neo4j and PostgreSQL) | ||
| if: matrix.edition == 'enterprise' | ||
| env: | ||
| INFRAHUB_TESTING_EXTERNAL_DB: database,task-manager-db | ||
| run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e/test_k8s_external_both.py | ||
|
|
||
| - name: Remove the external databases | ||
| if: always() && matrix.edition == 'enterprise' | ||
| run: | | ||
| docker rm -f ext-minio ext-neo4j ext-postgres 2>/dev/null || true | ||
| docker network rm ext-db 2>/dev/null || true | ||
|
|
||
| # A three-member Neo4j cluster, built like Infrahub's HA example, once inside the | ||
| # Infrahub namespace and once outside it. Every other Kubernetes case runs against one | ||
| # Neo4j server. Each topology runs on its own runner, because one cluster and one | ||
| # Infrahub deployment already fill one. | ||
| e2e-tests-k8s-neo4j-cluster: | ||
| if: | | ||
| !cancelled() && !contains(needs.*.result, 'failure') && | ||
| (needs.files-changed.outputs.go == 'true' || needs.files-changed.outputs.python == 'true') | ||
| needs: ["files-changed", "go-lint", "test"] | ||
| runs-on: | ||
| group: huge-runners | ||
| timeout-minutes: 45 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| topology: [internal, external] | ||
| env: | ||
| INFRAHUB_HELM_CHART: oci://registry.opsmill.io/opsmill/chart/infrahub-enterprise | ||
| INFRAHUB_TESTING_NEO4J_CLUSTER: ${{ matrix.topology }} | ||
| steps: | ||
| - uses: actions/checkout@v6 | ||
|
|
||
| - name: Set up Go | ||
| uses: actions/setup-go@v6 | ||
| with: | ||
| go-version: "1.25" | ||
|
|
||
| - name: Set up vCluster | ||
| uses: loft-sh/setup-vcluster@68e17a8c3618e3c47d635842b803d4f560f694e8 | ||
|
|
||
| - name: Set up Helm | ||
| uses: azure/setup-helm@v4 | ||
|
|
||
| - name: Set up kubectl | ||
| uses: azure/setup-kubectl@v4 | ||
|
|
||
| - name: Set up Python | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
|
|
||
| - name: Set up Python and uv | ||
| uses: astral-sh/setup-uv@v7 | ||
|
|
||
| - name: Install tini | ||
| run: apt-get update && apt-get install -y tini | ||
|
|
||
| - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) | ||
| run: make build | ||
|
|
||
| - name: Install Python dependencies | ||
| run: uv sync --all-packages | ||
|
|
||
| - name: "Set environment variables" | ||
| run: | | ||
| RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/') | ||
| echo "VCLUSTER_DIR=/var/lib/github/${RUNNER_NAME}/_temp/.vcluster/config.json" >> $GITHUB_ENV | ||
|
|
||
| # The fixtures build the whole topology: the three-node vcluster, the Neo4j cluster, | ||
| # the object store and the infrahub-backup image (tests/e2e/conftest.py, "Neo4j | ||
| # cluster topologies"). The cases of the other topology skip. | ||
| - name: Run e2e tests (Kubernetes, Neo4j cluster) | ||
| run: >- | ||
| exec tini -s -g -- uv run pytest -v -rs -m k8s | ||
| tests/e2e/test_k8s_neo4j_cluster.py tests/e2e/test_k8s_external_neo4j_cluster.py | ||
| --junitxml=neo4j-cluster-${{ matrix.topology }}-results.xml | ||
|
|
||
| - name: Upload the test results | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: neo4j-cluster-${{ matrix.topology }}-results | ||
| path: neo4j-cluster-${{ matrix.topology }}-results.xml | ||
| if-no-files-found: ignore | ||
|
|
||
| nix-build: | ||
| if: needs.files-changed.outputs.nix == 'true' | ||
| needs: ["files-changed"] | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.