diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42bbdb09..bd2bdf6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: - version: v2.7.2 + version: v2.13.2 test: if: needs.files-changed.outputs.go == 'true' @@ -130,12 +130,20 @@ jobs: PYTEST_MARKERS: ${{ matrix.edition == 'enterprise' && 'docker' || 'docker and not enterprise' }} run: exec tini -s -g -- uv run pytest -v -m "$PYTEST_MARKERS" tests/e2e + # Python changes run it too: the suites themselves are Python, and a change to them alone + # must still be exercised. `test` is skipped when no Go changed, and a skipped need skips + # this job unless the condition says otherwise, so it runs unless a need failed. e2e-tests-k8s: - if: needs.files-changed.outputs.go == 'true' - needs: ["files-changed", "test"] + if: | + !cancelled() && !contains(needs.*.result, 'failure') && + (needs.files-changed.outputs.go == 'true' || needs.files-changed.outputs.python == 'true') + # go-lint is listed so that its failure stops this job; `test` only inherits it as a skip. + needs: ["files-changed", "go-lint", "test"] runs-on: group: huge-runners - timeout-minutes: 30 + # The enterprise leg also runs the two external-database sessions, each with its own + # vcluster and Infrahub deployment. + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -173,6 +181,91 @@ jobs: - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) run: make build + # The external-database cases run the database tools from this image (ADR 0017) and + # pass it with --external-db-image, because a development build has no published image + # to default to. The vcluster is created by pytest, so the `external_db_image` fixture + # loads the image into the cluster's nodes; it is tagged with the commit rather than + # `latest` so that the workload takes it from the node instead of pulling it. + - name: Build the infrahub-backup image under test + run: | + docker build -t "infrahub-backup:e2e-${GITHUB_SHA}" . + echo "INFRAHUB_TESTING_EXTERNAL_DB_IMAGE=infrahub-backup:e2e-${GITHUB_SHA}" >> "$GITHUB_ENV" + + # The image's Java runtime and Neo4j jars are trimmed to what the tools load. This runner is + # amd64, so this is where the trimmed image is checked on that architecture. + - name: Check the image carries working database tools + run: SKIP_BUILD=1 ./scripts/check-image-tools.sh "infrahub-backup:e2e-${GITHUB_SHA}" + + # The external-database suites (tasks.md T072) need Neo4j Enterprise, PostgreSQL and an + # object store outside the cluster under test. They run as containers on one Docker + # network, so Neo4j can fetch a restore's seed from MinIO by name. Pods in the vcluster + # and the tool on the runner both reach them through the Docker host's bridge address, + # where the ports are published. Only the enterprise leg declares the topology, so the + # suites run once. + - name: Start the external databases + if: matrix.edition == 'enterprise' + run: | + docker rm -f ext-minio ext-neo4j ext-postgres 2>/dev/null || true + docker network rm ext-db 2>/dev/null || true + docker network create ext-db + # The tool verifies certificates by default (FR-029) and always encrypts PostgreSQL, + # so both servers present one self-signed certificate and the suites opt out of + # verifying it. Neo4j keeps plain Bolt available for Infrahub's own connections. + certs="${RUNNER_TEMP}/ext-db-certs" + rm -rf "${certs}" && mkdir -p "${certs}/bolt/trusted" "${certs}/bolt/revoked" + openssl req -x509 -newkey rsa:2048 -nodes -days 2 -subj "/CN=ext-db" \ + -keyout "${certs}/bolt/private.key" -out "${certs}/bolt/public.crt" + cp "${certs}/bolt/public.crt" "${certs}/bolt/trusted/" + chmod -R a+rX "${certs}" + docker run -d --name ext-minio --network ext-db -p 19000:9000 \ + -e MINIO_ROOT_USER=extaccess -e MINIO_ROOT_PASSWORD=extsecretkey \ + --entrypoint sh pgsty/minio:RELEASE.2026-08-04T00-00-00Z \ + -c 'mkdir -p /data/infrahub-seed && minio server /data' + docker run -d --name ext-neo4j --network ext-db -p 17687:7687 -p 6362:6362 \ + -v "${certs}:/ssl:ro" \ + -e NEO4J_dbms_ssl_policy_bolt_enabled=true \ + -e NEO4J_dbms_ssl_policy_bolt_base__directory=/ssl/bolt \ + -e NEO4J_dbms_ssl_policy_bolt_client__auth=NONE \ + -e NEO4J_server_bolt_tls__level=OPTIONAL \ + -e NEO4J_ACCEPT_LICENSE_AGREEMENT=yes -e NEO4J_AUTH=neo4j/admin12345 \ + -e NEO4J_server_backup_enabled=true \ + -e NEO4J_server_backup_listen__address=0.0.0.0:6362 \ + -e AWS_REGION=us-east-1 -e AWS_ACCESS_KEY_ID=extaccess -e AWS_SECRET_ACCESS_KEY=extsecretkey \ + -e AWS_ENDPOINT_URL_S3=http://ext-minio:9000 \ + neo4j:2025.10.1-enterprise + docker run -d --name ext-postgres --network ext-db -p 15432:5432 \ + -v "${certs}:/certs:ro" \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=prefect -e POSTGRES_DB=prefect \ + --entrypoint sh postgres:18-alpine -c ' + mkdir -p /etc/pgssl && cp /certs/bolt/public.crt /certs/bolt/private.key /etc/pgssl/ && + chown -R postgres /etc/pgssl && chmod 600 /etc/pgssl/private.key && + exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/pgssl/public.crt -c ssl_key_file=/etc/pgssl/private.key' + for i in $(seq 1 90); do + docker exec ext-neo4j cypher-shell -u neo4j -p admin12345 'RETURN 1' && break + sleep 2 + done + for i in $(seq 1 30); do + docker exec ext-postgres pg_isready -U postgres && break + sleep 1 + done + host=$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}') + { + echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_ADDRESS=${host}:17687" + echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_USERNAME=neo4j" + echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_PASSWORD=admin12345" + echo "INFRAHUB_TESTING_EXTERNAL_NEO4J_EDITION=enterprise" + echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_ADDRESS=${host}:15432" + echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_USERNAME=postgres" + echo "INFRAHUB_TESTING_EXTERNAL_POSTGRES_PASSWORD=prefect" + echo "INFRAHUB_TESTING_EXTERNAL_SEED_BUCKET=infrahub-seed" + echo "INFRAHUB_TESTING_EXTERNAL_SEED_ENDPOINT=http://${host}:19000" + echo "INFRAHUB_TESTING_EXTERNAL_SEED_ACCESS_KEY=extaccess" + echo "INFRAHUB_TESTING_EXTERNAL_SEED_SECRET_KEY=extsecretkey" + echo "INFRAHUB_TESTING_EXTERNAL_INSECURE_TLS=true" + # Neo4j 2025.x reports no store size, so a capture needs one (tasks.md T158). + echo "INFRAHUB_TESTING_EXTERNAL_SCRATCH_SIZE=2Gi" + } >> "$GITHUB_ENV" + - name: Install Python dependencies run: uv sync --all-packages @@ -184,6 +277,101 @@ jobs: - name: Run e2e tests (Kubernetes) run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e + # Each external topology is its own pytest session, because the deployment under test + # is created once per session from INFRAHUB_TESTING_EXTERNAL_DB. + - name: Run e2e tests (Kubernetes, external Neo4j) + if: matrix.edition == 'enterprise' + env: + INFRAHUB_TESTING_EXTERNAL_DB: database + run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e/test_k8s_external_neo4j.py tests/e2e/test_k8s_external_in_namespace.py + + - name: Run e2e tests (Kubernetes, external Neo4j and PostgreSQL) + if: matrix.edition == 'enterprise' + env: + INFRAHUB_TESTING_EXTERNAL_DB: database,task-manager-db + run: exec tini -s -g -- uv run pytest -v -m k8s tests/e2e/test_k8s_external_both.py + + - name: Remove the external databases + if: always() && matrix.edition == 'enterprise' + run: | + docker rm -f ext-minio ext-neo4j ext-postgres 2>/dev/null || true + docker network rm ext-db 2>/dev/null || true + + # A three-member Neo4j cluster, built like Infrahub's HA example, once inside the + # Infrahub namespace and once outside it. Every other Kubernetes case runs against one + # Neo4j server. Each topology runs on its own runner, because one cluster and one + # Infrahub deployment already fill one. + e2e-tests-k8s-neo4j-cluster: + if: | + !cancelled() && !contains(needs.*.result, 'failure') && + (needs.files-changed.outputs.go == 'true' || needs.files-changed.outputs.python == 'true') + needs: ["files-changed", "go-lint", "test"] + runs-on: + group: huge-runners + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + topology: [internal, external] + env: + INFRAHUB_HELM_CHART: oci://registry.opsmill.io/opsmill/chart/infrahub-enterprise + INFRAHUB_TESTING_NEO4J_CLUSTER: ${{ matrix.topology }} + steps: + - uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: "1.25" + + - name: Set up vCluster + uses: loft-sh/setup-vcluster@68e17a8c3618e3c47d635842b803d4f560f694e8 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Set up kubectl + uses: azure/setup-kubectl@v4 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Set up Python and uv + uses: astral-sh/setup-uv@v7 + + - name: Install tini + run: apt-get update && apt-get install -y tini + + - name: Build binaries (infrahub-backup, infrahub-taskmanager, infrahub-collect) + run: make build + + - name: Install Python dependencies + run: uv sync --all-packages + + - name: "Set environment variables" + run: | + RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/') + echo "VCLUSTER_DIR=/var/lib/github/${RUNNER_NAME}/_temp/.vcluster/config.json" >> $GITHUB_ENV + + # The fixtures build the whole topology: the three-node vcluster, the Neo4j cluster, + # the object store and the infrahub-backup image (tests/e2e/conftest.py, "Neo4j + # cluster topologies"). The cases of the other topology skip. + - name: Run e2e tests (Kubernetes, Neo4j cluster) + run: >- + exec tini -s -g -- uv run pytest -v -rs -m k8s + tests/e2e/test_k8s_neo4j_cluster.py tests/e2e/test_k8s_external_neo4j_cluster.py + --junitxml=neo4j-cluster-${{ matrix.topology }}-results.xml + + - name: Upload the test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: neo4j-cluster-${{ matrix.topology }}-results + path: neo4j-cluster-${{ matrix.topology }}-results.xml + if-no-files-found: ignore + nix-build: if: needs.files-changed.outputs.nix == 'true' needs: ["files-changed"] diff --git a/.gitignore b/.gitignore index bd8fbf26..65f95c99 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,11 @@ Thumbs.db # Spec-kit local state (records the active feature per checkout; not shared) .specify/feature.json + +# Spec-kit extension caches +.specify/**/.cache/ + +# Plakar/kloset writes a version-named repository directory into the working +# directory when the e2e suite drives it (e.g. `2.0.0/`). It is test residue, not +# source: a `git add -A` after an e2e run otherwise captures ~30 of its files. +/[0-9]*.[0-9]*.[0-9]*/ diff --git a/AGENTS.md b/AGENTS.md index 3dfb2c98..42b9625a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,26 @@ All three tools share common internal application logic but expose different com - `make fmt` - Format code with go fmt - `make vet` - Run go vet +Notes on running these directly rather than through `make`: + +- `golangci-lint` is installed by `make dev-setup` into `$(go env GOPATH)/bin`, which is **not** + necessarily on `PATH`. Invoking it as a bare command can fail with `command not found` while + `make lint` works. +- The Makefile passes `GO_TAGS=-tags untested_go_version`. Use the same tag when running `go test` + or `go vet` by hand, or results will not match CI. `golangci-lint` needs it too, spelled + `--build-tags untested_go_version`: without it every package fails `typecheck` on undefined + symbols in `cockroachdb/swiss`, which reads as a code error and is not one. +- The linter binary must be a **v2** (`.golangci.yaml` is `version: "2"`) built by a toolchain at + least as new as the one in use. A binary built by an older Go cannot decode the compiler's + export data and reports `export data version N is greater than maximum supported version M` for + every stdlib import — or panics outright in `goanalysis`. `make dev-setup` reinstalls it from + source at the pinned `GOLANGCI_VERSION`; re-run it after a Go upgrade. +- `make vet`, `make lint` and `go test ./...` fail on `tools/neo4jwatchdog` on darwin, because it + uses `unix.Inotify*`. This is expected and unrelated to any change under `src/`; scope to + `./src/...` when working on macOS — for the linter that is + `golangci-lint run --build-tags untested_go_version ./src/...`. CI runs on Linux and lints the + whole tree, so the targets are deliberately not narrowed. + ### Development Setup - `make dev-setup` - Install development dependencies including golangci-lint @@ -58,6 +78,29 @@ Add a fragment in the same PR as the change when it belongs in the release notes creates the GitHub Release, which starts the existing binary, image, and Homebrew publishing workflow. +### Spec-kit extensions: registered but only partly vendored + +`.specify/extensions.yml` lists `critique`, `opsmill` and `review` under `installed:`, but not all +of their assets are present in this repo. A skill step that names one of the missing ones needs a +substitute rather than a retry: + +| Named by a skill | Present? | Substitute | +|---|---|---| +| `speckit-checkpoint-commit` | **no** | Commit inline, following the conventions in `git log` | +| `.specify/templates/critique-template.md` | **no** | Follow the structure the critique command describes | +| git extension (`before_specify` hook) | **no** | `/speckit-specify` creates **no branch** — create one by hand if the work should not sit on `main` | +| `reconcile` command | **no** | Do the substance inline | +| `/speckit-critique-run`, `/speckit-review-run` | yes | Wired as *optional* `after_plan` / `after_implement` hooks, so they are offered, not run | + +Two further hazards when running the pipeline here: + +- `.specify/scripts/bash/update-agent-context.sh` writes its own `## Active Technologies` and + `## Recent Changes` sections into the agent file. `CLAUDE.md` in this repo is deliberately a + one-line `@AGENTS.md` router, so the script corrupts it and truncates values at the first `**`. + Restore `CLAUDE.md` afterwards and hand-write the entries into this file instead. +- `/code-review` reads the **working tree**, not the commit range passed to it. Do not run it while + an implementation subagent holds uncommitted edits. + ### Nix Vendor Hash Whenever Go modules change (any modification to `go.mod` or `go.sum` — adding, removing, or updating dependencies), ALWAYS run `scripts/update-vendor-hash.sh` to update the `vendorHash` in `flake.nix`. Do not compute or edit the vendor hash manually. @@ -112,15 +155,87 @@ The codebase follows a command-pattern architecture using Cobra for CLI structur - `masking.go` - Key-name secret masking for env and config dumps 8. **src/internal/app/utils.go** - Utility functions - - File operations, checksum validation + - File operations, checksum validation, tar/tarball handling - Environment variable handling - - Version detection and comparison + - This tool's own build version (`SetVersion`, `BuildRevision`) from ldflags — **not** a + version comparator. Comparators live elsewhere and are deliberately separate: + `src/internal/updater/version.go` compares this tool's own release tags with strict semver, + and `src/internal/app/external_endpoint.go` compares *database* versions, which must handle + both `5.x.y` and calendar `YYYY.MM.p` schemes. Do not merge them. 9. **src/internal/app/cli.go** - Shared CLI configuration - `ConfigureRootCommand()` - Sets up common flags and configuration - `AttachEnvironmentCommands()` - Adds environment detection commands - Shared between all three binaries +10. **src/internal/app/external_*.go** - Databases that live outside the deployment (spec 007) + - `external_backup_gate.go` - Where the location is settled *before* anything is stopped; + `resolveDatabaseTargets` is the only way a *run* obtains `databaseTarget`s, and holding one + is the evidence that the reachability question was answered. The constructor it calls, + `databaseTargetWith`, is directly callable within the package (tests drive it), so the + evidence is a `gated` field only that constructor sets and both preparers refuse a target + without — not a promise that the type could not keep + - `external_endpoint.go` - Endpoint resolution and discovery, the `databaseTarget` type, and + the *database* version comparator (see the note on `utils.go`) + - `external_capture.go` - The capture sequence: probe, edition/version gates, then the + capture workload. `externalCaptureOps` is the only function a transient object can come + from, so it is where a run that may not create one is refused (`infrahub-collect`, ADR-0003) + - `external_restore.go` - The restore sequence, which is the capture's mirror and not its + copy: one workload created at the gate and held for the run, the artifact staged where the + *server* can fetch it (FR-007), the seed statement in the form the server version accepts, + and the poll that decides whether the seed actually loaded (FR-021) — accepting the request + proves nothing + - `external_ca.go` - Carries the deployment's own certificate authority into the workload + that has to verify against it, once per workload + - `external_credential_refs.go` - Decides how the credentials reach a transient workload: + `secretKeyRef`s onto the deployment's own Secret when the run can show the credentials it + holds came from Secret-backed variables of the component's running pod (no Secret created, + no secrets verbs), or the Job-owned credential Secret otherwise. The pod spec is read with + a references-only jsonpath, so an inline value never leaves the API server + - `external_local.go` - Direct mode (FR-032, ADR 0017): when this process runs as a pod in + the target namespace (service-account namespace file) and finds every tool a database + needs on `PATH`, `externalCaptureOps` returns operations served by a `localWorkload` — a + run-specific scratch directory under `os.TempDir` — and creates no Job or Secret, mints no + run identifier and runs no reaper. The decision is taken after the collect refusal and + before the image refusal, once per database, and logged at info. The sequences do not + fork: the execution target is `local:--` (a colon no pod name can + carry), and the four seams — `execBounded`, `ExecStreamPipe`, + `copyAcrossTransientWorkload`, `refuseUnfittableRestoreDump` — serve such a target here. + `translate` maps `transientScratchPath` onto the scratch directory in argv, env values and + copy paths; credentials go in the child's `Env` only. `external_local_statfs*.go` read free + space (Linux and macOS; refused elsewhere) + - `external_timeout.go` - Every bound and deadline the external path spends + - `transient_workload.go` - The stand-in pod and its credential Secret: manifest + construction, lifecycle, the stray reaper (FR-011) and the FR-028 exclusion. The + exclusion is read from the pod's own `transientLabelMarker` before its name, in that + order: a name test rests on `transientObjectPrefix`, and the fallbacks that reach + `withoutTransientPods` go on to match a name against a service, so a prefix containing a + service name would have them claim the pod rather than merely miss it + +11. **src/internal/app/kubernetes_*.go** - The Kubernetes-side questions the above rests on + - `kubernetes_ownership.go` - The single answer to "whose resource is this?": label + evidence, release prefixes, `deploymentClaimsResource`, `unanchoredNamePolicyFor`, and + `parseLabelledPods`, the one parser for both pod listings. Nothing outside this file may + *decide* ownership from a name — the acceptance test is that + `grep -rn 'nameMatchesService(' src/ | grep -v _test` returns only its definition in + `kubernetes_pod_match.go` and its call sites here. `labelledPod.Labels` keeps the four + service-label values the listing carried, because a label *selector* and a *declaration* + are two readings of the same fields: `podsServiceSelectorsWouldMatch` reproduces the + running check's four `=` queries out of them, in selector order, so those + queries are not issued. Ownership is deliberately not that reproduction — it also claims + a pod carrying a release prefix, and `IsRunning` answering true from one of those is a + stop and a scale to zero on evidence no selector produced + - `kubernetes_pod_match.go` - Pod phases and name shape: the field selectors, their + client-side halves, and `nameMatchesService`, which answers what a name *looks like* and + never what it belongs to + - `kubernetes_workloads.go` - Workload (Deployment/StatefulSet) resolution and scaling + - `environment_kubernetes.go` - The backend itself: exec/copy, pod resolution and its caches + (`dropPodCaches` is what a scale clears), the running check, `locateService`. The running + check costs one kubectl call: it reads one namespace listing, answers the selector tier + from that listing's own label fields and falls back to ownership only where no selector + would have matched. That one call failing *is* the undetermined status, and + `stopAppContainers` fails the run on it + ### Key Design Patterns - **Split Binary Architecture**: Three specialized binaries sharing common internal logic for focused functionality @@ -306,8 +421,11 @@ The codebase uses explicit error wrapping with `fmt.Errorf` for context. All com - Plakar repository (local filesystem or S3 via integration backends) (002-plakar-integration) - Go 1.25.0 + cobra, viper, logrus; Docker/Kubernetes via `docker`/`kubectl` CLI shell-out through `CommandExecutor` — no client-go or Docker SDK (003-collect-tool) - Local filesystem bundle output under `--output-dir` (default `./infrahub_bundles`); no network egress beyond the target deployment (003-collect-tool) +- Go 1.25.0 + no new modules; external databases reached from a transient in-namespace pod created via `kubectl` on stdin, so vendor tooling still runs in a container and the existing exec/stream/copy primitives are unchanged (007-external-database-backup) +- Neo4j endpoint read from `INFRAHUB_DB_ADDRESS` (comma-separated `host[:port]`), `_PORT` (Bolt 7687, *not* the 6362 backup port), `_PROTOCOL`, `_TLS_*` (007-external-database-backup) ## Recent Changes +- 007-external-database-backup: External Neo4j Enterprise / PostgreSQL backup and restore for Kubernetes — transient stand-in pod registered under the existing service names, endpoint discovery from in-namespace component environment, remote restore by seeding from an object-store URI - 003-collect-tool: Added `infrahub-collect` troubleshooting-bundle binary (collector framework, log/metrics primitives on both backends, key-name secret masking, bundle manifest) - 002-plakar-integration: Added kloset (Plakar core library), integration-fs (filesystem storage/exporter), cobra, logrus diff --git a/Dockerfile b/Dockerfile index c83fa4e9..8b66acb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,15 +27,99 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build \ -ldflags "-X main.version=${VERSION} -s -w" \ -o /infrahub-backup ./src/cmd/infrahub-backup +# Neo4j tools stage (FR-030, ADR 0017). +# neo4j-admin and cypher-shell are POSIX sh launchers around Java jars, so only the launchers, the +# jar directory and the default configuration are taken. The Java runtime comes from Alpine in the +# runtime stage. The `products` directory (server plugins) and `data`, `logs`, `run` are not needed +# to run the tools and are left out. +FROM neo4j:2025.10.1-enterprise AS neo4j-tools + +# Jars neo4j-admin and cypher-shell never load: the server's browser, the S3, Azure and GCS client SDKs +# (the tool only writes backups to a local directory), Cypher planner and runtimes, +# HTTP server and metrics, Parquet/Arrow import, and native libraries for macOS and Windows. Checked +# by loading classes during a remote backup, inspect, restore, consistency check and cypher-shell +# over plain and verified TLS Bolt. +FROM alpine:3.23 AS neo4j-lib +COPY --from=neo4j-tools /var/lib/neo4j/lib /opt/neo4j/lib +RUN cd /opt/neo4j/lib && rm -f \ + neo4j-browser-*.jar \ + *-osx-*.jar *-windows-*.jar netty-transport-classes-kqueue-*.jar netty-resolver-dns-classes-macos-*.jar \ + neo4j-ast-*.jar neo4j-codegen-*.jar neo4j-cypher-[0-9]*.jar neo4j-cypher-cache-*.jar \ + neo4j-cypher-compiled-expressions-*.jar neo4j-cypher-config-*.jar neo4j-cypher-expression-evaluator-*.jar \ + neo4j-cypher-interpreted-runtime-*.jar neo4j-cypher-ir-*.jar neo4j-cypher-logical-plans-*.jar \ + neo4j-cypher-macros-*.jar neo4j-cypher-physical-planning-*.jar neo4j-cypher-pipelined-runtime-*.jar \ + neo4j-cypher-planner-*.jar neo4j-cypher-rendering-*.jar neo4j-cypher-runtime-util-*.jar \ + neo4j-cypher-slotted-runtime-*.jar neo4j-expressions-*.jar neo4j-front-end-*.jar neo4j-rewriting-*.jar scala-*.jar \ + aircompressor-*.jar arrow-*.jar flatbuffers-java-*.jar flight-core-*.jar parquet-*.jar snappy-java-*.jar \ + hk2-*.jar jakarta.*.jar javax.annotation-api-*.jar jaxb-*.jar jersey-*.jar \ + jetty-alpn-*.jar jetty-ee*.jar jetty-http*.jar jetty-io-*.jar jetty-security-*.jar jetty-server-*.jar \ + jetty-session-*.jar jetty-util-*.jar jetty-xml-*.jar jts-core-*.jar prometheus-metrics-*.jar \ + annotations-2.*.jar apache-client-2.*.jar arns-2.*.jar auth-2.*.jar aws-*.jar checksums-*.jar \ + crt-core-*.jar endpoints-spi-*.jar http-auth-*.jar http-client-spi-*.jar identity-spi-*.jar \ + json-utils-2.*.jar metrics-spi-*.jar netty-nio-client-*.jar profiles-2.*.jar protocol-core-*.jar \ + regions-*.jar retries-*.jar s3-2.*.jar s3-transfer-manager-*.jar sdk-core-*.jar \ + third-party-jackson-core-*.jar utils-2.*.jar \ + azure-core-[0-9]*.jar azure-core-http-netty-*.jar azure-identity-*.jar azure-json-*.jar \ + azure-storage-*.jar azure-xml-*.jar msal4j-*.jar accessors-smart-*.jar content-type-*.jar \ + json-smart-*.jar lang-tag-*.jar nimbus-jose-jwt-*.jar oauth2-oidc-sdk-*.jar jna-platform-*.jar \ + reactor-core-*.jar reactor-netty-*.jar \ + api-common-*.jar gax-*.jar google-api-*.jar google-auth-*.jar google-cloud-core-*.jar \ + google-cloud-monitoring-*.jar google-http-client-*.jar google-oauth-client-*.jar grpc-*.jar \ + proto-google-*.jar protobuf-java-*.jar opencensus-*.jar perfmark-api-*.jar threetenbp-*.jar \ + guava-*.jar failureaccess-*.jar j2objc-annotations-*.jar conscrypt-openjdk-uber-*.jar + +# Java runtime stage: a jlink runtime with the modules the Neo4j tool jars reference (jdeps) plus +# modules loaded by reflection or service lookup. binutils provides the objcopy that --strip-debug runs. +# No --compress: the image layer compresses an uncompressed lib/modules better than jlink does. +FROM alpine:3.23 AS java-runtime +ARG OPENJDK_VERSION=21.0.13.0.3_p1-r0 +RUN apk add --no-cache "openjdk21-jdk=${OPENJDK_VERSION}" "openjdk21-jmods=${OPENJDK_VERSION}" binutils && \ + /usr/lib/jvm/java-21-openjdk/bin/jlink \ + --add-modules java.base,java.compiler,java.datatransfer,java.desktop,java.instrument,java.logging,java.management,java.naming,java.net.http,java.prefs,java.rmi,java.scripting,java.security.jgss,java.security.sasl,java.sql,java.transaction.xa,java.xml,jdk.attach,jdk.charsets,jdk.crypto.cryptoki,jdk.crypto.ec,jdk.httpserver,jdk.incubator.vector,jdk.jdi,jdk.jfr,jdk.management,jdk.management.jfr,jdk.naming.dns,jdk.net,jdk.security.auth,jdk.unsupported,jdk.zipfs \ + --strip-debug --no-man-pages --no-header-files \ + --output /opt/java + # Runtime stage -FROM alpine:3.21 +# +# alpine:3.23 rather than 3.21: Alpine 3.21 and 3.22 do not package the PostgreSQL 18 client tools +# (they stop at postgresql17-client). The Neo4j tools run on Alpine (musl) with Alpine's OpenJDK 21, +# so the base image stays Alpine (checked by scripts/check-image-tools.sh). +FROM alpine:3.23 + +# Every database tool version is pinned. Alpine removes a package revision from its mirrors when it +# publishes the next one, so a stale pin fails the build loudly instead of drifting silently; bump +# the pin to the version `apk search -x ` reports for this Alpine release. +# - Java 21: the runtime neo4j:2025.10.1-enterprise ships (Temurin 21), here as a jlink runtime built from Alpine's OpenJDK (java-runtime stage) +# - PostgreSQL 18 client: pg_dump, pg_restore and psql for task-manager-db (postgres 18) +ARG POSTGRESQL_CLIENT_VERSION=18.7-r0 + +RUN apk add --no-cache ca-certificates curl kubectl \ + "postgresql18-client=${POSTGRESQL_CLIENT_VERSION}" + +# The Neo4j tools live under NEO4J_HOME. The launchers find their jars relative to their own path, +# and neo4j-admin reads NEO4J_CONF and writes under NEO4J_LOGS. Both point into a directory the +# non-root user owns, so neo4j-admin never needs to write into a root-owned path. +ENV JAVA_HOME=/opt/java \ + NEO4J_HOME=/opt/neo4j \ + NEO4J_CONF=/opt/neo4j/conf \ + NEO4J_LOGS=/opt/neo4j/logs +ENV PATH="${NEO4J_HOME}/bin:${JAVA_HOME}/bin:${PATH}" -RUN apk add --no-cache ca-certificates curl kubectl +COPY --from=neo4j-tools /var/lib/neo4j/bin/cypher-shell /var/lib/neo4j/bin/neo4j-admin /opt/neo4j/bin/ +COPY --from=java-runtime /opt/java /opt/java +COPY --from=neo4j-lib /opt/neo4j/lib /opt/neo4j/lib +COPY --from=neo4j-tools /var/lib/neo4j/conf /opt/neo4j/conf +COPY --from=neo4j-tools /var/lib/neo4j/LICENSE.txt /var/lib/neo4j/LICENSES.txt /var/lib/neo4j/NOTICE.txt /opt/neo4j/ COPY --from=builder /infrahub-backup /usr/local/bin/ -# Run as non-root user -RUN adduser -D -u 1000 infrahub +# Run as non-root user. The user owns NEO4J_HOME itself (not its bin and lib) and the directories +# neo4j-admin writes to; otherwise neo4j-admin warns on every run that the owner of NEO4J_HOME +# differs from the user running it. +RUN adduser -D -u 1000 infrahub && \ + mkdir -p /opt/neo4j/logs /opt/neo4j/data /opt/neo4j/run && \ + chown infrahub:infrahub /opt/neo4j && \ + chown -R infrahub:infrahub /opt/neo4j/logs /opt/neo4j/data /opt/neo4j/run /opt/neo4j/conf USER infrahub ENTRYPOINT ["/usr/local/bin/infrahub-backup"] diff --git a/Makefile b/Makefile index b22e1fd5..c7c7a7e4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-all clean install test lint fmt vet help docker-build docker-build-multi docker-push +.PHONY: build build-all clean install test lint fmt vet help docker-build docker-check-tools docker-build-multi docker-push # Variables BINARIES=infrahub-backup infrahub-taskmanager infrahub-collect @@ -7,7 +7,14 @@ SRC_ROOT=./src VERSION?= LDFLAGS=-ldflags "-X main.version=$(VERSION) -s -w" # cockroachdb/swiss (transitive dep of pebble cache) needs this tag for Go 1.26+ -GO_TAGS=-tags untested_go_version +GO_BUILD_TAGS=untested_go_version +GO_TAGS=-tags $(GO_BUILD_TAGS) +# golangci-lint must be a v2 to read .golangci.yaml (version: "2"), and must be +# built by a toolchain at least as new as the one in use — an older binary +# cannot read the compiler's export data and fails every package on typecheck. +# Installed from source rather than by the upstream script so it is built with +# whichever Go is installed here. +GOLANGCI_VERSION=v2.13.2 CGO_ENABLED=0 # Default target @@ -71,7 +78,7 @@ test-coverage: ## Run tests with coverage lint: ## Run golangci-lint @echo "Running linter..." - @golangci-lint run + @golangci-lint run --build-tags $(GO_BUILD_TAGS) fmt: ## Format Go code @echo "Formatting code..." @@ -105,7 +112,7 @@ run-example: build ## Run example commands dev-setup: ## Set up development environment @echo "Setting up development environment..." @go mod download - @which golangci-lint > /dev/null || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.54.2 + @go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_VERSION) @echo "Development environment ready!" DOCKER_REGISTRY?=registry.opsmill.io/opsmill @@ -117,6 +124,9 @@ docker-build: ## Build Docker image for current platform -t $(DOCKER_REGISTRY)/$(DOCKER_IMAGE):$(VERSION) \ -t $(DOCKER_REGISTRY)/$(DOCKER_IMAGE):latest . +docker-check-tools: ## Build the image for the current platform and check it carries the database tools + @VERSION=$(VERSION) ./scripts/check-image-tools.sh $(DOCKER_IMAGE):check-tools + docker-build-multi: ## Build multi-arch Docker images (amd64 + arm64) @echo "Building multi-arch Docker images..." @docker buildx build --platform linux/amd64,linux/arm64 \ diff --git a/changelog/+explicit-target-pins-backend.fixed.md b/changelog/+explicit-target-pins-backend.fixed.md new file mode 100644 index 00000000..5dcd8908 --- /dev/null +++ b/changelog/+explicit-target-pins-backend.fixed.md @@ -0,0 +1 @@ +When `--k8s-namespace` or `--project` is set, the tools now act only on that deployment. Previously, if the Kubernetes namespace could not be checked, they went on to detect a Docker Compose project on the same host and could back up or restore that project instead. diff --git a/changelog/+kubernetes-scale-permissions.fixed.md b/changelog/+kubernetes-scale-permissions.fixed.md new file mode 100644 index 00000000..b6acbcff --- /dev/null +++ b/changelog/+kubernetes-scale-permissions.fixed.md @@ -0,0 +1 @@ +The Kubernetes backup permissions now list `list` on `deployments` and `statefulsets` and `get` and `patch` on `deployments/scale` and `statefulsets/scale`. A Role that granted only `get` and `patch` on the workloads, as the documentation previously said, cannot stop and restart the Infrahub components, because `kubectl scale` writes to the `scale` resources. diff --git a/changelog/+namespace-check-reason.fixed.md b/changelog/+namespace-check-reason.fixed.md new file mode 100644 index 00000000..624ee1c6 --- /dev/null +++ b/changelog/+namespace-check-reason.fixed.md @@ -0,0 +1 @@ +When `--k8s-namespace` is set and the namespace cannot be checked, the error now includes what `kubectl` reported, such as a `forbidden` refusal or a missing namespace, instead of only `exit status 1`. diff --git a/changelog/172.added.md b/changelog/172.added.md new file mode 100644 index 00000000..8c4b0a22 --- /dev/null +++ b/changelog/172.added.md @@ -0,0 +1 @@ +Kubernetes deployments whose Neo4j Enterprise or PostgreSQL task-manager database runs outside the cluster can now be backed up and restored with the same `infrahub-backup create` and `restore` commands as any other deployment. Artifacts taken from an external database are byte-compatible with internal ones and restore by the same path, and deployments whose databases are all internal behave exactly as before. When the deployment's own pods take the database username and password from named Secret keys (`env[].valueFrom.secretKeyRef`), the backup reads the same keys and needs no permissions on secrets. Credentials from any other source, including `envFrom`, need permission to create a Secret for the run. When `infrahub-backup` itself runs as a pod in the Infrahub namespace, it runs the database tools from its own image and creates no Job or Secret, so it needs only the permissions of a backup of internal databases. diff --git a/changelog/172.fixed.md b/changelog/172.fixed.md new file mode 100644 index 00000000..5dd5defb --- /dev/null +++ b/changelog/172.fixed.md @@ -0,0 +1 @@ +On Docker, a restore now waits for the database container to be healthy again before it starts `infrahub-server` and `task-worker`, and a failed `docker compose start` or `stop` reports what compose printed instead of only its exit status. diff --git a/dev/README.md b/dev/README.md index e8972212..ff45f9e7 100644 --- a/dev/README.md +++ b/dev/README.md @@ -18,11 +18,24 @@ under `specs/archive/`. - [0005-uniform-bundle-and-manifest.md](adr/0005-uniform-bundle-and-manifest.md) - Single `.tar.gz` bundle + manifest, uniform across environments - [0006-include-backup-reuses-createbackup.md](adr/0006-include-backup-reuses-createbackup.md) - `--include-backup` reuses `CreateBackup` non-interactively - [0007-offline-by-default-opt-in-benchmark.md](adr/0007-offline-by-default-opt-in-benchmark.md) - Offline by default; benchmark/image pulls opt-in +- [0008-transient-workload-for-external-databases.md](adr/0008-transient-workload-for-external-databases.md) - A load-bearing image pull for external databases; offline-by-default survives it +- [0009-transient-workload-lifecycle-and-credential-ownership.md](adr/0009-transient-workload-lifecycle-and-credential-ownership.md) - Transient pod from a manifest on stdin, deadline-bounded, owning its credential Secret +- [0010-neo4j-restore-by-seed-uri.md](adr/0010-neo4j-restore-by-seed-uri.md) - Remote Neo4j restore by seeding from an object-store URI, confirmed by polling +- [0011-asymmetric-version-compatibility.md](adr/0011-asymmetric-version-compatibility.md) - Abort when the utility is older than the server, warn when newer; two version schemes +- [0012-location-from-positive-evidence.md](adr/0012-location-from-positive-evidence.md) - Database location from positive in-deployment evidence; a failed query is never "external" +- [0013-incomplete-captures-removed-metadata-additive.md](adr/0013-incomplete-captures-removed-metadata-additive.md) - Incomplete captures removed, completeness read from output, metadata additive +- [0014-separate-external-restore-authorisation.md](adr/0014-separate-external-restore-authorisation.md) - External restore needs its own authorisation; the granting channel is recorded +- [0015-verify-by-default-external-tls.md](adr/0015-verify-by-default-external-tls.md) - External connections encrypted and verified by default, one operator opt-out +- [0016-single-ownership-decider.md](adr/0016-single-ownership-decider.md) - One ownership decider from label evidence and release prefixes, never a bare name ## Current Knowledge - [infrahub-collect.md](knowledge/infrahub-collect.md) - The troubleshooting-bundle tool: CLI, bundle layout, manifest, backend seam +- [external-database-backup.md](knowledge/external-database-backup.md) - Backing up and restoring databases outside the deployment: gate, transient workload, discovery, CLI, prerequisites, metadata, bounds +- [container-image-user-semantics.md](knowledge/container-image-user-semantics.md) - The official database images declare no user; what that means for a pod's security context ## Current Guidelines - [collectors.md](guidelines/collectors.md) - Conventions for writing and maintaining collect collectors +- [kubernetes-backend.md](guidelines/kubernetes-backend.md) - Rules for code that talks to a Kubernetes deployment: stdout-only reads, one ownership decider, bounded operations, credentials, additive metadata +- [implementation-verification.md](guidelines/implementation-verification.md) - Grep for callers of every new symbol per chunk; verify external premises with a named command diff --git a/dev/adr/0003-read-only-collection.md b/dev/adr/0003-read-only-collection.md index 33b1ddec..699bb8d7 100644 --- a/dev/adr/0003-read-only-collection.md +++ b/dev/adr/0003-read-only-collection.md @@ -17,10 +17,13 @@ Two deliberate exceptions, both operating on the **tool's own** transient resour - `--benchmark` creates and then removes a short-lived benchmark container/pod it owns. - `--include-backup` delegates to `CreateBackup`, whose documented behaviour may stop/restart app containers; this is opt-in and called out in the docs (see ADR 0006). +Neither exception extends to creating or deleting objects that stand in for the deployment's own. Backing up a database that lives **outside** the cluster (spec 007) needs a transient Pod and Secret in the deployment's namespace, so `--include-backup` **declines** when a database resolves as external, naming `infrahub-backup create` as the tool that performs it. The decision above is therefore unqualified: nothing `infrahub-collect` does creates, deletes, or scales a workload in the deployment's namespace. `CreateBackup` is still reused unmodified (ADR 0006) — the run stops at the external path precisely so that this guarantee stays true rather than becoming a carve-out. + ## Consequences - Safe to run against a live or degraded production instance without risk of an outage attributable to the tool. - Kubernetes RBAC for the tool needs only read/exec verbs — no write or scale permissions. +- A deployment whose databases live outside the cluster gets no backup inside its bundle. That is the accepted cost: a bundle tool a customer can be handed because it provably cannot mutate their cluster is worth more than a backup collected alongside the diagnostics, and `infrahub-backup create` still takes that backup. - Interrupt handling is simpler: there is no workload state to restore on SIGINT, only a staging directory to clean up. ## Alternatives Considered diff --git a/dev/adr/0006-include-backup-reuses-createbackup.md b/dev/adr/0006-include-backup-reuses-createbackup.md index f2df368a..2a2b7321 100644 --- a/dev/adr/0006-include-backup-reuses-createbackup.md +++ b/dev/adr/0006-include-backup-reuses-createbackup.md @@ -12,6 +12,8 @@ Support frequently asks for "logs plus a backup" so they can reproduce a custome `--include-backup` runs a collector that delegates to the existing `CreateBackup` unmodified. The produced backup is a **standalone** `infrahub_backup_*.tar.gz` in the standard backup directory — referenced by the manifest's `artifact` field, **not embedded** in the bundle. The collector runs last in the plan (after every read-only collector), because the delegated backup may stop/restart app containers and must not taint the diagnostics. A backup failure is non-fatal: the bundle is still produced (US3 scenario 2). +"Unmodified" has one boundary, and it is a refusal rather than a modification. `CreateBackup` against a database that lives outside the cluster (spec 007) creates and then deletes a transient Pod and Secret in the deployment's namespace, which ADR 0003 forbids `infrahub-collect` without qualification. So a collection run is marked as one that may not capture an external database, and the delegated backup stops at that gate — before anything is stopped and before any object is created — with a message naming `infrahub-backup create`. The delegation itself is untouched: no divergent backup path exists, and the internal path is byte-for-byte the one the backup tool takes. + The backup is invoked with `force=true`. Collection is non-interactive and designed never to hang; `CreateBackup(force=false)` runs `waitForRunningTasks`, an unbounded loop that returns only when no Prefect tasks are running/pending, which never fully drains on a busy instance (e.g. the Enterprise edition's recurring background tasks). `--force` skips only that consistency gate; it does not weaken the backup's own integrity guarantees. ## Consequences diff --git a/dev/adr/0008-transient-workload-for-external-databases.md b/dev/adr/0008-transient-workload-for-external-databases.md new file mode 100644 index 00000000..a3c9a099 --- /dev/null +++ b/dev/adr/0008-transient-workload-for-external-databases.md @@ -0,0 +1,92 @@ +# 8. A load-bearing image pull for external databases; offline-by-default survives it + +**Status**: Accepted; the image choice is superseded by ADR 0017 +**Date**: 2026-09-03 +**Source**: specs/archive/007-external-database-backup/research.md (R1, R5), spec FR-004/FR-015/FR-019/FR-027 + +## Context + +ADR 0007 established offline-by-default as the posture for these tools: they run in air-gapped and +egress-restricted networks, so a default run performs no image pull, and the one feature that does +— `--benchmark` — is opt-in and records `skipped` when the image cannot be pulled. + +Spec 007 backs up a Neo4j or PostgreSQL database that lives outside the deployment. Spec 006 +settled that vendor utilities run inside a container and bytes stream out through the primitives +both deployment backends implement. An external database has no container in the deployment to run +those utilities in, and no remote protocol exists that would let this tool read a Neo4j store or a +PostgreSQL cluster without them. So reaching it means the tool creates a pod running a vendor +tooling image in the deployment's namespace — a backup client, in the vendor's own term (research +R1) — and that pod's image has to be pulled. + +That pull is not the benchmark's. It cannot degrade to `skipped`, because there is nothing left to +do without it. + +## Decision + +The transient workload's image pull is **load-bearing**: when a database resolves as external, a +pull that fails is a failed backup, reported as a failure and not as a skip. There is no +degradation path, because a run that skipped it would have captured nothing. + +Offline-by-default is narrowed rather than abandoned, and the narrowing is exactly one condition +wide: + +- A deployment whose databases are all internal pulls nothing, consults no new setting to a + different effect, and behaves as it did before (FR-015). ADR 0007's guarantee holds unchanged + for every deployment shape it was written about. +- Only a database positively established as external reaches the pull, and only after the + deployment query that established it succeeded. A query that failed is never read as "external" + (FR-002), so a permission problem cannot manufacture a pull. +- The image is pinned in the binary (`neo4j:-enterprise`, `postgres:18-alpine`) and can + be pointed at a mirror by `--external-db-image-neo4j` / `--external-db-image-postgres`. Those + two flags are deliberately **not** environment-bound (FR-027): the tool holds workload-creation + rights in the customer's namespace, so an environment-settable image would turn those rights + into a means of running arbitrary code there. +- What an operator's own cluster requires of a pod it did not author *is* environment-bound — + `--external-db-image-pull-secrets`, `--external-db-service-account`, + `--external-db-node-selector`, `--external-db-tolerations`, `--external-db-priority-class`. + These say where a pod may run and which credentials fetch its image, never what it runs, so + FR-027's reason does not reach them, and an unattended backup is configured by environment at + least as often as on the command line. + +## Consequences + +- An air-gapped operator with an external database must mirror the tooling image into a registry + the cluster can reach. That is now a stated prerequisite rather than a discovered one, and the + pull secret for such a registry is nameable — without it the pod sat in `ImagePullBackOff` until + the readiness bound expired. +- Only a deployment with a database outside it needs new permissions (FR-019). A deployment whose + databases are all internal needs none beyond what it needed before (FR-015). The permissions an + external database needs are listed in ADR 0009. They are added to the write permissions the + Kubernetes path already needs to stop and restart Infrahub: `patch` on the `scale` resources of + its Deployments and StatefulSets, which `kubectl scale` writes to. +- `infrahub-collect` does not inherit any of this. `--include-backup` refuses when a database + resolves as external, naming `infrahub-backup create` as the tool that performs the capture, so + ADR 0003's read-only guarantee stays unqualified and collect still needs only read and exec + verbs (see ADR 0003 and ADR 0006). +- External Neo4j Community is refused outright (FR-008). Its backup mechanism needs the database + stopped and its storage directly accessible, which no network-reachable design satisfies, so + there is no image that would make the pull worth attempting. +- The failure surface grows by one condition the operator can act on, which is the trade being + made: a pull that fails is loud and names the flag that fixes it, rather than a capture that + quietly did not happen. + +## Alternatives Considered + +- **Treat a failed pull as `skipped`, as the benchmark does** — rejected. The benchmark's skip + leaves a complete bundle missing one optional section; this skip would leave a deployment with + no recovery point while the run reported success. That is the silent failure constitution + Principle II exists to prevent. +- **Run the vendor utility on the operator's host** — deferred rather than rejected (spec User + Story 3). It needs a JVM on the operator's machine, and the vendor recommends a dedicated + machine because the backup command uses significant memory and processing. It stays specified as + the answer for a customer who will not grant workload-creation rights. +- **Exec the utility inside an existing application pod** — rejected. Infrahub's images carry no + `neo4j-admin`, and borrowing an application pod's resources for a backup is what the vendor + guidance argues against. +- **Vendor the database tooling into this tool's own binary** — rejected on licensing and size, + and it reverses spec 006's decision that vendor utilities run in a container. +- **Write the backup straight to cloud storage with `--to-path`** — rejected as the primary path + in research R1: it bypasses the snapshot pipeline and with it repeat-block removal, encryption, + retention and checksums, and still needs local scratch space. It is documented as a manual + escape hatch for customers who cannot meet the restore prerequisites, not as a way to avoid the + pull. diff --git a/dev/adr/0009-transient-workload-lifecycle-and-credential-ownership.md b/dev/adr/0009-transient-workload-lifecycle-and-credential-ownership.md new file mode 100644 index 00000000..242db603 --- /dev/null +++ b/dev/adr/0009-transient-workload-lifecycle-and-credential-ownership.md @@ -0,0 +1,87 @@ +# 9. Transient workload lifecycle: a Pod manifest on stdin, deadline-bounded, owning its credential Secret + +**Status**: Accepted; the bare-Pod choice and the Secret owner are superseded by ADR 0018 +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/research.md (R5, R7), spec FR-011/FR-014/FR-023/FR-028, data-model.md (`TransientWorkload`) + +## Context + +ADR 0008 settled that a database living outside the deployment is reached from a stand-in pod the +tool creates in the namespace. That pod carries the database password, needs a run-specific +identity, a resource and scratch allocation sized to the database, and a bounded life — and the +tool that created it can be killed at any point, including between creating the credential and +deleting the pod. FR-011 requires the workload gone on every exit path including abnormal +termination; FR-023 requires the credential unable to outlive the run; FR-014 forbids credentials +on a command line or in workload configuration a deployment observer can read. + +ADR 0001 forbids a Kubernetes client library, and the backend already drives everything through +`kubectl` subprocesses, including a primitive that writes a command's stdin. + +## Decision + +- **A full `Pod` manifest, piped to `kubectl create -f -` on stdin** through the executor's + write-pipe primitive, and reaped with `kubectl delete`. A manifest rather than `kubectl run`, + because the pod needs labels, a deadline, resource limits, ephemeral storage sizing, a + restricted security context and a Secret reference, all of which a manifest states directly. +- **The lifecycle bound lives on the object.** `restartPolicy: Never`; `activeDeadlineSeconds` + computed per role as readiness timeout + (calls the pod hosts × the bound each runs under) + + margin, in one function (`externalDBWorkloadDeadline`), so the cluster reclaims the pod after + the tool's own timeouts would have reported a stall, never before; a run-ID label; and a marker + label (`infrahub-backup/transient=true`) plus a name prefix (`infrahub-backup-xdb`) that contains + no service name. +- **Credentials travel in a Secret referenced by `envFrom`, owned by the pod.** The Secret is + created after the pod so it can carry an `ownerReference` to the pod's UID, and the cluster's + garbage collector removes it with the pod. The UID is read from the create's stdout alone: a + kubectl notice on stderr merged into it produces an owner reference no live pod matches, and a + Secret that outlives the run. +- **Three roles, each sized for its work**: a *probe* (fixed 1 GiB scratch, short deadline) that + reads version, edition, store size and member roles before anything is stopped; a *capture* + (scratch = 2× the queried store size, or `--external-db-scratch-size`) created around the + capture itself; a *restore* workload (10 GiB default) created at the gate and held for the run. + Probe and capture never hold scratch in the namespace at the same time. +- **A stray reaper** runs before a workload is created: pods and Secrets carrying the marker from + an earlier run are deleted when they are terminated or past their own deadline. A live peer's + workload is left alone, and an unreadable field leaves an object for the cluster's deadline + rather than guessing. +- **FR-028 exclusion**: every pod listing drops transient pods, reading the marker label first and + the name prefix second, so the stand-in never appears in service enumeration, replica counts or + diagnostic collection. + +## Consequences + +- Only a deployment with a database outside it needs new permissions. A deployment whose databases + are all internal never reaches this path and needs no permission beyond what it needed before + (FR-015). When a database is external, the namespace must grant: + - `create`, `get` and `delete` on jobs + - `get`, `list`, `watch` and `delete` on pods + - `create` on `pods/exec` + - `create`, `list` and `delete` on secrets, only when the run cannot reference the deployment's + own Secret keys and has to create its own credential Secret (see + `external_credential_refs.go`) + + `watch` is not optional: readiness is awaited with `kubectl wait`, which opens a watch. These are + the first write permissions these tools require. +- A tool killed mid-run leaves nothing that the cluster will not clean up by itself: the deadline + removes the pod, ownership removes the Secret. +- Each workload costs five kubectl calls (create pod, read UID, create Secret, wait, delete) plus + scheduling and an image pull; a backup of one external database creates two workloads. +- `ExecOptions.Env` becomes an `env KEY=VALUE` prefix on the exec command line, which is exactly + the exposure FR-014 forbids, so client commands never receive credentials that way; they read + them from the Secret the pod mounts. +- The deadline arithmetic has a history: budgeting for one bounded call where the pod hosted two + put the deadline inside the window a capture was still legitimately working in, and the cluster + deleted a completed capture part-way through the copy out. The arithmetic is therefore one + function, and only the per-role call count and bound vary. + +## Alternatives Considered + +- **`kubectl run` with `--overrides`** — the same result with more awkward escaping. +- **A `Job` rather than a `Pod`** — adds a controller and a second object to reap, and the tool + wants to `exec` into a specific pod anyway. +- **`kubectl debug`** — attaches to an existing pod, which is the case that does not exist here. +- **Inline `env` values in the pod spec** — readable by anyone who can `get pod`. +- **Credentials on stdin** — works for `pg_dump` through a passfile indirection, not uniformly for + `neo4j-admin`. +- **A projected service-account token** — these are database credentials, not cluster credentials. +- **Cleanup by the tool remembering to delete the Secret** — the requirement is precisely that the + tool need not survive to do it. diff --git a/dev/adr/0010-neo4j-restore-by-seed-uri.md b/dev/adr/0010-neo4j-restore-by-seed-uri.md new file mode 100644 index 00000000..21633665 --- /dev/null +++ b/dev/adr/0010-neo4j-restore-by-seed-uri.md @@ -0,0 +1,70 @@ +# 10. Remote Neo4j restore by seeding from an object-store URI, confirmed by polling database status + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/research.md (R2), spec FR-007/FR-020/FR-021, contracts/artifact-and-metadata.md ("Restore-side staging") + +## Context + +Every restore path in this toolset runs the vendor's restore utility inside the database container +against the server's own store. An external Neo4j Enterprise has no container in the deployment +and its store is not reachable over any network protocol the tool speaks. The vendor documents +two Cypher-level ways to restore a backup artifact on standalone and clustered servers; both make +the *server* fetch the artifact. That inverts the data path: instead of the tool moving bytes into +a container, the tool has to put the artifact somewhere the server can read and then ask. + +## Decision + +- **Seed from a URI over Bolt**, from the restore workload: + `CREATE OR REPLACE DATABASE OPTIONS { existingData: 'use', seedURI: '' }` where the + server version supports the atomic replace (2025.01 and later); an older server gets the + two-statement form. Executed in the form the probed server version accepts, never guessed. +- **Cloud object stores only**: `s3`, `gs` and `azb` are the seed providers a server carries as + shipped. Every other scheme (`http`, `https`, `file`) needs `dbms.databases.seed_from_uri_providers` + configured on each server, which the tool cannot observe and must not depend on. +- **Staging reuses the operator's configured object store** (`--s3-*`), under one extra key + segment: `s3:////seed/-.backup`. The raw vendor artifact is + staged, extracted from the archive, not the archive itself. A run with no object store configured + is refused before anything destructive, naming the flags. +- **Readability is verified by an independent reader before the destructive step** (FR-007): a + client built from the URI that will be published, not the one that uploaded, stats the object. + A prefix that does not compose or a bucket the endpoint resolves differently fails here, not as + a database left offline. +- **Acceptance is not completion** (FR-021). Seed download and validation happen only when the new + database starts, and a failure surfaces as `SHOW DATABASES` reporting the database offline with + a `statusMessage`. The tool polls that listing every 5 seconds within the operation bound, reads + every member's row, and reports success only when the database is online. A status that can no + longer be read at all is attributed to the path to the database, not to the database. +- **The staged object is removed only after the database reports online.** Until then the server + may still be fetching it. A failed restore leaves it in place with its URI in the error. +- **Both databases or neither** (FR-020). A restore that cannot reach one database refuses rather + than restoring the other and leaving two recovery points. + +## Consequences + +- Customer prerequisites that no earlier feature had: each Neo4j server needs a route to the + object store and, for the current `CloudSeedProvider`, the cloud provider's CLI and credentials + configured server-side. These are documented, not assumed, and the tool cannot verify them from + outside. +- The extra key segment keeps the seed invisible to retention, which recognises an object only + where its key is exactly the one a backup's own name produces. No keep slot is consumed and no + good backup is evicted. +- An external Neo4j Community database cannot be restored this way — seeding creates a database, + which Community cannot do — and is refused with a message saying so (FR-008's restore side). +- Restore version tolerance is forward-only: an artifact restores into the same or a later server + version. + +## Alternatives Considered + +- **`neo4j-admin database restore`** — requires filesystem access to the server's store. +- **`dbms.recreateDatabase()`** — a genuine second Cypher-level option; not chosen for v1 because + seed-from-URI is the documented restore-from-backup path and simpler to reason about. Worth + revisiting if the seed prerequisites prove onerous. +- **`neo4j-admin database copy` from a backup** (2026.03+) — too new to require of customers. +- **`S3SeedProvider` with `seedCredentials`** — deprecated in 5.26 and needs identical keystores on + every cluster member; credentials would also travel inside a Cypher statement. +- **HTTP or file seeds** — need server-side provider configuration the tool cannot see. +- **Operator-staged artifact** — considered and left out of v1: it breaks the promise of no manual + database-side steps. +- **Report success on `CREATE DATABASE` returning** — the silent data-loss failure Principle II + exists to prevent. diff --git a/dev/adr/0011-asymmetric-version-compatibility.md b/dev/adr/0011-asymmetric-version-compatibility.md new file mode 100644 index 00000000..cc2cb38c --- /dev/null +++ b/dev/adr/0011-asymmetric-version-compatibility.md @@ -0,0 +1,52 @@ +# 11. Asymmetric client/server version compatibility across two version schemes + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/research.md (R3), spec FR-006, Clarifications (session 2026-09-02) + +## Context + +The backup utility for an external database runs in the transient pod's image, and its version is +chosen by the operator (a pinned default, overridable by flag). The server's version is whatever +the customer runs. The spec originally assumed a strict major.minor match was required, which the +vendor documentation does not support: for the backup command no client/server matching rule is +documented at all, and for restore the documented rule is forward tolerance (an artifact restores +into the same or a later version). Neo4j has also moved from `5.x.y` to calendar versions +(`YYYY.MM.p`), so a naive comparison silently misorders the two schemes. + +For PostgreSQL the rule is documented: `pg_dump` reads servers older than itself and refuses +servers newer than itself, loudly. + +## Decision + +- **Probe the live server version before any data is read or written**, from the transient pod, + and compare it with the utility version the same pod carries. +- **Asymmetric enforcement**: abort, naming both versions, when the utility is *older* than the + server; warn and continue when it is *newer*. Only the direction with a known failure mode is an + abort condition. +- **One database-version comparator that understands both schemes**, ordering every calendar + version after every `5.x.y` one. It lives beside endpoint resolution and is deliberately not the + comparator the self-updater uses for this tool's own semver release tags; the two must not be + merged. +- **A version the server did not report is a failure to determine, not a pass.** The same holds for + the edition: a probe that did not answer never selects a branch. +- **The exact compatibility envelope is measured, not asserted**: the end-to-end suite establishes + it against real server versions, and the rule is not tightened on the page. + +## Consequences + +- The tool does not refuse combinations that work, which a symmetric rule would have done on + every minor-version skew. +- The one direction that can produce a corrupt or refused artifact is caught before a byte moves, + and the message names which version must change. +- Two version comparators exist in the codebase on purpose, with a note in `AGENTS.md` saying why. +- The remaining uncertainty is recorded as a known unknown rather than encoded as an abort. + +## Alternatives Considered + +- **Strict major.minor matching** — asserted a rule the vendor does not document, and would abort + on combinations that work. +- **Two-phase discovery** (start a default image, probe, then start a version-matched one) — two + pod lifecycles per run and a second failure surface, to derive something the operator knows. +- **A single semver comparator for everything** — misorders `2025.10.1` against `5.26.0`, and the + updater's strict semver would reject calendar versions outright. diff --git a/dev/adr/0012-location-from-positive-evidence.md b/dev/adr/0012-location-from-positive-evidence.md new file mode 100644 index 00000000..96b8946a --- /dev/null +++ b/dev/adr/0012-location-from-positive-evidence.md @@ -0,0 +1,63 @@ +# 12. Database location is decided from positive in-deployment evidence; a failed query is never "external" + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/research.md (R4), spec FR-001/FR-002/FR-003/FR-004, data-model.md (`DatabaseEndpoint`) + +## Context + +Before spec 007 the tool resolved a service name to a container and failed when none existed. The +external path gives that failure a second meaning — "the database lives outside the deployment" — +and the two meanings have opposite remedies. A permission problem, an unreachable control plane +or a misidentified namespace also yields "no container"; reading any of those as "external" would +send the tool to create workloads and pull images on the strength of a question that was never +answered, and would tell an operator to configure an external database they do not have. + +Discovery also has a limit: the components running inside the deployment carry the external host +list, protocol, client port and TLS settings in their environment, but the Neo4j *backup* listener +port is not part of Infrahub's configuration at all, because Infrahub never uses it. + +## Decision + +- **Location is decided per database at a gate, before anything is stopped**, and a run obtains + its database targets only from that gate. Holding a target is the evidence the question was + answered; both the capture and the restore preparers refuse a target that did not come through + it. +- **Internal** when a pod or workload the deployment claims exists (see ADR 0016 for what "claims" + means). **External** only when the namespace listing *succeeded* and nothing claimed the service. +- **A query that failed is an error, never a location.** The refusal names the query that failed + and the permission that would be missing, and explicitly does not say the database is absent. + This is what keeps a permission problem from manufacturing a transient pod or an image pull. +- **The endpoint is read from the deployment's own components**: `INFRAHUB_DB_ADDRESS` (a + comma-separated `host[:port]` list, already the shape the backup command wants), `_PORT` (the + Bolt port), `_PROTOCOL`, `_TLS_ENABLED`/`_TLS_INSECURE`/`_TLS_CA_FILE`; for PostgreSQL, the + Prefect connection URL — reading only what the string itself states, because `pgx` fills an + absent host from the *operator's* `PGHOST` and would make the operator's laptop the discovered + database. +- **The backup port is defaulted (6362) and overridable**, and every discovered value has a + per-database flag override. The resolved endpoint is reported before any connection is opened. +- **The decision is memoised per run**: a database does not move out of the deployment + mid-backup, and a restore's scale-down changes which pods exist, not where the database lives. + +## Consequences + +- A deployment whose databases are all internal never reaches a new setting, flag or pull; its path + is unchanged (FR-015). +- Discovery is partial by construction and says so: the message for an undiscoverable endpoint + names where it looked and the override flag, while the message for settings that could *not be + read* names the cluster's reason and both remedies — two different conditions, two different + texts. +- Overriding one database's endpoint never changes the other's resolution. +- Every downstream reader of "where does this database live" can read a memo instead of asking + the cluster again. + +## Alternatives Considered + +- **Treat "no container found" as external** — the original failure mode, restated as a feature. + Rejected: a denied listing would silently become a workload-creating path. +- **Operator declares the location** — pushes onto the operator a fact the deployment already + records, and offers no protection against a wrong declaration. +- **Dial the database from the operator's host** — deferred as User Story 3; needs the database to + accept inbound connections from wherever the operator sits, the harder ask in practice. +- **Adopt `pgx`'s parsed host and port** — pgx always supplies both, from libpq defaults, so a URL + with no host resolved to the machine running the tool. diff --git a/dev/adr/0013-incomplete-captures-removed-metadata-additive.md b/dev/adr/0013-incomplete-captures-removed-metadata-additive.md new file mode 100644 index 00000000..5ae2cdf5 --- /dev/null +++ b/dev/adr/0013-incomplete-captures-removed-metadata-additive.md @@ -0,0 +1,61 @@ +# 13. Incomplete captures are removed, never retained; completeness is read from output; metadata stays additive + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/research.md (R6), spec FR-005/FR-012/FR-016/FR-024, contracts/artifact-and-metadata.md + +## Context + +The vendor's backup command exits `1` both when the backup failed and when it "succeeded but +encountered problems such as some servers being uncontactable". FR-005 encourages supplying several +endpoints, which makes the partial case more likely, and FR-012 forbids presenting a partial +capture as a complete restore point. Retention decides from artifact names and recency ranks and +never reads metadata, and an older release of the tool ignores metadata fields it does not know. +Had an incomplete artifact been retained, it would have occupied a keep slot, evicted a good +backup, and been restorable by an older tool without notice. + +The artifact is also a cross-version contract: the constitution requires backups from prior +releases to remain restorable, or a documented migration. + +## Decision + +- **Completeness is derived from what the backup command reported, not from its exit status.** A + non-zero exit is a failure by default; the command's own output identifies the "succeeded with + uncontactable servers" case. +- **An incomplete capture is not retained.** Either every requested database is captured, or the + run fails and the partial artifact is removed from every location it reached, including the + object store. Retention and restore-point selection are never required to interpret metadata to + honour this. +- **Metadata additions are additive and carry a documented meaning for absence**: + `source_endpoints` (the list supplied, in try order), `source_roles` (each member's observed + role, where `unknown` is a real value distinct from the field being absent), `capture_complete`, + and `external_location` per database. A reader finding none of them treats the artifact as an + internal, complete capture from an unrecorded endpoint — which is exactly what every previous + release produced. +- **The metadata version does not change** (FR-024). Compatibility holds in both directions: a new + tool reads an old artifact, and an old tool reads a new one. +- **No field claims which endpoint served the capture.** With several endpoints supplied, no + documented interface reports it; the metadata records enough for an operator to see that a + follower *could* have served it, which is the fact that bears on the recovery point. + +## Consequences + +- `capture_complete: false` should never be read: it is defence in depth for the case where + removal itself fails, and a reader that meets it must not offer the artifact as a restore point. +- Retention code is untouched by the feature, and a partially failed multi-endpoint capture leaves + the retention policy's decisions identical to a run that failed before writing anything. +- Artifact layout is byte-compatible between internal and external captures, so an artifact + taken from an external database restores into an internal deployment and vice versa — a test + obligation, not only a statement. +- Any future change that renames a field, changes its meaning or alters the layout is a breaking + change needing a migration path and a release-note callout. + +## Alternatives Considered + +- **Back up from a single endpoint to avoid the ambiguity** — throws away the availability benefit + that made multiple endpoints the vendor's recommendation. +- **Retain the artifact and mark it incomplete in metadata** — requires retention and restore to + read metadata, and an older tool would still restore it. +- **Bump the metadata version** — makes every prior artifact a migration case for no reader that + needs one. +- **Record the first endpoint as the source** — a guess presented as provenance. diff --git a/dev/adr/0014-separate-external-restore-authorisation.md b/dev/adr/0014-separate-external-restore-authorisation.md new file mode 100644 index 00000000..49707879 --- /dev/null +++ b/dev/adr/0014-separate-external-restore-authorisation.md @@ -0,0 +1,48 @@ +# 14. Restoring into an externally-managed database needs its own authorisation, and the channel that granted it is recorded + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/spec.md (FR-009, FR-010, SC-005), contracts/cli-surface.md + +## Context + +A restore already has an override (`--force`) and a scheduled, unattended mode. Spec 007 adds a +destructive operation of a new kind: overwriting a database the deployment does not manage, on a +server the customer's other systems may also depend on. Neither existing switch was granted with +that in mind, and an unattended restore can reach the new path with nobody present. + +## Decision + +- **A distinct authorisation**: `--allow-external-restore` on the command line, or + `INFRAHUB_ALLOW_EXTERNAL_RESTORE=true` in configuration. `--force` does not imply it, and enabling + scheduled restore does not imply it. Making either imply it is a breaking change to the CLI + contract. +- **The channel is part of the decision.** An unattended restore into an external database is + possible only when the authorisation came through configuration (FR-010). The resolved value + therefore carries its source (`flag` or `config`), not only a boolean. +- **An explicit flag wins over the environment in either direction**, so an operator can refuse on + the command line what a scheduled job's configuration permits. A malformed environment value + authorises nothing, and says so in the log. +- **The flag is deliberately not bound through viper.** Viper reports a flag and its environment + equivalent through one key, and this decision turns on telling them apart; the two channels are + read separately. +- **Refused at the gate, before anything is stopped**, with a message naming the authorisation, its + flag and its configuration key. There is no path from "absent" to a destructive step. + +## Consequences + +- Nobody acquires the capability to overwrite an unmanaged database by enabling something else. +- A scheduled restore against an external database has exactly one configuration key that makes it + possible, and its presence is auditable. +- One flag on the root command is registered without a viper binding, with a comment saying why, + which is an exception to the repository's binding convention and looks like an omission until + read. + +## Alternatives Considered + +- **Reuse `--force`** — it already means "overwrite the deployment's own data"; extending it to a + database outside the deployment grants a capability nobody asked for when they set it. +- **Imply it from the scheduled-restore toggle** — the unattended path is where an absent operator + makes the risk highest, not lowest. +- **A single viper-bound key** — cannot distinguish an interactive grant from a configured one, + which FR-010 requires. diff --git a/dev/adr/0015-verify-by-default-external-tls.md b/dev/adr/0015-verify-by-default-external-tls.md new file mode 100644 index 00000000..b542f824 --- /dev/null +++ b/dev/adr/0015-verify-by-default-external-tls.md @@ -0,0 +1,59 @@ +# 15. Connections to external databases are encrypted and verified by default, with one operator opt-out + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/spec.md (FR-029), contracts/cli-surface.md (T100 change of record), research.md (R4) + +## Context + +The external path opens connections this toolset never opened before: Bolt from the transient pod +to the Neo4j servers for the version probe and the seed statement, and libpq to PostgreSQL for the +dump and the load. Both carry the tool's own database credentials out of the cluster. The +deployment records TLS settings for Infrahub's own connections (`INFRAHUB_DB_TLS_ENABLED`, +`_TLS_INSECURE`, `_TLS_CA_FILE`), but a deployment left at Infrahub's defaults exports none of +them — so "absent" is the common observation, and the first implementation, which encrypted only +when the deployment said `true`, sent the password in clear by default with no flag able to +prevent it. + +## Decision + +- **Encrypted and verified is the default for both databases.** The deployment's settings decide + *how* to verify, never *whether*. +- **`--external-db-insecure-tls` is the only opt-out** (FR-029). With it, the channel is the weakest + one the deployment's own settings describe: encrypted but unverified where the deployment declares + TLS, unencrypted where it declares none. An external Neo4j with no TLS at all is therefore still + reachable, but reaching it takes the operator saying so. +- **A discovered insecure setting is reported and never obeyed.** `INFRAHUB_DB_TLS_INSECURE=true` + is a decision the deployment made about Infrahub's connections, not about this tool's. +- **The deployment's own certificate authority is carried into the workload** that has to verify + against it, read from the running component that records it (`INFRAHUB_DB_TLS_CA_FILE`, or the + Prefect URL's `sslrootcert`) while that component is still up, and placed once per workload: a + keytool import for the JVM clients, `PGSSLROOTCERT` for libpq. A CA that cannot be read or placed + falls back to the image's default trust store — the weakest *verifying* configuration, never a + non-verifying one, so the worst outcome is an "unknown issuer" the operator can act on. +- **The decision is two-valued, so there is no policy-name flag.** `cypher-shell`'s verification is + chosen by the Bolt URI scheme and libpq's by `PGSSLMODE`, and in both the tool's choice is + verify or the operator's opt-out. `--external-db-tls-policy` was registered and removed for that + reason, and stays removed. + +## Consequences + +- A deployment that exports no TLS settings gets an encrypted, verified channel, and a server + presenting a privately signed certificate needs its CA recorded in the deployment or the + operator's opt-out. +- The CA read has an ordering dependency: it happens at the gate, before the deployment's + components are scaled down for an offline capture. Re-reading it later fails and degrades to + the system trust store. +- The TLS decision is resolved once per database per run and passed to every command builder, so + no builder decides it for itself. + +## Alternatives Considered + +- **Follow the deployment's `tls_enabled`** — true that a server not speaking TLS cannot be made to, + but the setting is usually absent, and absent sent the password in clear. +- **A `--external-db-tls-policy` string** — would have had nothing to carry beyond what the boolean + does. +- **Obey a discovered `tls_insecure`** — a setting made about something else, silently disabling + verification for credentials this tool sends. +- **Fail when the CA cannot be placed** — turns a degraded-but-verifying connection into no backup + at all, for a condition the operator can see in the warning. diff --git a/dev/adr/0016-single-ownership-decider.md b/dev/adr/0016-single-ownership-decider.md new file mode 100644 index 00000000..159d687c --- /dev/null +++ b/dev/adr/0016-single-ownership-decider.md @@ -0,0 +1,65 @@ +# 16. One ownership decider: label evidence and declared release prefixes, never a bare name + +**Status**: Accepted +**Date**: 2026-09-07 +**Source**: specs/archive/007-external-database-backup/retrospective.md (R10), spec FR-002/FR-028, the shipped design in `src/internal/app/kubernetes_ownership.go` + +## Context + +Three destructive decisions on the Kubernetes backend rest on "is this resource the deployment's": +where a database lives (ADR 0012), which workload to scale to zero, and whether a service is +running before an offline capture proceeds. Before spec 007 each answered it with its own +substring test — `strings.Contains(name, service)` — and swallowed kubectl errors as empty results. +The retrospective recorded the consequences: `redis-cache` resolved as `cache` and was scaled to +zero, a `postgres-database-0` from an unrelated StatefulSet would have put a genuinely external +database on the in-deployment path, and a transient API error fell through to the name match. +The remediation phases replaced the three rules with one. + +## Decision + +- **`deploymentClaimsResource` is the only function that decides ownership**, and every caller + that acts destructively goes through it. Nothing else may decide from a name; the acceptance + check is that `grep -rn 'nameMatchesService(' src/ | grep -v _test` returns only its definition + and its call sites inside the ownership file. +- **Evidence is read in order.** First, a label *declaration* — the pod or workload says which + service it is, under any of the four service-label keys, including the chart's shortened values + such as `infrahub/service=server`. Second, a name the deployment's *declared release prefixes* + claim: `--`, where the prefixes are read namespace-wide from + every labelled resource, across both workload kinds together. Third, only for a namespace that + declared no prefix at all, a per-service policy: an app service still claims an unanchored + prefixed name (the cost of narrowing is a workload left running through an offline capture); a + database refuses it (the cost of widening is scaling or restoring the wrong database). +- **The right-hand side is read the same way in every tier**: a controller-generated suffix (a + StatefulSet ordinal, a ReplicaSet hash) is what follows the service; a word does not, so + `infrahub-cache-exporter` is never the cache. +- **`nameMatchesService` answers what a name *looks like*, never what it belongs to.** The gap + between the two — a name that names the service and is not claimed — is logged as a near miss. +- **One parser for every pod listing**, carrying name, phase and the four label values, so a label + selector's four queries can be answered out of a single namespace listing and a declaration read + from the same fields. +- **A kubectl failure is collected and reported, never treated as an empty result.** A namespace + that answered every query and matched nothing reports "no workload"; one that failed to answer + reports the failures, and the caller's non-destructive branch depends on telling those apart. + +## Consequences + +- Every ownership decision is now one grep away, and a change to the rule is made once. +- The running check costs one kubectl call per service instead of four selector queries plus a + fallback, because the selectors are reproduced from the listing's own label fields. +- Two shapes remain deliberately distinct: ownership claims a pod carrying a release prefix, and a + selector reproduction does not, so `IsRunning` cannot answer true from evidence no selector + produced. +- The generated-suffix recogniser is a shape heuristic with an acknowledged miss (a hash with no + digit is declined). Reading `pod-template-hash` and the StatefulSet pod-name label from the + listing would make it exact; that is recorded as follow-up, not done. + +## Alternatives Considered + +- **`strings.Contains(name, service)`** — the shipped behaviour that scaled another product's + workload to zero. +- **A rule per caller** — the three callers had drifted on the unanchored case without any of them + recording it as a choice. +- **Label evidence only** — the chart does not label every resource, and a deployment with no + labels at all still has to be backed up; the prefix tier is what serves it. +- **Fall back to a bare name listing when the labelled listing fails** — the one remaining path + where a kubectl failure could end in a foreign workload being scaled. diff --git a/dev/adr/0017-database-tools-in-the-infrahub-backup-image.md b/dev/adr/0017-database-tools-in-the-infrahub-backup-image.md new file mode 100644 index 00000000..9ffba8dc --- /dev/null +++ b/dev/adr/0017-database-tools-in-the-infrahub-backup-image.md @@ -0,0 +1,71 @@ +# 17. The database tools ship in the infrahub-backup image + +**Status**: Accepted +**Date**: 2026-10-12 +**Supersedes**: the image choice in ADR 0008 (the rest of ADR 0008 stands) +**Source**: review of [PR #172](https://github.com/opsmill/infrahub-backup/pull/172) on ADR 0008 ("ship `neo4j-admin` and `pg_dump` `pg_restore` within our infrahub-backup docker image instead, would be easier"), decided by the PR author + +## Context + +ADR 0008 settled that a database outside the deployment is reached from a temporary workload in +the deployment's namespace, and that the workload runs the vendors' own images +(`neo4j:-enterprise` for `neo4j-admin` and `cypher-shell`, `postgres:18-alpine` for +`pg_dump`, `pg_restore` and `psql`). Both images are pinned in the binary. + +That choice puts two third-party images on the external path. An air-gapped operator has to +mirror both, alongside the `infrahub-backup` image they may already mirror to run the tool in the +cluster. The tool also cannot run the database tools itself when it already runs as a pod in the +namespace, so even that case creates a second pod. + +## Decision + +- **The `infrahub-backup` image carries the database tools**: `neo4j-admin` and `cypher-shell` + at the Neo4j version the binary previously pinned for the vendor image, plus `pg_dump`, + `pg_restore` and `psql` for PostgreSQL 18. The tool versions are pinned in the `Dockerfile`, so + one image tag has one known tool set. +- **The temporary workload runs the `infrahub-backup` image** of the same version as the running + binary, instead of a vendor image. One flag points it at a mirror, in place of the two + per-vendor image flags. ADR 0008's reason for not reading that flag from the environment + (FR-027) still applies. +- **When `infrahub-backup` itself runs as a pod in the target namespace and finds the tools in + its own image, it runs them directly** and creates no temporary workload. Running from an + operator's machine still uses the temporary workload. + +ADR 0008's other decisions stand. A failed pull on the external path is still a failed backup, +and deployments whose databases are all internal still pull nothing. + +## Consequences + +- An air-gapped operator mirrors one image, `infrahub-backup`, instead of three. +- Running the tool as a pod in the namespace needs no permission to create workloads for the + external path, because nothing is created. +- The image grows by a Java runtime and the Neo4j tool libraries, from 34 MiB to 156 MiB + compressed (arm64, measured 2026-10-13). The growth applies to every user of the image, + including users whose databases are all internal. To keep it at that size, the Java runtime is + built with `jlink` from the modules the tools use, and the Neo4j jars that the tools never load + are removed, including the S3, Azure and Google Cloud client libraries, because the tool only + ever writes a backup to a local directory. The image's `neo4j-admin` therefore cannot write to + cloud storage. A Neo4j upgrade therefore needs that jar list checked again with a real backup, + restore and `cypher-shell` run; `scripts/check-image-tools.sh` only checks that the tools start. +- OpsMill now redistributes Neo4j Enterprise tool binaries in this image. The PR author confirmed + that this is cleared, as OpsMill already redistributes Neo4j Enterprise with + `infrahub-enterprise`. +- A Neo4j server upgrade that needs a newer `neo4j-admin` now needs an `infrahub-backup` release, + where before it needed only a new pinned vendor tag. The version gate (FR-006) still refuses a + tool older than the server. +- A development build has no published image of its own version, so it has to be given the image + explicitly. The end-to-end tests have to build the image and load it into the test cluster. +- Two code paths now run the tools: in a temporary workload, and in the tool's own process. Both + must keep FR-014: credentials stay out of every command line. + +## Alternatives Considered + +- **Keep the vendor images (ADR 0008 as written)**: rejected in review. It needs no image + changes and follows the vendors' own releases, but leaves three images to mirror and a second + pod even when the tool already runs in the cluster. +- **A separate tooling image published next to `infrahub-backup`**: keeps the main image small, + but leaves two images to mirror and two release artefacts to version together. Not chosen + because the licensing question was cleared and one image is the simpler contract. +- **Run the tools only when `infrahub-backup` runs in the cluster, and refuse external databases + from an operator's machine**: removes the temporary workload entirely, but drops a way of + running the tool that ADR 0008 supports. Rejected. diff --git a/dev/adr/0018-transient-workload-is-a-job.md b/dev/adr/0018-transient-workload-is-a-job.md new file mode 100644 index 00000000..716ebff7 --- /dev/null +++ b/dev/adr/0018-transient-workload-is-a-job.md @@ -0,0 +1,56 @@ +# 18. The transient workload is a Job that owns its pod and its credential Secret + +**Status**: Accepted +**Date**: 2026-10-12 +**Supersedes**: the choice of a bare `Pod` in ADR 0009, and the owner of the credential Secret (the rest of ADR 0009 stands) +**Source**: `src/internal/app/transient_workload.go` (`transientFinishedTTL`, `transientPodSpec`, `buildTransientWorkloadManifest`); spec FR-011/FR-023; [PR #172](https://github.com/opsmill/infrahub-backup/pull/172) + +## Context + +ADR 0009 decided that the temporary workload for an external database is a bare `Pod`, piped to +`kubectl create -f -`, bounded by `activeDeadlineSeconds`, and owning its credential Secret +through an `ownerReference`. It rejected a `Job` because a Job adds a controller and a second +object to remove. + +FR-011 requires the workload to be gone on every exit path, including when the tool is killed. +FR-023 requires the credential to be unable to outlive the run. A pod that reaches +`activeDeadlineSeconds` is marked `Failed`, but the cluster does not delete a bare pod object, so +the pod and the Secret it owns stay in the namespace until something deletes them. When the tool +is killed, nothing does until a later run's stray reaper reaches them. + +The code moved to a `Job` during the implementation of spec 007. The change was not recorded as a +decision at the time. + +## Decision + +- **The workload is a `Job`** with `backoffLimit: 0`, an `activeDeadlineSeconds` computed as in + ADR 0009, and `ttlSecondsAfterFinished` (`transientFinishedTTL`, 60 seconds). The Job creates + one pod with `restartPolicy: Never`. +- **The Job owns the credential Secret**, when the run creates one. The Secret is created after + the Job, with an `ownerReference` to the Job's UID. When the run references the deployment's own + Secret keys instead, no Secret is created. +- **The pod restates the Job's deadline** in its own `activeDeadlineSeconds`. The Job controller + does not copy the Job's deadline onto its pod, and the stray reaper lists pods, not Jobs. The + kubelet also enforces the pod's deadline when the Job controller does not. + +## Consequences + +- When the tool is killed, the cluster removes everything the run created without help, as long + as its Job, time-to-live and garbage-collection controllers are running: the deadline ends the + Job, the time-to-live deletes it, and the garbage collector deletes its pod and its credential + Secret. Without the Job controller, the kubelet still ends the pod at its own deadline, but the + Job and its Secret stay until that controller runs again. +- The namespace must grant `create`, `get` and `delete` on jobs, in addition to the pod + permissions. ADR 0009 lists the full set. +- Two deadlines must agree: the Job's and the pod's. Both come from + `externalDBWorkloadDeadline`, so they cannot drift apart. +- The tool still execs into a specific pod, so it has to resolve the Job's pod after creating the + Job. This is the controller and the extra object ADR 0009 wanted to avoid. + +## Alternatives Considered + +- **A bare `Pod` (ADR 0009 as written)**: one object fewer and no pod lookup, but a pod that + reaches its deadline is not deleted, so its Secret outlives a killed run until a later run + removes it. That fails FR-023 in exactly the case it exists for. +- **A bare `Pod` that the stray reaper always cleans up**: depends on a later run happening, + which is the dependency FR-011 rules out. diff --git a/dev/guidelines/implementation-verification.md b/dev/guidelines/implementation-verification.md new file mode 100644 index 00000000..27de2aa1 --- /dev/null +++ b/dev/guidelines/implementation-verification.md @@ -0,0 +1,53 @@ + + +# Implementation Verification Guidelines + +## Overview + +Two checks that a chunk of implementation must pass before it is reported done, whoever or +whatever wrote it. Both come from the spec 007 retrospective: eight implementation chunks each +reported full success with accurate, honest unit-test evidence, and a later review found five +requirements that were completely inert — about 470 lines with no non-test caller — plus a pod the +kubelet rejected unconditionally, built on a premise about the official images that ten seconds of +`docker image inspect` would have falsified. + +Unit tests written alongside the code prove that it compiles and that the author's assertions are +self-consistent. They cannot tell working code from unreachable code, correct logic wired to the +wrong thing, or a guarantee that holds per chunk and fails in composition. + +## Grep for callers of every new symbol, per chunk + +Before a chunk is reported complete, every newly added exported or package-level function, type +and variable must have at least one caller outside `_test.go` files, or a written reason why not. + +```sh +grep -rn 'newSymbol(' src/ | grep -v _test +``` + +Zero hits means the requirement the symbol implements is not wired in, whatever the tests say. Do +not defer this to a final review: the cost of finding it per chunk is one grep, the cost of finding +it at review was a partial reset. + +## Verify premises about external artifacts with a named command + +Any factual claim about something outside this repository — an image's user, a chart's labels, a +server's exit codes, a metric's presence on a version — is verified by running a command and +recording it, not asserted from memory or documentation. The retrospective's example: + +```console +$ docker image inspect postgres:18-alpine --format '{{.Config.User}} {{.Config.Entrypoint}}' + [docker-entrypoint.sh] +``` + +The empty first field is the finding: the image declares no user. + +Record the command and its output where the premise is used (a code comment, the chunk report, +or a `dev/knowledge/` page when the fact is durable, as +[container-image-user-semantics.md](../knowledge/container-image-user-semantics.md) does). + +## What a passing test suite does prove + +It proves the change compiles, that its own assertions hold, and that it did not break an +assertion someone else wrote. It is necessary. It is not evidence of reachability, of composition +across chunks, or of any fact about the world outside the repository — which is what the two +checks above are for. diff --git a/dev/guidelines/kubernetes-backend.md b/dev/guidelines/kubernetes-backend.md new file mode 100644 index 00000000..727e1e0d --- /dev/null +++ b/dev/guidelines/kubernetes-backend.md @@ -0,0 +1,107 @@ + + +# Kubernetes Backend Guidelines + +## Overview + +Conventions for code that talks to a Kubernetes deployment through the backend in +`src/internal/app/environment_kubernetes.go`, `kubernetes_*.go` and the external-database path +(`external_*.go`, `transient_workload.go`). Every one of these rules exists because its absence +shipped a defect; the background is in ADRs 0008 to 0016 and the retrospective archived with +spec 007. + +## Read kubectl values from stdout alone + +Anything parsed as data — a pod name, a phase, a UID, a replica count, a JSONPath listing — goes +through a `podRunner` that returns the command's stdout and folds stderr into the error +(`stdoutRunner` unbounded, `boundedRunner`/`separatedRunner` under a timeout). Never pass the +merged `executor.runCommand` as a runner: kubectl writes deprecation and kubeconfig notices to +stderr on healthy calls, and merged output puts the notice in front of the value. + +```go +// yes +return k.getPodStatusesWith(k.stdoutRunner(), service) +// no — a warning becomes part of the UID, the ownerReference matches nothing, the Secret leaks +return k.getPodStatusesWith(k.executor.runCommand, service) +``` + +`runCommand` stays correct for commands whose output is not parsed: `scale`, `cp`, `exec` of a +utility whose output is only logged. + +## Decide ownership only in `kubernetes_ownership.go` + +Whether a pod or workload is the deployment's is answered by `deploymentClaimsResource`, and +nowhere else. Do not compare a name against a service with `strings.Contains`, `HasPrefix` or a +regular expression in a caller. `nameMatchesService` says what a name looks like and must not be +used to *decide*; the acceptance check is + +```sh +grep -rn 'nameMatchesService(' src/ | grep -v _test # definition + ownership-file call sites only +``` + +Listings are parsed by `parseLabelledPods` only. If a new listing needs another field, add it to +`podOwnershipJSONPath` and `labelledPod`; do not write a second parser. + +## A failed cluster query is never evidence of absence + +Collect kubectl errors and return them. An empty result on the strength of queries that failed +must not become "not deployed", "external", or "nothing to scale" (FR-002). The pattern the +resolvers use: + +```go +if len(selectors) > 0 && len(selectorErrs) == len(selectors) { + return nil, fmt.Errorf("…: every label selector query failed: %w", errors.Join(selectorErrs...)) +} +return nil, fmt.Errorf("…: %w", errNoPodsMatched) +``` + +The two returns are different facts with opposite remedies; keep them distinct in wording and in +sentinel. + +## Bound every operation against an external database + +Nothing on the external path calls the unbounded `Exec`. Use `execBoundedAgainst(bound, +operation, target, service, command, opts)` with the right bound — `externalDBBound` for the +operation itself, `externalDBProbeBound` for probe statements, `externalDBControlBound` for +kubectl reads and housekeeping — so a timeout names what it was waiting on. + +A transient pod's `activeDeadlineSeconds` comes from `externalDBWorkloadDeadline(calls, bound)`: +count every bounded call the pod hosts and multiply by the bound each runs under. Budgeting for one +call where the pod hosts two put the deadline inside the window a capture was still working in, +and the cluster deleted a completed capture during the copy out. + +## Credentials never touch the exec command line + +`ExecOptions.Env` becomes an `env KEY=VALUE` prefix on the exec command, visible in API-server +audit logs and the pod's process list. It carries TLS mode and trust paths, never a password. Client +tools read credentials from the Secret the transient pod mounts through `envFrom`. + +## Read cypher-shell results by header name + +Use `cypherResult`/`parseCypherRow` and read columns by the name the statement asked for. Never +take the first non-empty line as the header or read a column by position: cypher-shell prints +notices above the result, and a server may return columns in another order. Column names are +matched case-insensitively and trimmed in one place (`cypherColumnKey`). + +## Images are flag-only; placement is environment-bound + +A flag that chooses *what runs* in the customer's namespace (`--external-db-image-*`) is never +bound to viper, because the tool holds workload-creation rights and an environment-settable image +turns those into arbitrary code execution. Flags that say *where* a pod may run and *which +credentials* fetch its image are bound. Register a new external-database string flag in the +`externalDBStrings` table in `cli.go`, which registers, binds and reads it in one place. + +## Metadata additions are additive + +A new metadata field needs a documented meaning for absence, must not change the metadata +version, and must be safe for an older tool to ignore. Retention never reads metadata; do not make +it. An artifact that must not be restored is removed, not marked. + +## Testing + +- Every resolver takes a `podRunner` (`…With(run, …)`); test through it with a fake runner rather + than a stubbed binary where possible. +- A fake listing must round-trip through the real parser (`ownershipLine`), so a filter that reads + a label field can be tested at all. +- A new exported or package-level symbol needs a non-test caller in the same change; see + [implementation-verification.md](implementation-verification.md). diff --git a/dev/knowledge/container-image-user-semantics.md b/dev/knowledge/container-image-user-semantics.md new file mode 100644 index 00000000..b2338daf --- /dev/null +++ b/dev/knowledge/container-image-user-semantics.md @@ -0,0 +1,83 @@ + + +# Container image user semantics + +## Overview + +Several planned features run vendor database tooling inside a container this tool creates, rather +than inside a container the deployment already owns. Getting the pod's security context right +depends on a fact about the official database images that is easy to assume and wrong: **they do +not declare a non-root user.** + +This page records the measured behaviour, because a spec-007 implementation chunk asserted the +opposite in a code comment, shipped a pod the kubelet rejects unconditionally, and its unit tests +passed — they asserted manifest *shape*, which no unit test can connect to an admission decision. + +## The measurement + +```console +$ docker image inspect postgres:18-alpine --format '{{.Config.User}} {{.Config.Entrypoint}}' + [docker-entrypoint.sh] + +$ docker image inspect neo4j:2025.10.1-enterprise --format '{{.Config.User}} {{.Config.Entrypoint}}' + [tini -g -- /startup/docker-entrypoint.sh] +``` + +`Config.User` is **empty** for both. An empty user means uid 0. Both images run as root and drop +privileges *inside their own entrypoint* — `postgres` via `gosu`, `neo4j` via +`/startup/docker-entrypoint.sh`. + +The uids often quoted for these images (70 for the alpine `postgres` image, 7474 for `neo4j`) are +real users that exist inside the filesystem. They are not declared in the image config, so +Kubernetes does not see them. + +## Why it matters + +Two consequences, and the second is the one that bites: + +1. `runAsNonRoot: true` with **no** `runAsUser` makes the kubelet resolve the effective user from + the image config, find root, and refuse the container with + `CreateContainerConfigError: container has runAsNonRoot and image will run as root`. There is no + partial success and no fallback — every pod using that combination fails to start. +2. Setting `Command` (or `Args` alone in some shapes) **replaces the image ENTRYPOINT**, so the + privilege drop those images rely on never runs. A pod that keeps the entrypoint and one that + overrides it have different effective users for the same image. + +A manifest that overrides the entrypoint *and* asserts `runAsNonRoot` without `runAsUser` combines +both, which is how spec 007 produced a workload that could never start. + +## An image that declares its user by name + +Since ADR 0017 the transient workload runs the `infrahub-backup` image instead of the vendor +images. That image does declare a non-root user, but by name: + +```console +$ docker image inspect infrahub-backup: --format '{{.Config.User}}' +infrahub + +$ docker run --rm --entrypoint id infrahub-backup: +uid=1000(infrahub) gid=1000(infrahub) groups=1000(infrahub) +``` + +The kubelet checks `runAsNonRoot` against a numeric uid only. With a named user and no +`runAsUser`, it refuses the container with `CreateContainerConfigError: container has runAsNonRoot +and image has non-numeric user (infrahub), cannot verify user is non-root`. So `runAsUser` stays +explicit, set to 1000 with `runAsGroup` and `fsGroup` (FR-033, `databaseImageUser`). The value is +the same for both databases, because one image carries both sets of tools. + +## What to do instead + +- If the pod must satisfy a restricted pod-security policy, set `runAsUser` **explicitly**. Do not + rely on the image to declare it. +- Decide what happens for an **operator-supplied** image, whose user may differ from the pinned + default's. Either require the operator to supply the uid alongside the image, or verify the + image's config before building the manifest. Silently reusing the default's uid produces a pod + that starts and then cannot read its own volume. +- Check the premise rather than inheriting it. `docker image inspect ... --format + '{{.Config.User}}'` takes seconds and is the only authority. + +## Verification + +This is a claim about third-party images, so it expires. Re-measure when a pinned image tag +changes, and prefer an end-to-end case that actually admits the pod on a namespace enforcing +restricted pod security over any unit assertion about manifest fields. diff --git a/dev/knowledge/external-database-backup.md b/dev/knowledge/external-database-backup.md new file mode 100644 index 00000000..435a7732 --- /dev/null +++ b/dev/knowledge/external-database-backup.md @@ -0,0 +1,212 @@ + + +# External Database Backup and Restore + +## Overview + +`infrahub-backup` can back up and restore an Infrahub deployment on Kubernetes whose Neo4j +Enterprise database, PostgreSQL task-manager database, or both run **outside** the deployment's +namespace. Every backup and restore path in this toolset runs the vendor's utility inside a +container and streams bytes out through the exec, stream and copy primitives; an external +database has no such container, so the tool creates one — a short-lived *transient workload* in +the namespace — and registers it under the existing service name. The layers above cannot tell +the difference, and a deployment whose databases are all internal is untouched. + +Code: `src/internal/app/external_*.go`, `transient_workload.go`, `kubernetes_ownership.go`, +`kubernetes_pod_match.go`, `kubernetes_workloads.go`. Operator documentation: +`docs/docs/backup/external-databases.mdx`. Decisions: ADRs 0008 to 0016. + +Scope of v1: Kubernetes only. Neo4j **Enterprise** only for the external path — external +Community is refused, because its backup mechanism needs the database stopped and its storage +directly accessible. + +## How a run decides where a database lives + +Location is settled per database at a *gate* (`resolveDatabaseTargets`) before anything is +stopped, and it is the only way a run obtains a `databaseTarget`. Holding one is the evidence the +question was answered; the capture and restore preparers refuse a target that did not come through +the gate. + +| Evidence | Location | +|---|---| +| A pod or workload the deployment *claims* (label declaration, or a name the declared release prefixes claim — ADR 0016) | internal | +| The namespace listing succeeded and nothing claimed the service | external | +| The listing failed (permission, unreachable control plane) | **error** — never "external" (ADR 0012) | + +The decision is memoised for the run (`serviceLocations`); a restore's scale-down changes which +pods exist, not where a database lives. + +## Endpoint discovery + +The endpoint is read from the environment of the components still running inside the deployment. +Reading these is additive; a deployment that does not set them behaves as before. + +| Variable | Used for | +|---|---| +| `INFRAHUB_DB_ADDRESS` | Neo4j host list, comma-separated `host[:port]`, already the shape `--from` wants | +| `INFRAHUB_DB_PORT` | Bolt client port (default 7687) — **not** the backup port | +| `INFRAHUB_DB_PROTOCOL` | `bolt` or `neo4j` (routed form for multi-member deployments) | +| `INFRAHUB_DB_DATABASE`, `_USERNAME`, `_PASSWORD` | Already read before this feature | +| `INFRAHUB_DB_TLS_ENABLED`, `_TLS_INSECURE`, `_TLS_CA_FILE` | Client-connection TLS; `_TLS_INSECURE` is reported, never obeyed | +| Prefect connection URL (task-manager) | PostgreSQL host, port, database, user, password, `sslrootcert` | + +Two things discovery cannot supply: the Neo4j **backup** port (6362 by default, Infrahub never +records it) and any value the deployment does not export. The PostgreSQL URL is parsed for what +it *states* — an absent host stays absent, because `pgx` would otherwise fill it from the +operator's own `PGHOST`. The resolved endpoint is logged before any connection is opened. + +## The transient workload + +One pod per role per external database, created from a full manifest piped to `kubectl create` +on stdin. The credentials reach it in one of two ways (`external_credential_refs.go`): + +- **Reference the deployment's Secret.** When the username and password the run holds were both + read from the deployment component's environment, and that component's running pod declares the + variables that held them as Secret keys (`env[].valueFrom.secretKeyRef`), the Job maps those keys onto `NEO4J_USERNAME`/`NEO4J_PASSWORD` or + `PGUSER`/`PGPASSWORD` with `secretKeyRef` (never optional). No Secret is created. The pod spec is + read with a jsonpath that returns references only, never inline values. The decision is + remembered once per database after a pod spec was read, while the deployment is still up; when + the read fails, that workload carries an owned Secret and a later workload reads again. For PostgreSQL the URL is a single value, so + reuse needs variables beside it that held the same user and password; prefect-server renders + `PREFECT_SERVER_DATABASE_USER`/`_PASSWORD` that way when `externalDatabase.usernameSecretKey` and + `passwordSecretKey` are set. The default chart puts only the URL in a Secret, so it falls back. +- **Owned Secret (fallback).** Operator-supplied credentials, inline values, ConfigMaps, variables + reached only through `envFrom` (which names no keys, and whose Secret may lack the key while the + image sets the variable) or an unreadable pod: a credential Secret is created with an `ownerReference` + to the Job (ADR 0009), exactly as before. + +| Role | When | Scratch | Hosts | +|---|---|---|---| +| probe | at the gate, before anything is stopped | 1 GiB fixed | version, edition, store size, member roles | +| capture | around the capture itself | 2× queried store size, or `--external-db-scratch-size` | `neo4j-admin database backup --from=…` or `pg_dump`, then the copy out | +| restore | at the gate, held for the run | 10 GiB default, overridable | the seed statement and status poll, or the dump copy-in and `pg_restore` | + +Identity and lifecycle: + +- name prefix `infrahub-backup-xdb`, label `infrahub-backup/transient=true`, a run-ID label; + `restartPolicy: Never`; `activeDeadlineSeconds` = readiness timeout + calls hosted × bound + + margin (`externalDBWorkloadDeadline`). +- one image for both databases: `registry.opsmill.io/opsmill/infrahub-backup:`, the + binary's own release (tag `v2.3.0` → image tag `2.3.0`), which carries `neo4j-admin`, + `cypher-shell` and the PostgreSQL 18 client (ADR 0017). Overridable by `--external-db-image` + only; the pull is load-bearing (ADR 0008). A development build (empty version, commit hash, + `git describe` output, build metadata — see `publishedImageTag`) has no default and is refused in + `externalCaptureOps`, before the reaper and before any service is stopped (FR-031). +- restricted pod-security context; the image declares its user by name (`infrahub`), which the + kubelet cannot check `runAsNonRoot` against, so `runAsUser`, `runAsGroup` and `fsGroup` are set + to its uid 1000 for every role and database (FR-033) — see + [container-image-user-semantics.md](container-image-user-semantics.md). The vendor images this + replaced needed 7474 and 70. +- a stray reaper deletes marker-labelled pods and Secrets from earlier runs that are terminated or + past their deadline, before a new workload is created. A forbidden secrets listing is a warning, + not a failure: owned Secrets go with their Job through the garbage collector. +- excluded from every pod listing by the marker label first and the name prefix second, so it never + appears in service enumeration, replica counts or `infrahub-collect` output. + +## CLI surface + +Flags follow the repository convention (root command, viper-bound, `INFRAHUB_` environment +equivalent with `-` → `_`) except where marked. + +| Flag | Default | Env-bound | Purpose | +|---|---|---|---| +| `--neo4j-address` | discovered | yes | Override the Neo4j host list | +| `--neo4j-backup-port` | 6362 | yes | Backup listener port (not discoverable) | +| `--postgres-address` | discovered | yes | Override the PostgreSQL endpoint | +| `--external-db-scratch-size` | 2× store size | yes | Scratch for the capture workload | +| `--external-db-timeout` | 2h | yes | Bound on any single external operation | +| `--external-db-insecure-tls` | false | yes | The only TLS opt-out (ADR 0015) | +| `--allow-external-restore` | false | **separate channel** | Authorisation for a destructive external restore; the env key `INFRAHUB_ALLOW_EXTERNAL_RESTORE` is read on its own so flag and config are distinguishable (ADR 0014) | +| `--external-db-image` | `infrahub-backup:`; none on a development build | **no** | The workload's image; flag-only because the tool holds workload-creation rights | +| `--external-db-image-pull-secrets`, `--external-db-service-account`, `--external-db-node-selector`, `--external-db-tolerations`, `--external-db-priority-class` | none | yes | What the operator's cluster requires of a pod it did not author | + +## Prerequisites + +Customer side (documented, verified by the end-to-end suite, not assumable by the tool): + +1. The Neo4j backup service listens on a non-loopback address (`server.backup.listen_address`); it + is bound to localhost by default, so this is a server configuration change, not a firewall rule. +2. Each Neo4j server can fetch from the object store: a route, plus the cloud provider's CLI and + credentials configured server-side for `CloudSeedProvider`. +3. A privately signed server certificate has its CA recorded in the deployment (`_TLS_CA_FILE` or + `sslrootcert`), or the operator opts out of verification. +4. In an air-gapped cluster where the tool creates transient Jobs, the image they run is mirrored + and its pull secret named: by default the `infrahub-backup` image at the binary's version, or + the image given with `--external-db-image`. It is the only image the tool pulls. + +Cluster side, in the Infrahub namespace: `create`, `get`, `delete` on jobs; `get`, `list`, +`watch`, `delete` on pods; `create` on `pods/exec`. `watch` is needed by `kubectl wait`. `create`, +`list`, `delete` on secrets only when the run falls back to an owned credential Secret (see above). These are the first +write verbs the tools require and only the external path needs them. `infrahub-collect` needs none +of this and refuses `--include-backup` for an external database (ADR 0003). + +## Backup + +1. Gate: location per database, edition and version probe (abort when the utility is older than + the server, warn when newer — ADR 0011), store size for scratch sizing, member roles. +2. Capture: `neo4j-admin database backup --from=,…` with the full endpoint list in try + order, or `pg_dump` from the member that answered the probe. Artifact layout is byte-compatible + with an internal capture, so either restores by the same code. +3. Completeness is read from the command's output, not its exit status (exit `1` means "failed" or + "succeeded with uncontactable servers"). An incomplete capture fails the run and is removed from + every location it reached (ADR 0013). + +## Restore + +1. Gate: `--allow-external-restore` (or its env key for unattended runs), location, and a restore + workload created and probed before anything is stopped. +2. Neo4j: the raw `-.backup` is staged at + `s3:////seed/…` in the operator's configured object store, verified readable by + an independent client, then `CREATE OR REPLACE DATABASE … seedURI` is issued over Bolt. The tool + polls `SHOW DATABASES` every 5 s within the operation bound and reports success only when the + database is online; a status that cannot be read is attributed to the path to the database, not + the database. The staged object is removed only then (ADR 0010). +3. PostgreSQL: the dump is copied into the restore workload and loaded with the client tooling. +4. Infrahub workloads are quiesced first, confirmed stopped (not merely asked), and returned to + their prior scale on every exit path. Both databases restore or neither does. + +## Metadata + +Additive fields, metadata version unchanged; absence keeps its meaning (ADR 0013). + +| Field | Absent means | +|---|---| +| `source_endpoints` (per database, try order) | a pre-feature artifact | +| `source_roles` (endpoint → leader/follower/unknown) | roles not observed; `unknown` is a real value | +| `capture_complete` | complete — should never be read as `false`, that artifact was removed | +| `external_location` (per database) | internal | + +No field claims which endpoint served the capture; with several supplied that is not knowable. + +## Bounds + +| Bound | Value | Applies to | +|---|---|---| +| operation bound (`--external-db-timeout`) | 2 h default | capture, copy out, seed, restore, status poll window | +| probe bound | 5 min | each probe statement | +| control bound | 2 min | `kubectl` reads, scratch cleanup, CA placement | +| workload readiness | 10 min | `kubectl wait` after create (covers the image pull) | +| deadline margin | 10 min | added to every pod's `activeDeadlineSeconds` | +| quiesce confirmation | 5 min | Infrahub workloads reporting stopped | +| return to scale | 10 min | Infrahub workloads reporting running again | + +## Failure-message contract + +Each message names the condition, the resource and the operator action. The distinctions that +matter most: a deployment query that *failed* is never reported as an absent database; settings +that could not be *read* are reported differently from settings that record no address; a +restored database that did not come online is reported differently from one whose status cannot be +read at all. The full table is in `specs/archive/007-external-database-backup/contracts/cli-surface.md`. + +## Gotchas + +- The CA is read from a running component at the gate. A path that needs it after an offline + capture has scaled that component away gets the memoised copy; a fresh read there would fail and + degrade to the system trust store. +- An explicit `--k8s-namespace` that fails a soft `kubectl` check can still fall back to Docker + detection on a host running a live compose project; pin the namespace and check + `environment detect` first (residual finding T156). +- The store-size metric the probe queries is absent on Neo4j 2025.x, so those captures need + `--external-db-scratch-size` (residual finding T158). +- Every kubectl value the backup path parses is read from stdout alone; a kubectl notice on stderr + used to be prepended to a Secret's owner UID and leave the credential behind. diff --git a/dev/knowledge/infrahub-collect.md b/dev/knowledge/infrahub-collect.md index 85ee856d..68e1133e 100644 --- a/dev/knowledge/infrahub-collect.md +++ b/dev/knowledge/infrahub-collect.md @@ -105,3 +105,5 @@ Every subprocess on the collect path is timeout-bounded (see [ADR 0002](../adr/0 - **Docker services log to stderr.** `docker logs` demuxes a container's output onto the matching process streams, and Infrahub services write to stderr. Capturing stdout alone yields near-empty logs, so the Docker log path uses a **combined** stdout+stderr pipe (`runCommandCombinedPipeContext`). The Kubernetes `kubectl logs` path needs no such merge. - **A stopped-but-deployed Docker service** still has its logs collected and is recorded `failed` (not `skipped`); `skipped` is reserved for a service that is not deployed at all. + +- **`--include-backup` refuses a database that lives outside the deployment.** Capturing an external Neo4j or PostgreSQL needs a transient pod and a credential Secret created in the namespace, and collection creates no workload ([ADR 0003](../adr/0003-read-only-collection.md), [ADR 0008](../adr/0008-transient-workload-for-external-databases.md)). The refusal names `infrahub-backup create` as the tool that performs the capture; see [external-database-backup.md](external-database-backup.md). diff --git a/docs/docs/backup/external-database-direct-to-cloud.mdx b/docs/docs/backup/external-database-direct-to-cloud.mdx new file mode 100644 index 00000000..563ce6ed --- /dev/null +++ b/docs/docs/backup/external-database-direct-to-cloud.mdx @@ -0,0 +1,139 @@ +--- +title: Capture an external Neo4j database directly to cloud storage +--- + +If your Neo4j servers cannot fetch a staged artifact from an object store, `infrahub-backup +restore` has no way to seed them, and the supported round trip does not close. Neo4j Enterprise can +instead write a backup straight to cloud storage itself, which gives you an artifact in your own +bucket without any of the tool's machinery. + +This is a manual procedure. `infrahub-backup` does not perform it, does not schedule it, and does +not know the artifact exists. Use it when you need a backup in your bucket and cannot meet the +restore prerequisites in +[Prepare an external database for backup and restore](./external-databases.mdx); use +`infrahub-backup create` whenever you can. + +## What you take on + +Everything `infrahub-backup` normally does for you becomes yours to do: + +| Concern | With `infrahub-backup create` | Direct to cloud storage | +|---------|-------------------------------|-------------------------| +| Artifact format | Single archive with metadata and checksums | Vendor artifacts, one per database, unrelated to each other | +| Restore | `infrahub-backup restore` finds and validates the archive | You drive the vendor restore yourself | +| Both databases | Neo4j and PostgreSQL captured in one coordinated run | Two independent runs you sequence | +| Storage efficiency and encryption | The `plakar` backend removes repeated blocks and encrypts them | Whatever your bucket provides | +| Retention | `--retention-days` and `--retention-count` prune archives | Bucket lifecycle rules, written by you | +| Completeness | Partial captures are detected and removed | You read the command output and judge | + +Two of those deserve their own warning. + +:::warning +The artifacts this procedure produces are **not** restorable with `infrahub-backup restore`. It +looks for an Infrahub backup archive with its metadata, and finds neither. Plan the restore before +you rely on the backup. +::: + +:::warning +Your two databases are captured by two separate commands, so their recovery points differ by +however long you leave between them. A restore that mixes recovery points leaves Infrahub's graph +and its task history disagreeing. Quiesce Infrahub around both captures, or accept and record the +gap. +::: + +## Prerequisites + +- Neo4j Enterprise 2025.03 or later on the server. Writing a backup directly to cloud storage was + added in that release. +- A backup client machine on the same network as the database, with a matching Neo4j installation + so `neo4j-admin` is available, and with local disk at least the size of the database. Writing to + cloud storage still stages through local disk. The `neo4j-admin` in the `infrahub-backup` + container image cannot write to cloud storage, because the image leaves out the S3, Azure and + Google Cloud client libraries. Use a full Neo4j installation for this procedure. +- The Neo4j backup service listening on a non-loopback address. This prerequisite is the same one + the supported path needs — see + [Step 1 of the preparation guide](./external-databases.mdx#step-1-expose-the-neo4j-backup-service-on-a-non-loopback-address). +- Cloud storage credentials on the backup client, and a PostgreSQL client at least as new as your + task manager database server. + +## Step 1: Capture Neo4j to your bucket + +From the backup client, run the vendor backup command with a cloud destination. Pass every cluster +member so the capture can fall back when one is unreachable: + +```bash +neo4j-admin database backup \ + --from=neo4j-a.example.internal:6362,neo4j-b.example.internal:6362 \ + --to-path=s3://my-infrahub-backups/manual \ + neo4j +``` + +Check the vendor documentation for your release before you script this. The flags and the +supported destination schemes belong to Neo4j, not to Infrahub, and they change between releases. + +## Step 2: Capture the task manager database + +Infrahub's task history lives in PostgreSQL and is not covered by the command above. Take a +custom-format dump, which restores into the same or a later major version: + +```bash +pg_dump --format=custom --no-owner \ + --host task-manager-db.example.internal \ + --username prefect \ + --dbname prefect \ + --file prefect-$(date -u +%Y%m%dT%H%M%SZ).dump +``` + +Upload the dump beside the Neo4j artifact, using whichever client your storage provider supports. + +## Step 3: Verify what you captured + +The backup command's exit status is not enough. Exit code `1` means either that the backup failed +or that it succeeded while some servers were unreachable, and there is no exit code that +separates the two. Passing several members makes the second case more likely, not less. + +Read the command's own output for servers it could not reach, then confirm the objects landed: + +```bash +aws s3 ls s3://my-infrahub-backups/manual/ --recursive --human-readable +``` + +Record the timestamp of each artifact and which endpoint served it. Nothing else does this for +you, and a recovery point you cannot state is not a recovery point. + +## Step 4: Plan and rehearse the restore + +Decide the restore route now, while the databases are healthy, and rehearse it against a +disposable database: + +- If your servers can reach the bucket by the time you restore, seed from the artifact with + `CREATE OR REPLACE DATABASE ... OPTIONS { existingData: 'use', seedURI: '...' }`. That is the + path `infrahub-backup restore` automates, so at that point you no longer need this procedure. +- If they cannot, restore on the server itself with the vendor restore command, which needs + filesystem access to the server's store directory. `dbms.recreateDatabase()` is a further + Cypher-level route the vendor documents for both standalone and clustered servers. +- Restore the PostgreSQL dump with `pg_restore` against your task manager database. + +:::warning +Whichever route you take, confirm the database came online. A seed is downloaded and validated +when the database starts, not when the statement returns, so a failed fetch leaves the database +offline with a `statusMessage` of `Unable to start database` rather than failing the statement. +Check `SHOW DATABASES` for `currentStatus` and `statusMessage` before you call the restore done. +::: + +## Returning to the supported path + +This procedure is a stopgap, and the gap it covers is narrow: your Neo4j servers cannot reach an +object store, or lack the provider tooling and credentials to fetch from one. Once either is +arranged, `infrahub-backup create` and `infrahub-backup restore` handle the whole round trip, with +metadata, checksums, retention and a restore that verifies the database came online. Revisit the +prerequisites in +[Prepare an external database for backup and restore](./external-databases.mdx) when your database +estate changes. + +## Related resources + +- [Prepare an external database for backup and restore](./external-databases.mdx) +- [Back up your Infrahub instance](./create.mdx) +- [Manage backup retention](./retention.mdx) +- [Configuration reference](../reference/configuration.mdx) diff --git a/docs/docs/backup/external-databases.mdx b/docs/docs/backup/external-databases.mdx new file mode 100644 index 00000000..19adaab3 --- /dev/null +++ b/docs/docs/backup/external-databases.mdx @@ -0,0 +1,513 @@ +--- +title: Prepare an external database for backup and restore +--- + +Infrahub deployments on Kubernetes often point at a Neo4j or PostgreSQL database that runs outside +the cluster — a managed service, or a server the deployment only reaches over the network. Backing +up such a database needs one-time changes on the database side, because `infrahub-backup` cannot +reach the storage of a server it does not host. This guide covers each change, and how to confirm +it took effect before you depend on it for recovery. + +Work through the steps that apply to you. If your Neo4j and PostgreSQL databases both run inside +the deployment, none of this applies and nothing here changes how your backups behave. + +## Prerequisites + +- Infrahub is deployed on Kubernetes. Docker Compose deployments with external databases are not + supported yet. +- Your external Neo4j server runs Neo4j Enterprise. Community Edition cannot be captured over the + network — its backup mechanism requires the database to be stopped and its store directory to be + directly accessible — so `infrahub-backup` refuses the capture rather than producing a partial + one. Keep Community databases inside the deployment. +- If you run `infrahub-backup` as a pod in the Infrahub namespace, the Job and Secret permissions + below do not apply: see + [Run `infrahub-backup` as a pod in the Infrahub namespace](#run-infrahub-backup-as-a-pod-in-the-infrahub-namespace). +- Your Kubernetes credentials can create and remove a Job in the Infrahub namespace. The namespace + must grant `create`, `get` and `delete` on `jobs`; `get`, `list`, `watch` and `delete` on `pods`; + and `create` on `pods/exec`. The `watch` verb is what the readiness wait needs: the tool waits + for the Job's pod with `kubectl wait`, which opens a watch. The Job has a deadline and a + time-to-live (TTL), so the cluster removes it and its pod if the CLI terminates unexpectedly. +- Permissions on `secrets` are needed only in some cases. When the Infrahub server (for Neo4j) or + the task manager (for PostgreSQL) takes the database username and password from Secret keys, the + Job reads the same Secret keys and the tool creates no secret. In every other case, the tool + stores the credentials in a secret of its own, owned by the Job, and the namespace must also + grant `create`, `list` and `delete` on `secrets`. Those other cases are: + - you supply the credentials in the environment of `infrahub-backup` + - the pod specification states the username or the password as a plain value + - the PostgreSQL username and password exist only inside the task manager's connection URL, + which is how the Prefect server chart renders them unless you set + `externalDatabase.usernameSecretKey` and `externalDatabase.passwordSecretKey` + + The log states which case applies to each database. +- Administrative access to the Neo4j server configuration, and the ability to restart it. +- An S3-compatible bucket, if you intend to restore. Restore stages the artifact there for the + database to fetch. + +:::info +`infrahub-backup` discovers the external endpoint from the environment of the Infrahub components +still running in the namespace, so you do not normally supply the host list or the credentials. +The one value it cannot discover is the Neo4j backup service port, because Infrahub never uses it. +::: + +## Step 1: Expose the Neo4j backup service on a non-loopback address + +Neo4j enables its backup service by default, but binds it to `127.0.0.1:6362`. A backup taken from +anywhere other than the database host itself therefore fails to connect until you change the +server configuration. This is a configuration change and a restart, not only a firewall rule. + +To let the Infrahub namespace reach the service, set a non-loopback listen address in `neo4j.conf` +on every server you want to back up from: + +```ini +server.backup.enabled=true +server.backup.listen_address=0.0.0.0:6362 +``` + +Restart the server, then allow traffic from the Infrahub namespace to port `6362`. + +:::warning +Port `6362` is the backup service, not the client port. Bolt traffic uses `7687`, and pointing the +backup at `7687` fails even though Infrahub connects there successfully. `infrahub-backup` defaults +to `6362`; use `--neo4j-backup-port` only if your server listens elsewhere. +::: + +If your Neo4j deployment is clustered, list every member so the capture can fall back when one is +down or does not host the database: + +```bash +infrahub-backup create \ + --k8s-namespace infrahub \ + --neo4j-address "neo4j-a.example.internal,neo4j-b.example.internal,neo4j-c.example.internal" +``` + +Members without an explicit port use the discovered client port, and the backup port applies to +all of them. Omit `--neo4j-address` to use the host list the deployment already carries. + +## Step 2: Give each Neo4j server a route to the artifact store + +Restore reverses the direction data travels. A capture pulls: a transient pod in your namespace +runs the vendor backup client against the server, and the bytes come back through the pod. A +restore pushes nothing — there is no remote equivalent of a local restore command — so +`infrahub-backup` stages the artifact in your object store and asks the server to fetch it: + +```cypher +CREATE OR REPLACE DATABASE neo4j OPTIONS { existingData: 'use', seedURI: 's3://...' } +``` + +Three consequences follow, and all three land on the database host rather than on the machine +running `infrahub-backup`: + +- **The Neo4j servers need network egress to the object store.** The tool proves that an + independent reader can address the staged object, which is the closest check it can make from + the namespace. It cannot verify the route from the database host, and does not claim to. +- **The servers need credentials for the object store, configured server-side.** The seed provider + is built into the server and uses the cloud vendor's own SDK, so the credentials live in your + server configuration rather than travelling inside a Cypher statement. On a Neo4j 5.26 or 2025.x + server the built-in providers are `S3SeedProvider` and `CloudSeedProvider`, which you can confirm + with `CALL dbms.listConfig('dbms.databases.seed_from_uri_providers')`. +- **The servers also need a region, not only credentials.** The AWS SDK resolves one from + `AWS_REGION`, the `aws.region` system property, or a profile — and if it resolves none, the + `CREATE DATABASE` statement still *succeeds* and the database then sits `offline` with a + `statusMessage` reading `Unable to load region from any of the providers in the chain …`. This is + the most common way a seed that looks accepted turns out not to have loaded, so set the region + explicitly rather than relying on instance metadata. +- **Only object stores work without further server configuration.** The `s3`, `gs` and `azb` + schemes are built in. Any other scheme — including HTTP and local files — requires + `dbms.databases.seed_from_uri_providers` to be configured on every server, which + `infrahub-backup` cannot see, so it refuses such a URI before the destructive step rather than + after it. + +Point the tool at the bucket you want it to stage into: + +```bash +infrahub-backup restore \ + --k8s-namespace infrahub \ + --allow-external-restore \ + --s3-bucket my-infrahub-backups \ + --s3-region us-east-1 +``` + +S3-compatible stores work, not only AWS. For a self-hosted store such as MinIO, add +`--s3-endpoint` so the tool stages there, and point the servers' own fetch at the same place with +`AWS_ENDPOINT_URL_S3` (alongside `AWS_REGION`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) in +each Neo4j server's environment. The built-in `s3` provider honours that endpoint, so you do not +need an AWS account to use restore. + +The staged copy is written below your archives, at `seed//` beneath the configured +`--s3-prefix`. Retention only recognizes archive keys, so a staged seed is never a pruning +candidate. Each artifact gets a folder of its own because a Neo4j server reads the other backup +files beside a seed as one backup chain and refuses the seed when they do not form one. A restore +that fails leaves its staged copy in place so you can inspect it; delete it by hand once you no +longer need it. + +:::warning +Confirm both prerequisites with whoever administers your database before you rely on +`infrahub-backup restore` for recovery, and rehearse a restore into a disposable database first. +A server that cannot fetch the artifact fails the restore after the destructive step, not before +it. If you cannot meet these prerequisites, see +[Capture an external Neo4j database directly to cloud storage](./external-database-direct-to-cloud.mdx). +::: + +## Step 3: Choose the client TLS policy + +Connections to an external database server are encrypted and the server's certificate is verified, +by default, for both databases. You do not name a policy: the choice is between verifying and +opting out. + +`infrahub-backup` carries your deployment's own trust anchor into the transient workload, so a +privately issued certificate verifies without any action from you: + +- For Neo4j, it reads the certificate authority named by `INFRAHUB_DB_TLS_CA_FILE` out of the + Infrahub server container and installs it where the client looks. +- For PostgreSQL, it reads the `sslrootcert` named by the task manager connection URL. Where the + deployment names none, the client falls back to the trust store in the `infrahub-backup` image, + which is what lets a publicly issued certificate verify. + +If verification cannot succeed — the certificate is issued by an authority your deployment does not +record, or the server offers no TLS at all — opt out explicitly: + +```bash +infrahub-backup create --k8s-namespace infrahub --external-db-insecure-tls +``` + +:::warning +`--external-db-insecure-tls` is wider than skipping certificate verification. It also accepts a +channel with no encryption at all where the deployment records TLS as disabled, which means your +database password leaves the cluster in clear text. Treat it as a temporary measure while you fix +the trust chain, not as a setting. + +An external Neo4j server with TLS disabled **requires** this flag. A discovered +`INFRAHUB_DB_TLS_INSECURE` in your deployment does not confer the opt-out either — the decision has +to be made about this tool, deliberately. +::: + +Neo4j's backup protocol is separate from this. It authenticates through the server's own backup SSL +policy rather than through `INFRAHUB_DB_TLS_CA_FILE`, so configure that policy on the server if you +require TLS on port `6362`. + +## Step 4: Authorize a destructive restore + +Restoring into a database the deployment does not manage overwrites data in a system Infrahub does +not own, so it needs its own authorization. Grant it on the command line: + +```bash +infrahub-backup restore --k8s-namespace infrahub --allow-external-restore +``` + +Or in the configuration a scheduled restore reads: + +```bash +export INFRAHUB_ALLOW_EXTERNAL_RESTORE=true +``` + +:::warning +This authorization stands on its own. `--force` does not confer it, and enabling scheduled restore +does not confer it. Setting `INFRAHUB_ALLOW_EXTERNAL_RESTORE=true` in the configuration of a +scheduled restore is the only way an unattended destructive restore into an external database can +happen — so set it deliberately, and review it when you review the schedule. +::: + +## Step 5: Make the `infrahub-backup` image reachable + +Capturing an external database runs `neo4j-admin`, `cypher-shell` and the PostgreSQL client in a +transient pod. That pod runs the `infrahub-backup` image at the version of the binary you run: a +`2.3.0` binary runs `registry.opsmill.io/opsmill/infrahub-backup:2.3.0`. The image carries the +database tools at versions it pins, so the pod needs no other image. Unlike the opt-in benchmark in +`infrahub-collect`, this pull cannot be skipped: without it there is no client to reach the +database with, and a pull that fails is a failed backup. + +If your cluster can pull from `registry.opsmill.io`, you need nothing here. If it cannot, mirror +the `infrahub-backup` image at your binary's version into a registry the cluster can reach, and +name it: + +```bash +infrahub-backup create \ + --k8s-namespace infrahub \ + --external-db-image registry.example.com/opsmill/infrahub-backup:2.3.0 \ + --external-db-image-pull-secrets my-registry-credentials +``` + +`--external-db-image` accepts no environment variable, by design — the tool can create workloads +in your namespace, so an image settable from ambient environment would turn that into a way to run +arbitrary code there. Supply it on the command line. + +A build of `infrahub-backup` that is not a release, such as one built from a source checkout, has +no published image. It refuses to back up or restore an external database until you name one with +`--external-db-image`, for example an image you built from the repository's `Dockerfile`. + +If your namespace admits a pod only under particular placement rules, the remaining flags cover +them, and each is also settable through the environment: + +```bash +infrahub-backup create \ + --k8s-namespace infrahub \ + --external-db-service-account backup-runner \ + --external-db-node-selector "workload=maintenance" \ + --external-db-tolerations "maintenance=true:NoSchedule" \ + --external-db-priority-class backup-priority +``` + +The pod mounts no service-account token, so naming an account grants it no cluster access — what +it carries is registry credentials and whatever your admission rules require. + +## Run `infrahub-backup` as a pod in the Infrahub namespace + +You can run `infrahub-backup` inside the cluster instead of from your workstation, for example as +the Job or CronJob that the [backup Helm chart](./kubernetes-backup.mdx) deploys. When that pod +runs the `infrahub-backup` image, it already carries `neo4j-admin`, `cypher-shell` and the +PostgreSQL client. In that case `infrahub-backup` runs these tools in its own container and creates +no Job and no Secret in the namespace. +To set up the chart for an external database, see +[Configure the `infrahub-backup` Helm chart](#configure-the-infrahub-backup-helm-chart). + +`infrahub-backup` runs the tools in its own container when both of these conditions hold: + +- The pod's namespace, which Kubernetes writes to + `/var/run/secrets/kubernetes.io/serviceaccount/namespace`, is the namespace you back up or + restore (`--k8s-namespace`). +- Every program the database needs is on the container's `PATH`: + - for Neo4j: `neo4j-admin`, `cypher-shell`, `keytool`, `sh`, `tar`, `rm`, `mkdir`, `ls` and + `base64` + - for PostgreSQL: `pg_dump`, `pg_restore`, `psql`, `sh`, `rm` and `base64` +- The container's temporary directory (`TMPDIR`, or `/tmp` when it is unset) is a path made only + of letters, digits and the characters `/`, `.`, `_`, `-` and `+`. If you point `TMPDIR` at a larger + volume, keep its path to those characters. + +In every other case, including any run from outside the cluster, `infrahub-backup` uses the +transient Job. Each run logs one line at the `info` level that names the mode it took and why, for +example `Running the external database tools in this process, with no transient Job: this pod runs +in the target namespace "infrahub" and finds neo4j tools on PATH`. + +### Permissions in this mode + +The pod's service account needs the same permissions as a backup of a deployment whose databases +run inside it, listed in [Required permissions](./kubernetes-backup.mdx#required-permissions): + +- `list` and `get` on `pods`, to find the Infrahub components +- `create` on `pods/exec`, to read the database address, credentials and certificate authority from + the Infrahub components +- `get` on `pods/log`, to follow the progress of commands run in the Infrahub components +- `get`, `list` and `patch` on `deployments` and `statefulsets`, to find the Infrahub components' + workloads by label +- `get` and `patch` on `deployments/scale` and `statefulsets/scale`, to stop and restart the + Infrahub components around an offline capture or a restore. `kubectl scale` writes to these + resources, and `patch` on the workload itself does not cover them + +It needs no permission on `jobs` or `secrets`, and no `create`, `delete` or `watch` on `pods`. + +### What the pod itself needs + +- Network access from the pod to each database: the Neo4j client port (`7687` by default), the + Neo4j backup port you exposed in Step 1, and the PostgreSQL port (`5432` by default). If your + namespace enforces default-deny egress, add these to the backup pod's egress rules. +- Free disk space under the pod's temporary directory. The capture writes the artifact there + before the run archives it, and the run checks the free space against the same size it would + give the Job: the store size with headroom, or `--external-db-scratch-size`. To use a larger + volume, mount it in the pod and set `TMPDIR` to its path. + +Each run works in a directory of its own under the temporary directory and removes it on every +exit path, including a failure. The credentials reach the database tools only through their +environment, never through their command line. + +The flags that describe the transient Job — `--external-db-image`, +`--external-db-image-pull-secrets`, `--external-db-service-account`, `--external-db-node-selector`, +`--external-db-tolerations` and `--external-db-priority-class` — have no effect in this mode, +because no Job is created. All other checks are unchanged: the version and edition checks, the +scratch-size check, the removal of an incomplete capture, and the authorization of a restore. + +`infrahub-collect --include-backup` still refuses to capture an external database in this mode. + +### Configure the `infrahub-backup` Helm chart + +The Job and the CronJob of the `infrahub-backup` Helm chart run in the Infrahub namespace, so they +always run the database tools in their own container. The chart's `externalDatabase` values set +the environment variables of `infrahub-backup` for an external database: + +- `externalDatabase.neo4jAddress` sets `INFRAHUB_NEO4J_ADDRESS`, the Neo4j hosts to back up from. + For a Neo4j cluster, list every member. Empty uses the address the Infrahub server is configured + with. +- `externalDatabase.neo4jBackupPort` sets `INFRAHUB_NEO4J_BACKUP_PORT`, the port you exposed in + Step 1. Empty uses `6362`. +- `externalDatabase.postgresAddress` sets `INFRAHUB_POSTGRES_ADDRESS`. Empty uses the address the + task manager is configured with. +- `externalDatabase.insecureTLS` sets `INFRAHUB_EXTERNAL_DB_INSECURE_TLS`, the opt-out from Step 3. + An external Neo4j server with TLS disabled needs it. +- `externalDatabase.timeout` sets `INFRAHUB_EXTERNAL_DB_TIMEOUT`, the time limit of each operation + against the database, such as `4h`. Empty uses the default, 2 hours. +- `externalDatabase.allowRestore` sets `INFRAHUB_ALLOW_EXTERNAL_RESTORE`, the authorization from + Step 4. The chart sets it on the restore containers only. +- `restore.options.neo4jClusterOnlineTimeout` passes `--neo4j-cluster-online-timeout`: how long a + restore waits for the database to come online on every member of a Neo4j cluster. + +The chart has no value for the following settings. Set them with the chart's `extraEnv`, +`extraVolumes` and `extraVolumeMounts` values: + +- The scratch size: set `INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE`, such as `20Gi`. A backup of a Neo4j + 2025 server needs it, because that server does not report the size of its store. +- A larger volume for the temporary files: mount it with `extraVolumes` and `extraVolumeMounts`, and + set `TMPDIR` to its mount path with `extraEnv`. The chart runs the pod as user `1000`, so a + `PersistentVolumeClaim` usually also needs `podSecurityContext.fsGroup: 1000` for the pod to write + to it. + +The chart has no value for `--external-db-image` or for the other flags of the transient Job, +because its pods never create one. + +The pods must run an `infrahub-backup` image that supports external databases. If the chart's +default image does not, set `image.tag` to a release that does. If you deploy the chart through the +Infrahub Helm chart, put the values below under the `infrahub-backup:` key, and use an Infrahub +chart version that pins an `infrahub-backup` chart with the `externalDatabase` values. + +To back up an external Neo4j cluster every night: + +```yaml +# backup-values.yaml +backup: + enabled: true + mode: "cronjob" + schedule: "0 2 * * *" + storage: + type: "s3" + s3: + bucket: "my-infrahub-backups" + region: "us-east-1" + secretName: "backup-s3-credentials" + +externalDatabase: + neo4jAddress: "neo4j-a.example.internal,neo4j-b.example.internal,neo4j-c.example.internal" + +extraEnv: + - name: INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE + value: "20Gi" +``` + +A restore of an external Neo4j database has these further requirements: + +- `restore.storage.type` must be `s3`. The restore copies the archive to `seed//` + beneath `restore.storage.s3.prefix` in `restore.storage.s3.bucket`, and the Neo4j servers + download it from there. +- The credentials in `restore.storage.s3.secretName` must be able to write to that bucket. This is + also true when `restore.storage.s3.key` names the archive. +- The Neo4j servers must be able to read from that bucket, as described in Step 2. + +To restore one archive into the external database: + +```yaml +# restore-values.yaml +backup: + enabled: false + +restore: + enabled: true + mode: "job" + storage: + type: "s3" + s3: + bucket: "my-infrahub-backups" + key: "infrahub_backup_20250120_020000.tar.gz" + region: "us-east-1" + secretName: "backup-s3-credentials" + +externalDatabase: + allowRestore: true +``` + +## Validation + +Confirm where the tool resolved each database to, before you run a backup: + +```bash +infrahub-backup environment detect --k8s-namespace infrahub +``` + +Each database is reported as internal or external, with the endpoint it resolved to. An endpoint +you do not recognize means discovery read a setting you did not expect, so correct it with +`--neo4j-address` or `--postgres-address` rather than proceeding. + +Then take a backup and read what it reports: + +```bash +infrahub-backup create --k8s-namespace infrahub +``` + +A run that reached every endpoint exits `0`. A run that produced an artifact but could not reach +every requested endpoint exits non-zero and removes the partial artifact from every location it +reached, so an incomplete capture never becomes a restore candidate. + +Finally, rehearse a restore into a disposable database: + +```bash +infrahub-backup restore --k8s-namespace infrahub --allow-external-restore --backup-id +``` + +:::warning +A `CREATE DATABASE` statement that returns successfully proves nothing about the seed. Neo4j +downloads and validates the artifact when the database starts, not when the statement returns, so +a failed fetch leaves the database offline rather than failing the statement. `infrahub-backup` +polls `SHOW DATABASES` and treats "created but not online" as a failed restore. If you ever drive +a seed by hand, read `currentStatus` and `statusMessage` yourself — `Unable to start database` +means the seed failed. +::: + +## Troubleshooting + +### The capture cannot connect to the Neo4j backup port + +The listener is still on loopback, or the namespace has no route to it. Re-check +`server.backup.listen_address` on the server, confirm the server was restarted after the change, +and confirm you are targeting `6362` rather than the client port. + +### The capture is refused because the server is Community Edition + +This is by design and there is no flag for it. Community's backup mechanism requires the database +to be stopped and its store directory to be readable, so it cannot be performed over the network. +Move the database inside the deployment, or move to Enterprise. + +### The capture is refused on version compatibility + +The database tools in the `infrahub-backup` image are older than the server. A newer client reads +an older server, so the tool aborts in that direction and only warns in the other. Use an +`infrahub-backup` release whose image carries tools at least as new as your server, or point +`--external-db-image` at an image that does. + +### The run is refused as a development build + +The `infrahub-backup` binary you ran is not a release, so no image was published for its version. +Nothing was created or stopped. Name an image with `--external-db-image`, or run a released +`infrahub-backup`. + +### The transient pod never becomes ready + +In a cluster that mirrors images, this is usually an image pull the registry rejected. Name the +pull secret with `--external-db-image-pull-secrets`. If your namespace enforces a restricted pod +security standard, the failure names the security context requirement instead — check the service +account and priority class your namespace admits pods under. If the pod waits with +`CreateContainerConfigError` and names a Secret key, the Job reads the database credentials from +the same Secret as your Infrahub pods, and that Secret no longer holds the key. Restore the key in +that Secret, then run the command again. Restarting the Infrahub pods does not help: pods that read +the same key cannot start without it either. + +### `infrahub-collect --include-backup` refuses to run + +Collection creates no workload in your namespace, and capturing an external database needs a +transient pod and secret there. Take the backup with `infrahub-backup create`, and re-run the +collection without `--include-backup` for the diagnostics. + +### An operation times out on a large database + +Every operation against an external database is time-bound, defaulting to two hours. Raise it with +`--external-db-timeout 4h` or `INFRAHUB_EXTERNAL_DB_TIMEOUT=4h`. The value needs a unit — a bare +number is refused rather than read as nanoseconds. Values below 30 seconds are raised to that +floor. + +### The capture runs out of scratch space + +The backup client writes the artifact to local disk before it streams out, so the transient pod is +sized from the database. Override the sizing with `--external-db-scratch-size 200Gi` when the +derived value is wrong for your data. + +## Related resources + +- [Back up your Infrahub instance](./create.mdx) +- [Restore from a backup](./restore.mdx) +- [Capture an external Neo4j database directly to cloud storage](./external-database-direct-to-cloud.mdx) +- [Configuration reference](../reference/configuration.mdx) diff --git a/docs/docs/backup/kubernetes-backup.mdx b/docs/docs/backup/kubernetes-backup.mdx index a68bf91b..4a35981c 100644 --- a/docs/docs/backup/kubernetes-backup.mdx +++ b/docs/docs/backup/kubernetes-backup.mdx @@ -49,6 +49,7 @@ Before configuring backups: - Infrahub is deployed on Kubernetes via the Infrahub Helm chart - You have access to modify Helm values (directly or through GitOps) - For S3 storage: An S3-compatible bucket and credentials +- If Neo4j or PostgreSQL runs outside the Infrahub namespace: the database is prepared as described in [Prepare an external database for backup and restore](./external-databases.mdx), and the chart values are set as described in [Configure the `infrahub-backup` Helm chart](./external-databases.mdx#configure-the-infrahub-backup-helm-chart) ## Installation methods @@ -389,8 +390,10 @@ The backup process needs these Kubernetes permissions: |----------|-------|---------| | `pods` | list, get | Discover Infrahub pods | | `pods/exec` | create | Execute backup commands in pods | -| `deployments` | get, patch | Scale down/up during backup | -| `statefulsets` | get, patch | Scale down/up during backup | +| `deployments` | get, list, patch | Find the Infrahub workloads by label | +| `statefulsets` | get, list, patch | Find the Infrahub workloads by label | +| `deployments/scale` | get, patch | Scale down/up during backup | +| `statefulsets/scale` | get, patch | Scale down/up during backup | | `pods/log` | get | Monitor backup progress | ### Network access to the API server diff --git a/docs/docs/backup/kubernetes-restore.mdx b/docs/docs/backup/kubernetes-restore.mdx index 3dad9a51..8640ad12 100644 --- a/docs/docs/backup/kubernetes-restore.mdx +++ b/docs/docs/backup/kubernetes-restore.mdx @@ -26,6 +26,7 @@ Before restoring: - Access to modify Helm values for your Infrahub deployment - S3 credentials with read access to the backup bucket - Network egress from the restore pod to the Kubernetes API server (see [Network access to the API server](./kubernetes-backup.mdx#network-access-to-the-api-server)) +- If Neo4j or PostgreSQL runs outside the Infrahub namespace: the database is prepared as described in [Prepare an external database for backup and restore](./external-databases.mdx), and the chart values are set as described in [Configure the `infrahub-backup` Helm chart](./external-databases.mdx#configure-the-infrahub-backup-helm-chart). A restore of an external Neo4j database also needs S3 credentials that can write to the bucket ## Step 1: Identify the backup to restore @@ -316,7 +317,7 @@ With a Neo4j cluster, the restore waits for the restored database to be `online` level=error msg="Database neo4j is not online on this server" reason="Neo4j gave no status message" server="neo4j-1:7687" status="store copying" ``` -- If the status is `store copying` or `starting`, the server was still copying the restored data. Run the restore again with a longer wait, for example `--neo4j-cluster-online-timeout=30m`. With the Helm chart, add the flag to `restore.options.extraArgs`. +- If the status is `store copying` or `starting`, the server was still copying the restored data. Run the restore again with a longer wait, for example `--neo4j-cluster-online-timeout=30m`. With the Helm chart, set `restore.options.neo4jClusterOnlineTimeout`. If your chart version does not have this value, add the flag to `restore.options.extraArgs`. - For any other status, the `reason` field holds the message Neo4j reports for that server, such as `No store found on any of the seeders`. ### Restore fails because the Neo4j server still holds cluster state for the database diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx index 6490daeb..e6f9fee2 100644 --- a/docs/docs/reference/configuration.mdx +++ b/docs/docs/reference/configuration.mdx @@ -53,6 +53,57 @@ Both rules combine as a union, and the most recent backup at each location alway |----------|-------------|---------|---------| | `PREFECT_API_DATABASE_CONNECTION_URL` | PostgreSQL connection string | Auto-detect | `postgresql://user:pass@localhost/prefect` | +### External database configuration + +These apply on Kubernetes when a database resolves as external — running outside the cluster rather +than as a deployment container. For a deployment whose databases are all internal they are inert, +so no existing configuration needs to change. See [Prepare an external database for backup and +restore](../backup/external-databases.mdx). + +| Variable | Description | Default | Example | +|----------|-------------|---------|---------| +| `INFRAHUB_NEO4J_ADDRESS` | Override the Neo4j host list as `host[:port][,host[:port]...]` | Discovered from the deployment | `neo4j-a.example.internal,neo4j-b.example.internal` | +| `INFRAHUB_NEO4J_BACKUP_PORT` | Port the Neo4j backup service listens on. Not the client port | `6362` | `6362` | +| `INFRAHUB_POSTGRES_ADDRESS` | Override the PostgreSQL endpoint as `host[:port]` | Discovered from the deployment | `pg.example.internal:5432` | +| `INFRAHUB_ALLOW_EXTERNAL_RESTORE` | Authorize a destructive restore into a database the deployment does not manage | `false` | `true` | +| `INFRAHUB_EXTERNAL_DB_INSECURE_TLS` | Do not protect connections to an external database server | `false` | `true` | +| `INFRAHUB_EXTERNAL_DB_TIMEOUT` | Bound on any single operation against an external database. Needs a unit | `2h` | `4h` | +| `INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE` | Scratch space for the transient backup workload | Derived from the database size | `200Gi` | +| `INFRAHUB_EXTERNAL_DB_IMAGE_PULL_SECRETS` | Comma-separated secrets in the deployment namespace holding registry credentials for the `infrahub-backup` image the transient workload runs | unset | `my-registry-credentials` | +| `INFRAHUB_EXTERNAL_DB_SERVICE_ACCOUNT` | Service account the transient workload runs under | unset | `backup-runner` | +| `INFRAHUB_EXTERNAL_DB_NODE_SELECTOR` | Comma-separated `key=value` node labels the transient workload must be scheduled onto | unset | `workload=maintenance` | +| `INFRAHUB_EXTERNAL_DB_TOLERATIONS` | Comma-separated taints the transient workload tolerates, as `key=value:Effect`, `key:Effect` or a bare `key` | unset | `maintenance=true:NoSchedule` | +| `INFRAHUB_EXTERNAL_DB_PRIORITY_CLASS` | PriorityClass the transient workload is admitted under | unset | `backup-priority` | + +:::warning +There is deliberately no environment variable for the transient workload's image. +`--external-db-image` is the only flag in these tools with no environment equivalent: the tool can +create workloads in your namespace, so an image settable from ambient environment would turn that +permission into a way to run arbitrary code there. The placement +variables above are bound, because they say where a pod may run and which credentials fetch its +image, never what it runs. + +`INFRAHUB_EXTERNAL_DB_TIMEOUT` needs a unit — `4h`, `90m`, `1h30m`. A bare number is refused and +the bound is left unchanged, rather than being read as nanoseconds. Values below 30 seconds are +raised to that floor. +::: + +### Settings read from the deployment + +`infrahub-backup` reads these out of the Infrahub components still running in the namespace to +discover an external endpoint. You do not set them for the tool — they are Infrahub's own settings, +listed here so you can see where a reported endpoint came from. + +| Variable | Used for | +|----------|----------| +| `INFRAHUB_DB_ADDRESS` | Neo4j host list, as comma-separated `host[:port]` | +| `INFRAHUB_DB_PORT` | Neo4j client port. Not the backup port | +| `INFRAHUB_DB_PROTOCOL` | `bolt` or `neo4j` | +| `INFRAHUB_DB_TLS_ENABLED` | Whether the deployment encrypts its own database connections | +| `INFRAHUB_DB_TLS_INSECURE` | Whether the deployment verifies its own database certificate. Does not confer `--external-db-insecure-tls` | +| `INFRAHUB_DB_TLS_CA_FILE` | Certificate authority the tool carries into the transient workload | +| `PREFECT_API_DATABASE_CONNECTION_URL` | PostgreSQL host, port, database, user, password and `sslrootcert` | + ### Collect configuration | Variable | Description | Default | Example | @@ -80,6 +131,46 @@ Both rules combine as a union, and the most recent backup at each location alway | `--retention-days` | Prune backups older than N days after a successful backup | | `--retention-count` | Keep only the N most recent backups after a successful backup | +### External database flags + +Available on both `create` and `restore`, and inert unless a database resolves as external. Every +one has an environment override except `--external-db-image`. + +| Flag | Environment Override | Description | +|------|---------------------|-------------| +| `--neo4j-address` | `INFRAHUB_NEO4J_ADDRESS` | Override the Neo4j host list as `host[:port][,host[:port]...]` | +| `--neo4j-backup-port` | `INFRAHUB_NEO4J_BACKUP_PORT` | Port the Neo4j backup service listens on, default `6362`. Not the client port | +| `--postgres-address` | `INFRAHUB_POSTGRES_ADDRESS` | Override the PostgreSQL endpoint as `host[:port]` | +| `--allow-external-restore` | `INFRAHUB_ALLOW_EXTERNAL_RESTORE` | Authorize a destructive restore into a database the deployment does not manage | +| `--external-db-insecure-tls` | `INFRAHUB_EXTERNAL_DB_INSECURE_TLS` | Skip certificate verification, and accept a channel with no encryption where the deployment records TLS as disabled | +| `--external-db-timeout` | `INFRAHUB_EXTERNAL_DB_TIMEOUT` | Bound on any single operation against an external database, default `2h` | +| `--external-db-scratch-size` | `INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE` | Scratch space for the transient backup workload, such as `200Gi` | +| `--external-db-image` | none, by design | Image the transient workload runs for both databases. Default `registry.opsmill.io/opsmill/infrahub-backup:`, the version of the binary you run. A build that is not a release has no default and requires this flag | +| `--external-db-image-pull-secrets` | `INFRAHUB_EXTERNAL_DB_IMAGE_PULL_SECRETS` | Secrets in the deployment namespace holding registry credentials for the `infrahub-backup` image | +| `--external-db-service-account` | `INFRAHUB_EXTERNAL_DB_SERVICE_ACCOUNT` | Service account the transient workload runs under. It mounts no service-account token | +| `--external-db-node-selector` | `INFRAHUB_EXTERNAL_DB_NODE_SELECTOR` | Node labels the transient workload must be scheduled onto | +| `--external-db-tolerations` | `INFRAHUB_EXTERNAL_DB_TOLERATIONS` | Taints the transient workload tolerates | +| `--external-db-priority-class` | `INFRAHUB_EXTERNAL_DB_PRIORITY_CLASS` | PriorityClass the transient workload is admitted under | + +#### External restore authorization + +Restoring into a database the deployment does not manage overwrites data in a system Infrahub does +not own, so it carries its own authorization, separate from every other gate: + +- `--force` does **not** confer it. `--force` skips the running-tasks consistency check and + nothing else. +- Enabling scheduled restore does **not** confer it. +- For an unattended restore, set `INFRAHUB_ALLOW_EXTERNAL_RESTORE=true` in the schedule's + configuration. The flag form exists for interactive use, and the tool reports which of the two + channels supplied the authorization. +- A value that is not a boolean leaves external restore unauthorized, with a warning. + +Without it, an external restore exits non-zero and the database is untouched. + +There is no TLS policy flag. `--external-db-insecure-tls` is the only opt-out from verifying an +external database server, and `--external-db-tls-policy` does not exist — an earlier plan named it, +but the channel it would have configured takes no policy name. + ### Collect command flags | Flag | Environment Override | Description | @@ -98,6 +189,11 @@ Both rules combine as a union, and the most recent backup at each location alway ## Auto-detection behavior +When you set `--project` (`INFRAHUB_PROJECT`) or `--k8s-namespace` (`INFRAHUB_K8S_NAMESPACE`), the +command acts only on that deployment. If it cannot reach it, the command fails and does not look for +another deployment on the same host. If you set both, the command tries the Kubernetes namespace +first and then the Docker Compose project, and fails only when it reaches neither. + ### Docker Compose project detection Order of detection: diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 5a3dab4b..cefc1c43 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,6 +22,8 @@ const sidebars: SidebarsConfig = { 'backup/kubernetes-backup', 'backup/kubernetes-restore', 'backup/kubernetes-scheduled-restore', + 'backup/external-databases', + 'backup/external-database-direct-to-cloud', ], }, { diff --git a/scripts/check-image-tools.sh b/scripts/check-image-tools.sh new file mode 100755 index 00000000..8a533d73 --- /dev/null +++ b/scripts/check-image-tools.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Build the infrahub-backup image and check that it carries the database tools (FR-030, ADR 0017). +# +# Each tool runs inside the image as the image's own user (no --user override), so the check fails +# if a tool is missing, cannot start, or needs root. The image must not run as root. +# +# Usage: scripts/check-image-tools.sh [image-tag] +# image-tag tag to build and check (default: infrahub-backup:check-tools) +# Environment: +# SKIP_BUILD=1 check an existing image instead of building it +set -euo pipefail + +IMAGE="${1:-infrahub-backup:check-tools}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ "${SKIP_BUILD:-0}" != "1" ]]; then + echo "Building ${IMAGE} from ${REPO_ROOT}..." + docker build --build-arg VERSION="${VERSION:-}" -t "${IMAGE}" "${REPO_ROOT}" +fi + +uid="$(docker run --rm --entrypoint id "${IMAGE}" -u)" +user="$(docker image inspect "${IMAGE}" --format '{{.Config.User}}')" +echo "Image user: ${user} (uid ${uid})" +if [[ "${uid}" == "0" ]]; then + echo "FAIL: the image runs as root" >&2 + exit 1 +fi + +failed=0 +for tool in neo4j-admin cypher-shell pg_dump pg_restore psql; do + if output="$(docker run --rm --entrypoint "${tool}" "${IMAGE}" --version 2>&1)"; then + echo "ok ${tool}: ${output}" + else + echo "FAIL ${tool}: ${output}" >&2 + failed=1 + fi +done + +size="$(docker image inspect "${IMAGE}" --format '{{.Size}}')" +echo "Image size: ${size} bytes ($((size / 1024 / 1024)) MiB)" + +if [[ "${failed}" != "0" ]]; then + echo "FAIL: at least one database tool is missing or does not run" >&2 + exit 1 +fi +echo "All database tools present in ${IMAGE}" diff --git a/specs/archive/007-external-database-backup/EXTRACTED.md b/specs/archive/007-external-database-backup/EXTRACTED.md new file mode 100644 index 00000000..37e84195 --- /dev/null +++ b/specs/archive/007-external-database-backup/EXTRACTED.md @@ -0,0 +1,39 @@ +# Extraction Record + +**Extracted on**: 2026-09-07 +**Extracted by**: speckit.opsmill.extract + +## ADRs Created + +- dev/adr/0009-transient-workload-lifecycle-and-credential-ownership.md (from R5, R7; FR-011/014/023/028; data-model `TransientWorkload`) +- dev/adr/0010-neo4j-restore-by-seed-uri.md (from R2; FR-007/020/021; artifact contract "Restore-side staging") +- dev/adr/0011-asymmetric-version-compatibility.md (from R3; FR-006) +- dev/adr/0012-location-from-positive-evidence.md (from R4; FR-001/002/003/004; data-model `DatabaseEndpoint`) +- dev/adr/0013-incomplete-captures-removed-metadata-additive.md (from R6; FR-005/012/016/024; artifact contract) +- dev/adr/0014-separate-external-restore-authorisation.md (from spec FR-009/010; cli-surface contract) +- dev/adr/0015-verify-by-default-external-tls.md (from FR-029; cli-surface contract, T100) +- dev/adr/0016-single-ownership-decider.md (from retrospective R10; FR-002/028; `kubernetes_ownership.go`) + +Previously extracted from this spec, before this record: dev/adr/0008-transient-workload-for-external-databases.md (2026-09-03), dev/knowledge/container-image-user-semantics.md (2026-09-02). + +## Knowledge Updated + +- dev/knowledge/external-database-backup.md (new file) +- dev/knowledge/infrahub-collect.md (Gotchas) +- dev/adr/0008-transient-workload-for-external-databases.md (Source path moved to the archive) + +## Guidelines Updated + +- dev/guidelines/kubernetes-backend.md (new file) +- dev/guidelines/implementation-verification.md (new file) + +## Not extracted + +- plan.md, tasks.md, quickstart.md, checklists/, critiques/, opsmill-implement-report.md — execution artifacts; the quickstart scenarios live on as `tests/e2e/test_k8s_external_*.py`, and the report's §5 residuals are ticket material. +- research R1 — recorded as context in ADR 0008. +- research R8 — restates constitution Principle IV. +- retrospective R1–R5, R11–R13 — applied to AGENTS.md on 2026-09-02, or declined. + +## Archive + +Spec directory moved to `specs/archive/007-external-database-backup/` as a historical record. diff --git a/specs/archive/007-external-database-backup/checklists/requirements.md b/specs/archive/007-external-database-backup/checklists/requirements.md new file mode 100644 index 00000000..14a4f603 --- /dev/null +++ b/specs/archive/007-external-database-backup/checklists/requirements.md @@ -0,0 +1,71 @@ +# Specification Quality Checklist: External Database Backup and Restore + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-09-02 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +**Iteration 1 (2026-09-02)** — two items required attention; one resolved, one outstanding. + +- *No implementation details*: initially failed. The spec is derived from a code-grounded + design session, and the first draft named internal functions, source files, and the + pod-cache mechanism. Rewritten so requirements state observable behaviour only. Domain + vocabulary that the constitution already treats as a public contract (Kubernetes, Neo4j + editions, PostgreSQL, deployment service names) is retained deliberately — omitting it + would make the spec unusable for its actual audience of infrastructure operators. The + mechanism-level findings are held for `/speckit-plan` Phase 0 rather than lost. +- *No [NEEDS CLARIFICATION] markers remain*: failed with 2 markers. Both were deliberately + retained rather than defaulted, because each decided a different half of User Story 1: + whether the transient-workload permission exists at all, and whether the restore half of the + round trip closes. Two further unknowns from the design session were downgraded rather than + kept as markers — whether the remote Neo4j seed mechanism works on the target server + versions (recorded as a load-bearing assumption with a mandatory Phase 0 research task), and + whether external PostgreSQL instances are themselves highly available (already covered by + FR-005, which accepts an endpoint set). + +**Iteration 2 (2026-09-02)** — both markers resolved by the feature owner; all items pass. + +- *Transient-workload permission*: assumed grantable. Recorded as an explicit assumption with a + stated consequence (a customer who refuses is served by User Story 3, not by a fallback inside + User Story 1) and promoted to FR-019 as a documented prerequisite the tool must not work + around. The reasoning is that this is a smaller grant than one the tool already exercises, + since restore scales Infrahub workloads. +- *Artefact reachability from the database host*: required, documented as a hard prerequisite of + restore (FR-020). Two alternatives were considered and explicitly left out of v1 rather than + silently dropped: an operator-staged artefact (breaks User Story 1's no-manual-steps promise, + so adding it later revises that story rather than extending it) and a PostgreSQL-only restore + (leaves the two databases at different recovery points, and re-opens the backup-only shape + already ruled out). + +**Style checks**: Vale reports template-derived heading-case warnings and four false-positive +spellings ("toolset", "misconfigured", "Reachability"). Not acted on — CI runs Vale against +`./docs/docs` only and rumdl excludes `specs/`, and the profile matches existing spec 005. +Template headings are preserved deliberately. + +All items pass. Ready for `/speckit-plan`. diff --git a/specs/archive/007-external-database-backup/contracts/artifact-and-metadata.md b/specs/archive/007-external-database-backup/contracts/artifact-and-metadata.md new file mode 100644 index 00000000..aa3cc9e7 --- /dev/null +++ b/specs/archive/007-external-database-backup/contracts/artifact-and-metadata.md @@ -0,0 +1,113 @@ +# Contract: Backup artifact and metadata + +**Feature**: 007-external-database-backup + +The artifact is a cross-version contract. The constitution requires that any change to snapshot +layout or metadata preserve the ability to restore backups created by prior released versions, or +document a migration path. This feature takes the first option: **additive only, no migration**. + +--- + +## Layout: unchanged + +The artifact layout produced for an external database is byte-compatible with the one produced for +an internal database of the same edition. This is the point of the design — the vendor tooling runs +in a container either way, and only the container's origin differs. + +Two consequences worth stating, because a previous branch in this repository hit both: + +- The entry MUST be named from the **target database**, not a literal default name. A restore reads + ``-derived names from the source path, so an artifact named for the wrong database + restores only when the names happen to coincide. +- The **raw vendor artifact** is carried, not an archive of a directory containing it. An online + Neo4j Enterprise backup produces exactly one artifact file, so there is nothing to archive, and + an archive is not a valid input to restore. + +An artifact taken from an external database MUST therefore be restorable to an internal deployment +and vice versa. This is a test obligation, not just a statement (see quickstart). + +## Metadata: additive fields + +| Field | Type | Absent means | +|---|---|---| +| `source_endpoints` | ordered list, per database | Endpoints not recorded — a pre-feature artifact | +| `source_roles` | map endpoint → role, per database | Roles not observed | +| `capture_complete` | bool | Complete — a pre-feature artifact was never partial by this definition | +| `external_location` | bool, per database | Internal | + +`source_roles` carries `unknown` as a real value, distinct from the field being absent, and the +contract deliberately stops short of naming which endpoint served the capture. With several +endpoints supplied there is no documented interface that reports it, so recording one would assert +provenance the tool never established — the class of quiet false confidence Principle II targets. +What the operator gets instead is enough to see that a follower *could* have served the capture, +which is the fact that actually bears on the recovery point. + +## Reader rules + +A reader encountering an artifact with **none** of these fields MUST treat it as an internal, +complete capture from an unrecorded endpoint — which is precisely what every previously released +version produced. No operator action, no migration, no warning. + +A reader should never encounter `capture_complete: false`, because FR-012 requires an incomplete +capture to be removed rather than retained. The field is defence in depth: if removal itself fails, +the artifact is still identifiable as unusable. A reader that does encounter it MUST NOT offer it as +a restore point. + +This is what makes the version story safe. Retention decides from artifact names and recency ranks +and never reads metadata, so had incomplete artifacts been retained they would have occupied a +keep slot and evicted a good backup — and an older tool, which ignores unknown fields, would have +restored one without noticing. Removing them instead resolves both without touching retention and +without bumping the metadata version (FR-024). + +## Writer rules + +- `capture_complete` is derived from what the backup operation **reported**, not from its exit + status. A backup that reached only some of several endpoints exits with the same status as one + that failed outright, so exit status alone cannot establish completeness (research R6). +- `source_endpoints` records what was supplied, in try order. It is not a claim that the first + entry served the capture, and no field claims that of any entry. +- Metadata is written for internal captures too, so the fields do not themselves signal + externality — `external_location` does. + +## Restore-side staging (added by T048/T049, and this is the change of record) + +Restoring into an **external** Neo4j inverts the data path: the server fetches the artifact +itself, so the artifact has to be somewhere it can read from. The tool therefore writes one object +the backup path never wrote — the raw `-.backup` extracted from the archive, +staged in the configured bucket under one extra key segment: + +```text +s3:////seed/-.backup +``` + +Four properties of that key are load-bearing: + +- **It is the existing `--s3-*` surface, not a new flag.** The artefact store a restore stages into + is the bucket the operator already configures; a run with none configured is refused before any + destructive step, naming those flags (FR-007, FR-020). +- **The extra segment keeps it out of retention.** Retention recognises an object only where its key + is exactly the one `buildS3Key` would produce for a backup's own base name, so an object one + segment deeper is invisible to it — no keep slot is occupied and no good backup is evicted. +- **It is removed only once the restored database reports online.** Until then the server may still + be fetching it. A failed restore deliberately leaves it in place, with its URI in the error, so it + can be examined. +- **Only `s3`, `gs` and `azb` are accepted.** Those are the seed providers a Neo4j server carries as + shipped; every other scheme needs `dbms.databases.seed_from_uri_providers` configured on each + server, which the tool cannot observe and must not depend on (research R2). + +An internal restore stages nothing and writes no object: this section applies only where the gate +established that the database lives outside the deployment (FR-015). + +## Restore-side compatibility + +Restore version tolerance is forward-only: an artifact restores into the same or a later database +version, not an earlier one. The existing edition-resolution rules are unchanged — a Community +artifact restores by the Community path, and an Enterprise artifact still cannot be restored onto +a Community server. + +## Change control + +Adding a field is additive and requires only that absence keeps its documented meaning. Changing +the meaning of an existing field, renaming one, or altering the layout is a breaking change and +requires a documented migration path plus a release-note callout, per the constitution's +backup-engine and deployment-contract clauses. diff --git a/specs/archive/007-external-database-backup/contracts/cli-surface.md b/specs/archive/007-external-database-backup/contracts/cli-surface.md new file mode 100644 index 00000000..446e8b18 --- /dev/null +++ b/specs/archive/007-external-database-backup/contracts/cli-surface.md @@ -0,0 +1,177 @@ +# Contract: CLI surface + +**Feature**: 007-external-database-backup + +The public interface of a CLI tool is its flags, its environment variables, its exit behaviour and +the wording of its failures. All four are contracts here — the failure wording especially, because +SC-008 requires an operator to identify the remedy from the message alone. + +Flags follow the existing convention: registered on the root command, bound through viper, and +overridable by `INFRAHUB_`-prefixed environment variables with `-` replaced by `_`. + +--- + +## New flags + +| Flag | Applies to | Default | Purpose | +|---|---|---|---| +| `--neo4j-address` | backup, restore | *(discovered)* | Override the Neo4j host list. Accepts `host[:port][,host[:port]...]`, the same shape Infrahub's own address setting accepts. | +| `--neo4j-backup-port` | backup | `6362` | The backup service port. Not discoverable; see research R4. | +| `--postgres-address` | backup, restore | *(discovered)* | Override the PostgreSQL endpoint as `host[:port]`. | +| `--external-db-image` | backup, restore | `registry.opsmill.io/opsmill/infrahub-backup:` *(none on a development build)* | Image the transient workload runs, for both databases. Flag only — not environment-settable (FR-027, FR-031). | +| `--external-db-scratch-size` | backup | *(derived from database size)* | Scratch space for the transient workload (FR-022). | +| `--external-db-insecure-tls` | backup, restore | `false` | Do not protect connections to an external database server: skip certificate verification, and accept an unencrypted channel where the deployment records TLS as disabled. The **only** opt-out from FR-029; neither a discovered `INFRAHUB_DB_TLS_INSECURE` nor an absent `INFRAHUB_DB_TLS_ENABLED` implies it. | +| `--allow-external-restore` | restore | `false` | The authorisation required by FR-009. Distinct from `--force`. | +| `--external-db-timeout` | backup, restore | *(generous default)* | Bound on any single operation against an external database (FR-025). | +| `--external-db-image-pull-secrets` | backup, restore | *(none)* | Comma-separated secrets in the deployment's namespace holding registry credentials for the tooling image. | +| `--external-db-service-account` | backup, restore | *(none)* | Service account the transient workload runs under. It still mounts no service-account token. | +| `--external-db-node-selector` | backup, restore | *(none)* | Comma-separated `key=value` node labels the transient workload must be scheduled onto. | +| `--external-db-tolerations` | backup, restore | *(none)* | Comma-separated taints the workload tolerates, written as a taint is: `key=value:Effect`, `key:Effect`, or a bare `key`. | +| `--external-db-priority-class` | backup, restore | *(none)* | PriorityClass the transient workload is admitted under. | + +**T111 added the five placement flags above, and this is the change of record.** The manifest +declared no `imagePullSecrets`, `serviceAccountName`, `nodeSelector`, `tolerations` or +`priorityClassName`, and nothing could supply them — which made the image flags (now the one +`--external-db-image`) unusable for the case they exist for. An operator who must mirror the tooling image into their own registry has a +registry that needs credentials, so the workload sat in `ImagePullBackOff` until the readiness +bound expired, and the only diagnosis available named a condition the operator had no input to +fix. The same held for a namespace whose capable nodes are tainted, or which admits a pod only +under a named service account or priority class. + +Each field is absent from the manifest unless its flag is supplied, so a deployment that needs +none of this gets exactly the pod it got before (FR-015). A malformed value is refused before any +object is created, by a message naming the flag — the same reason `--external-db-scratch-size` is +parsed by this tool rather than by the API server. + +Unlike the image flag, these follow the binding convention and are environment-settable. FR-027's +reason for keeping the image out of ambient environment does not reach them: the danger there is +that the tool holds workload-creation rights, so an environment-settable image turns those rights +into a means of running arbitrary code in the customer's namespace. These say where a pod may be +placed and which credentials fetch its image, never what it runs, and the image itself stays +the release's own or flag-only. A named service account grants the pod no API access either — +`automountServiceAccountToken` stays `false`, so what an account carries here is its registry +credentials and whatever the namespace's admission requires. An unattended backup is configured by +environment at least as often as by argv, and a scheduled run in a mirrored-registry cluster with +no way to name its pull secret is the defect restated rather than fixed. + +**T164 and T165 replaced the two image flags, and this is the change of record.** +`--external-db-image-neo4j` and `--external-db-image-postgres` selected the vendor images +`neo4j:2025.10.1-enterprise` and `postgres:18-alpine`. The transient workload now runs the +`infrahub-backup` image, which carries `neo4j-admin`, `cypher-shell` and the PostgreSQL 18 client +(FR-030, ADR 0017), so one flag, `--external-db-image`, selects it for both databases. Its default +is the image the release workflow publishes for the running binary's version: a binary built from +tag `v2.3.0` runs `registry.opsmill.io/opsmill/infrahub-backup:2.3.0`. A development build — a +version that is empty, a commit hash, `git describe` output, or carries build metadata — has no +published image and so no default; the external path refuses it before creating or deleting +anything unless `--external-db-image` is given (FR-031). The two old flags were never released, so +they were removed rather than deprecated. An air-gapped operator now mirrors one image, the one +they already run. + +`--external-db-tls-policy` was registered by T001 and removed by T095, which recorded that T041 +would reintroduce it alongside the channel it configures. T041 landed that channel and **did not** +reintroduce it. The channel takes no policy name: `cypher-shell`'s verification is selected by the +Bolt URI scheme (`+s` verifies, `+ssc` accepts a self-signed certificate) and `psql`/`pg_dump`'s by +`PGSSLMODE` (`verify-full` or `require`), and in both the decision this tool has to make is +two-valued — verify, or the operator's explicit opt-out. `--external-db-insecure-tls` above carries +exactly that, and a string naming a policy would have had nothing left to carry. The removal +therefore stands; anything finer-grained is a change to this contract rather than a restoration. + +**T100 widened what the opt-out covers, and this is the change of record.** The Neo4j channel was +encrypted only when the deployment exported `INFRAHUB_DB_TLS_ENABLED=true` — but a deployment that +leaves that setting at Infrahub's own default exports nothing, so the common observation is +*absent*, and absent sent this tool's own database password out of the cluster in clear with no +flag able to prevent it. Encryption is now the default for both databases, and +`--external-db-insecure-tls` is the one input that turns it off: with the flag, the channel is the +weakest one the deployment's own settings describe — unverified where the deployment declares TLS, +unencrypted where it does not. So an external Neo4j with no TLS at all is still reachable, but +reaching it takes the operator saying so. The decision stays two-valued, so no new flag is +introduced and `--external-db-tls-policy` stays removed. + +**Interaction rules**: + +- `--force` MUST NOT imply `--allow-external-restore`, and enabling scheduled restore MUST NOT + imply it either (FR-009). Any change making one imply the other is a breaking change to this + contract. +- Unattended restore MUST read the authorisation from configuration, not from an interactive + prompt (FR-010). The flag form exists for interactive use. +- Address overrides apply per database. Overriding one MUST NOT change resolution of the other + (FR-001). + +## Configuration keys + +Every flag above has an `INFRAHUB_`-prefixed environment equivalent, per the existing binding +convention — **except `--external-db-image`**, which stays the sole exception (the placement flags +added by T111 are bound; see the reasoning above). It resolves from its default or an explicitly +supplied flag only, never from ambient environment (FR-027): the tool holds workload-creation +rights, so an environment-settable image would turn those rights into a means of running arbitrary +images in the customer's namespace. + +The authorisation key is the one that matters for unattended operation: + +```text +INFRAHUB_ALLOW_EXTERNAL_RESTORE=true +``` + +Its presence in a scheduled restore's configuration is the only way an unattended destructive +restore into an externally-managed database can occur. + +## Consumed environment (read from in-deployment components, not from the operator) + +| Variable | Used for | +|---|---| +| `INFRAHUB_DB_ADDRESS` | Host list — comma-separated `host[:port]` | +| `INFRAHUB_DB_PORT` | Client (Bolt) port; **not** the backup port | +| `INFRAHUB_DB_PROTOCOL` | `bolt` or `neo4j` | +| `INFRAHUB_DB_DATABASE`, `_USERNAME`, `_PASSWORD` | Already read today | +| `INFRAHUB_DB_TLS_ENABLED`, `_TLS_INSECURE`, `_TLS_CA_FILE` | Client connection TLS | +| Prefect connection URL | PostgreSQL host, port, database, user, password | + +Reading these is additive: a deployment that does not set them behaves exactly as today. + +## Exit behaviour + +| Condition | Exit | Requirement | +|---|---|---| +| Backup complete, all endpoints reached | 0 | | +| Backup produced an artifact but some endpoints were unreachable | non-zero, partial artifact removed from every location it reached | FR-012 | +| Deployment query failed (permission, unreachable control plane) | non-zero, **never** treated as external | FR-002 | +| Client tooling older than the server | non-zero, before any data is read | FR-006 | +| Unsupported edition for external capture | non-zero, naming the mechanism limitation | FR-008 | +| External restore attempted without authorisation | non-zero, database untouched | FR-009 | +| Restore artifact unreachable by the server | non-zero, database untouched | FR-007 | +| Restore accepted but database did not reach a usable state | non-zero | FR-021 | +| Restore accepted and the database's state can no longer be read | non-zero well before the operation bound, attributed to the path to the database rather than to the database | FR-021, FR-025 | +| An external database stops responding mid-operation | non-zero at the bound, Infrahub returned to prior scale | FR-025 | +| Restricted pod-security namespace rejects the workload | non-zero, naming the security context requirement | FR-027 | +| Development build, an external database, and no `--external-db-image` | non-zero, before any workload is created or deleted and before any Infrahub service is stopped | FR-031 | + +The tool MUST NOT surface a database vendor's exit code directly where that code conflates +outcomes — notably a backup exit status that means either failure or partial success (research R6). + +## Failure message contract + +Each message names the condition, the specific resource, and the operator action. Wording is +illustrative; the three required elements are not. + +| Condition | Must name | +|---|---| +| No container and no discoverable endpoint | that discovery was attempted, where it looked, and the override flag | +| No container and the deployment's own settings could not be read | that the settings were *not* read and so what they record is unknown, the cluster's own reason, and both remedies (the override flag, or restoring access) — explicitly *not* that no setting records an address (FR-019, distinct from the row above) | +| Deployment query denied | that the query failed and which permission is missing — explicitly *not* that the database is absent | +| Missing workload-creation permission | the verbs and resource required | +| External Community edition | that the Community mechanism requires direct access to the database's own storage, and is therefore unavailable remotely — not "service not found" | +| Version incompatibility | both versions, and which one must change | +| Missing authorisation for external restore | the authorisation, its flag and its configuration key | +| Server cannot read the artifact | the URI attempted and the server-side prerequisite | +| Restore did not come online | the database's reported status, and where the server logs the cause | +| The restored database's state cannot be read at all | that the failure is in the path to the database (the transient pod, the cluster API, the exec permission) and not a verdict on the database, which may still be loading — explicitly *not* the row above's "the database is not usable", and not the server's own logs | +| `infrahub-collect --include-backup` against an external database | that the capture needs a transient pod and secret in the namespace, that collection creates no workload (ADR-0003), and that `infrahub-backup create` performs it | +| Development build with no `--external-db-image` | that reaching the database needs a transient workload, that a development build has no published image for it, `--external-db-image` and a released `infrahub-backup` as the two remedies, and that nothing was created or stopped (FR-031) | + +## Backwards compatibility + +For a deployment whose databases are all internal, the entire surface above is inert: no new flag +is required, no new environment variable is consulted to a different effect, and behaviour and +artifact layout are unchanged (FR-015). New flags are additive and default to the current +behaviour. No documented deployment service name is added or renamed (FR-017) — the constitution +treats those names as a public contract, so the transient workload registers under an existing one. diff --git a/specs/archive/007-external-database-backup/critiques/critique-20260902-1330.md b/specs/archive/007-external-database-backup/critiques/critique-20260902-1330.md new file mode 100644 index 00000000..df2a2838 --- /dev/null +++ b/specs/archive/007-external-database-backup/critiques/critique-20260902-1330.md @@ -0,0 +1,303 @@ +# Critique: External Database Backup and Restore + +**Feature**: 007-external-database-backup | **Date**: 2026-09-02 +**Artifacts reviewed**: spec.md, plan.md, research.md, data-model.md, contracts/, quickstart.md +**Constitution**: v1.1.0 +**Note**: the `critique` extension is registered in `.specify/extensions.yml` but ships no +`critique-template.md`; this report follows the structure the command prescribes. + +## Executive Summary + +The design is sound and the problem statement is now accurate — Phase 0 confirmed the transient-pod +approach is the database vendor's own documented topology, and it already overturned two of the +author's assumptions, which is evidence the research was doing real work rather than confirming a +preference. + +The weaknesses are not in the architecture. They are in **the seams between this feature and code +that already exists**. Three of the seven must-address findings are cases where a new requirement +cannot actually be satisfied by the module that owns the behaviour (retention, metadata versioning, +command timeouts), and were verified against the source rather than inferred. One is a security +surface the spec creates and does not mention. One is a genuine single-point-of-failure in the half +of the round trip the feature owner explicitly insisted on. + +None require redesign. All are additive requirements or bounded decisions. + +**Verdict: ⚠️ PROCEED WITH UPDATES** + +--- + +## Product Lens Findings + +### 3a. Problem Validation + +**P1 💡** — The spec's evidence for demand is the phrase "several customers". No count, no accounts, +no ticket references. That is thin for 23 functional requirements, a new destructive operation, and +a new cluster permission requirement. More pointedly: the single most load-bearing assumption in the +feature — that customers will grant pod-creation rights (FR-019) — was resolved by assumption, and +the customers who could answer it definitively are identifiable and already asking for the feature. + +*Suggestion*: before implementation starts, confirm the permission question with one named customer. +It is a single question, and a "no" would promote User Story 3 to P1 and change the task breakdown +substantially. This is the cheapest de-risking available. + +### 3b. User Value Assessment + +**P2 💡** — The tool already ships `environment detect` and `environment list`. An operator whose +backup just failed will run `environment detect` first, and the spec says nothing about what it +reports for a deployment with an external database. Today it would presumably report the environment +as found while the thing the operator cares about — that the database is not in it — goes unmentioned. +User Story 2 exists to make this population diagnosable, and it addresses the backup path only. + +*Suggestion*: add an acceptance scenario to User Story 2 covering `environment detect` reporting +external database locations. Cheap, and it meets operators where they already look. + +The MVP shape is otherwise right: US1 is one indivisible slice because a round trip that does not +close has no value, and US2/US3 are correctly deferred. + +### 3c. Alternative Approaches + +**P3 💡** — Research R1 rejected `--to-path` writing directly to cloud storage in order to preserve +the Plakar pipeline. Correct for feature parity, but it means a customer who wants only "a backup in +our bucket" is being routed through substantially more machinery than they need. The rejection is +recorded in research and invisible to users. + +*Suggestion*: document it as a supported manual escape hatch. Costs nothing, and it gives support a +one-line answer for customers who cannot meet the seed prerequisites. + +### 3d. Edge Cases & UX + +**P4 💡** — FR-012 keeps incomplete artifacts out of restore-point selection, and scheduled restore +selects the latest backup. Together these mean a scheduled restore can silently use an *older* +recovery point than the operator expects, with no signal that it skipped one. + +*Suggestion*: require that skipping an incomplete artifact during automatic selection is reported, +not merely done. + +Accessibility and localisation are not applicable to this surface. Error, empty and failure states +are unusually well covered by the CLI contract's wording table. + +### 3e. Success Measurement + +**P5 💡** — SC-001–008 are measurable but all are pre-release acceptance gates. There is no +post-launch signal and no rollback trigger. For a backup feature the obvious signal is whether +customers' scheduled backups are completing, and the obvious rollback trigger is any artifact that +fails at restore. + +*Suggestion*: add one post-launch criterion and name a rollback trigger. Also note SC-003 is phrased +as a property of the test suite ("no test case produces…") rather than a user outcome. + +--- + +## Engineering Lens Findings + +### 4a. Architecture Soundness + +**E6 🎯** — Pod resolution falls back to **substring matching on pod names** +(`environment_kubernetes.go:305-313`, and the same fallback in `getAllPodsWith`). The transient pod +must contain the canonical service name for that fallback to find it — but the same property makes it +visible to every other caller of that resolver. `GetAllPods` and `ServiceReplicas` feed replica +counting and the `infrahub-collect` bundle, so while the pod exists it would be reported as a +replica of the `database` service. A troubleshooting bundle collected during a backup would describe +a database replica that is really a backup client. + +*Suggestion*: make the transient pod's name and labels explicitly excluded from the shared resolvers, +and add a unit test asserting `GetAllPods` does not return it. Registering it in the cache under a +canonical name must not mean it is discoverable *as* that service. + +*Resolved (note added 2026-10-05)*: the line numbers above describe the code as of this critique. +The name-substring fallback now lives in `podsForServiceWith` and `claimedPodNamesWith` +(`environment_kubernetes.go`), and the transient pod is excluded from every pod listing by +`parseLabelledPods` (`kubernetes_ownership.go`), which applies `withoutTransientPods` (FR-028). +`TestGetAllPodsExcludesTheTransientWorkload` asserts it. + +### 4b. Failure Mode Analysis + +**E1 🎯** — **The remote backup command has no timeout.** The 30-minute guard +(`defaultStreamIdleTimeout`, `backup_neo4j.go:108`) protects only the *data stream*. For the +Enterprise path the backup command is deliberately run as a separate step before any streaming +("Run backup command separately so its stdout logs don't contaminate the data stream", +`backup_neo4j.go:40`), and that step goes through `iops.Exec`, which uses the **unbounded** +executor — the code comments in `environment_kubernetes.go` say so explicitly, contrasting it with +the timeout-bounded runner the collect primitives use. + +In-cluster this was tolerable: the database was a pod one had other ways to observe. An external +database that accepts the connection and then stalls — a checkpoint on a very large store, a network +partition mid-transfer, a firewall dropping an idle connection — hangs the tool indefinitely, with +Infrahub possibly still scaled down if this happens on the restore path. Research R1 flagged +checkpointing as a known cause of backup stalls. + +*Suggestion*: bound the remote capture with a timeout, defaulting generously and overridable, and +reuse the existing bounded runner rather than adding a mechanism. This is the finding most likely to +produce a support incident. + +**E4 🎯** — The restore sequence replaces the database Infrahub connects to, while Infrahub is +scaled down and then scaled back up. Two things are unspecified: whether a scale-down completes +before the replace begins (task workers retry, and termination is not instantaneous), and whether +scaling back up is sufficient for Infrahub to bind to a *recreated* database rather than a restarted +one. If it is not, the restore reports success and leaves a non-functional instance — a Principle II +failure mode arriving through the back door. + +*Suggestion*: specify the ordering as a requirement (quiesce confirmed before the destructive step), +and add an acceptance scenario asserting Infrahub serves data after scale-up without manual +intervention. Quickstart Scenario 3 asserts the data is served but not that no manual step was needed. + +**E2 🎯** — **FR-012 cannot be satisfied by retention as it exists.** Retention decides entirely +from artifact *names*: `parseBackupName`, `backupTimestamp`, `sortBackupRefsNewestFirst`, and +`retentionKeeps(ref, rank, policy, now)` where Count "keeps the Count most recent backups at a +storage location" by rank (`retention.go:238-310`). It never opens metadata for any decision. + +So an artifact marked `capture_complete: false` still occupies a rank slot, and under a Count policy +it **evicts a good backup** while itself being unusable as a restore point. The requirement as +written silently assumes retention is metadata-aware. It is not, and making it so is a change to the +largest and most heavily tested module in the codebase (919 lines, 2123 lines of tests). + +*Suggestion*: decide explicitly between (a) teaching retention to read metadata, (b) encoding +incompleteness in the artifact name so the existing name-based logic can act on it, or (c) never +retaining an incomplete artifact at all — write it, report it, and delete it. Option (c) is the +smallest change and the safest default; (b) is a naming-contract change. This decision belongs in the +spec before tasks are generated, because it determines whether `retention.go` is touched at all. + +### 4c. Security & Privacy Review + +**E5 🎯** — Two problems in one place, both created by this spec. + +First, the transient pod's image is operator-declared (FR-004) and, per the binding convention in +the CLI contract, settable by environment variable. Combined with the pod-creation rights the feature +requires, that is a mechanism for running an **arbitrary container image in the customer's namespace** +by influencing the tool's environment. The threat model is not stated anywhere. + +Second, the pod spec has no `securityContext`. Hardened clusters enforcing restricted Pod Security +Standards will reject a pod that does not declare `runAsNonRoot`, drop capabilities, and disallow +privilege escalation — and hardened, compliance-driven estates are precisely the population that +runs its databases outside the application cluster. The feature could be unusable for its own target +audience. Note also that the vendor documents the backup command as needing to run as the database's +own user, so the security context and that requirement must be reconciled rather than assumed +compatible. + +*Suggestion*: add a requirement that images resolve from an allowed registry or a pinned default with +the override requiring an explicit flag rather than ambient environment; and specify the pod's +security context, validating against a restricted-PSA namespace in the end-to-end suite. + +Also worth stating (💡): credentials for an externally-managed database now pass through the tool's +own process and into the cluster API, where previously they never left the database pod. FR-014 and +FR-023 cover storage and lifetime but the widened trust boundary is not acknowledged. + +TLS deserves a mention too (💡): Infrahub's config carries `INFRAHUB_DB_TLS_INSECURE`, and if the +tool inherits it, credentials cross the network without certificate verification. + +### 4d. Performance & Scalability + +No must-address items. FR-022 now covers scratch sizing with a queried size and headroom, which was +the main exposure. Two notes: the headroom factor is unquantified, and the capture adds a full +database-sized transfer *inside* the cluster that did not exist before, which is worth stating for +capacity planning in large deployments. + +### 4e. Testing Strategy + +**E7 💡** — The plan describes the end-to-end work as "a fixture change — where the database runs and +what the application's environment says". That understates it: the suite must stand up a database +**outside** the cluster under test, reachable from inside it, with an Enterprise licence, plus an +object store the database itself can reach. That is new CI topology, not a fixture parameter. +Principle IV makes this non-optional, so under-estimating it risks the requirement being quietly +dropped. + +*Suggestion*: treat the CI topology as its own task with its own estimate. + +The two-concurrent-runs scenario also has no obvious home in the current suite. + +### 4f. Operational Readiness + +**E3 🎯** — `BackupMetadata` carries `MetadataVersion` (currently `2025111200`), and Principle II +requires restore to "refuse incompatible versions unless the user explicitly overrides". The spec's +compatibility story (FR-016, artifact contract) covers only new-tool-reads-old-artifact. The reverse +is unaddressed and forks badly: + +- If adding fields bumps `metadataVersion`, an older binary **refuses** the new artifacts — so rolling + the tool back after an upgrade strands every backup taken in between. +- If it does not bump, an older binary ignores unknown JSON fields silently — including + `capture_complete` — and will therefore **restore an artifact this feature marked incomplete**. + +*Suggestion*: choose deliberately and record it. The defensible choice is not to bump, and to encode +incompleteness where an old tool cannot miss it — which converges with E2's option (b) or (c). +Whichever is chosen, add a test that pins the old-reads-new behaviour, because right now it is +accidental either way. + +Observability is otherwise adequate under Principle V, though nothing emits a signal an operator +could alert on for "external mode engaged" (💡). + +### 4g. Dependencies & Integration Risks + +**E8 💡** — ADR-0007 establishes offline-by-default, and its rationale states the general position: +"offline-by-default is the correct posture for an ops tool that runs in restricted networks." Its +*decision* is scoped to `infrahub-collect`, so this feature does not violate it — but it does cross +that posture for `infrahub-backup`, and unlike the benchmark's opt-in pull, this one cannot degrade +to "skipped" because it is load-bearing. ADR-0007 also supplies the pattern worth copying: an +overridable image variable so air-gapped-with-private-registry keeps working, which FR-004 already +provides. + +*Suggestion*: record an ADR for the posture change. It is the kind of decision a future maintainer +will otherwise read as an oversight. + +The dependency on the AWS command-line tool being present on customer database servers has no +version constraint stated (💡). + +--- + +## Cross-Lens Insights + +**X1 🎯 — Restore is a single mechanism whose prerequisites are least likely to hold for the target +population, and both fallbacks were explicitly rejected.** + +The product lens and engineering lens converge hard here. Remote restore requires the cloud seed +provider, which requires the AWS command-line tool installed and credentials configured **on every +Neo4j server** (research R2). The customers who put their database outside the application cluster +are disproportionately those with locked-down, centrally-administered database estates — the ones +least able to add tooling and credentials to a database host. + +If that prerequisite cannot be met, the round trip does not close. And the spec has already rejected +both alternatives: operator-staged artifacts (breaks the no-manual-steps promise) and +PostgreSQL-only restore (different recovery points). Research R2 noted `dbms.recreateDatabase()` as +"a genuine second option… worth revisiting if the seed prerequisites prove onerous" — with no +criterion for when that revisit is triggered. + +Every other risk in this feature degrades to a clear error. This one degrades to "the feature does +not work for you", discovered after the backup half has been built and shipped. + +*Suggestion*: validate the seed prerequisite against a real target customer's database estate before +implementing the restore half, and record a decision criterion for falling back to the recreate +procedure. This is the same de-risking action as P1 and can be the same conversation. + +--- + +## Findings Summary + +| ID | Lens | Severity | Category | Finding | Suggestion | +|----|------|----------|----------|---------|------------| +| E1 | Engineering | 🎯 | Failure Modes | Remote capture runs on the unbounded executor; the 30-min guard covers only the data stream, not the backup command | Bound the capture with an overridable timeout using the existing bounded runner | +| E2 | Engineering | 🎯 | Architecture | Retention decides from names and ranks only; an incomplete artifact consumes a keep slot and evicts a good backup, so FR-012 is unsatisfiable as written | Choose: metadata-aware retention, name-encoded incompleteness, or never retain an incomplete artifact | +| E3 | Engineering | 🎯 | Operational Readiness | `metadataVersion` interaction unspecified: bumping strands artifacts on rollback, not bumping lets an old tool restore an artifact marked incomplete | Decide deliberately, record it, pin old-reads-new with a test | +| E4 | Engineering | 🎯 | Failure Modes | Quiesce/replace ordering and post-replace reconnection unspecified; restore could report success on a broken instance | Require confirmed quiesce before the destructive step; assert no manual step after scale-up | +| E5 | Engineering | 🎯 | Security | Environment-settable image = arbitrary image execution in the customer namespace; no securityContext, so restricted PSA clusters may reject the pod | Constrain image sourcing; specify securityContext; test against restricted PSA | +| E6 | Engineering | 🎯 | Architecture | Substring pod resolution makes the transient pod visible as a `database` replica to shared resolvers and to collect bundles | Exclude it from shared resolvers; unit-test that `GetAllPods` omits it | +| X1 | Both | 🎯 | Value × Risk | Restore depends on a prerequisite the target population is least likely to satisfy, with both fallbacks already rejected | Validate against a real customer estate before building restore; record a fallback criterion | +| P1 | Product | 💡 | Problem Validation | Demand is asserted as "several customers"; the load-bearing permission assumption is unvalidated | Confirm with one named customer before implementation | +| P2 | Product | 💡 | User Value | `environment detect` — the operator's first command — is not addressed for external databases | Add a US2 acceptance scenario | +| P3 | Product | 💡 | Alternatives | Direct-to-cloud capture rejected in research and invisible to users | Document as a supported escape hatch | +| P4 | Product | 💡 | Edge Cases | Automatic selection can silently move the recovery point when it skips an incomplete artifact | Require the skip to be reported | +| P5 | Product | 💡 | Success Measurement | No post-launch signal, no rollback trigger; SC-003 is a test-suite property | Add one post-launch criterion and a rollback trigger | +| E7 | Engineering | 💡 | Testing | "Fixture change" understates new CI topology: external DB reachable from the cluster, Enterprise licence, reachable object store | Treat CI topology as its own task | +| E8 | Engineering | 💡 | Dependencies | Crosses ADR-0007's offline-by-default posture for a pull that cannot degrade to skipped | Record an ADR | +| E9 | Engineering | 💡 | Security | Trust boundary widened: credentials now traverse the tool and the cluster API; TLS-insecure may be inherited | Acknowledge in the spec; require certificate verification or an explicit opt-out | +| E10 | Engineering | 💡 | Performance | Headroom factor unquantified; capture adds a full database-sized in-cluster transfer | State both for capacity planning | + +**Totals**: 7 must-address, 9 recommendations, 0 open questions requiring stakeholder input beyond +P1/X1 (which are the same conversation). + +## Verdict + +⚠️ **PROCEED WITH UPDATES** + +No finding requires redesign. E1, E4, E5 and E6 are additive requirements. E2 and E3 are bounded +decisions that converge on a single choice. X1 and P1 are one customer conversation that should +happen in parallel with implementing the backup half — it does not block starting, but it should +block *finishing* the restore half. diff --git a/specs/archive/007-external-database-backup/data-model.md b/specs/archive/007-external-database-backup/data-model.md new file mode 100644 index 00000000..91496f9d --- /dev/null +++ b/specs/archive/007-external-database-backup/data-model.md @@ -0,0 +1,108 @@ +# Phase 1 Data Model: External Database Backup and Restore + +**Feature**: 007-external-database-backup | **Date**: 2026-09-02 + +This feature adds no persistent storage. The entities below are configuration resolved at run +time, one transient cluster object, and additive fields on the existing backup metadata. + +--- + +## `DatabaseEndpoint` (new, in-memory) + +Where a logical database actually lives, resolved once per run per database and thereafter the +single source of truth for both the backup and restore paths. + +| Field | Type | Source | Validation | +|---|---|---|---| +| `Service` | string | Caller | One of the documented deployment service names (`database`, `task-manager-db`). Never a new name (FR-017). | +| `Location` | enum `internal` \| `external` | Resolved | `external` only on a positively-empty container match; a query failure is an error, never `external` (FR-002). | +| `Hosts` | `[]HostPort` | `INFRAHUB_DB_ADDRESS`, or the Prefect connection URL, or flag override | At least one entry when `Location` is `external`. Order is significant — it is the order the backup command will try (FR-005). | +| `ClientPort` | int | `INFRAHUB_DB_PORT` (Bolt, default 7687) / connection URL | Used for version probing and for restore. | +| `BackupPort` | int | Defaulted to 6362; flag override | Not discoverable — Infrahub does not record it. | +| `Protocol` | string | `INFRAHUB_DB_PROTOCOL` | `bolt` or `neo4j`; the routed form is required for multi-member deployments. | +| `TLS` | `TLSSettings` | `INFRAHUB_DB_TLS_*` | Applies to the client connection, independently of the backup channel's own policy. | +| `Database` | string | `INFRAHUB_DB_DATABASE` / connection URL | Names the artifact entry, so it must be carried through rather than assumed (a cross-name restore trap already fixed once in this codebase). | +| `Credentials` | reference | Existing credential resolution | Never rendered into a command line or into workload configuration (FR-014). | + +`HostPort` is `{Host string; Port int}`, where a member given without an explicit port inherits +`ClientPort` — mirroring how Infrahub itself interprets its address list. + +**Lifecycle**: resolved during environment detection, immutable for the rest of the run, and +reported before any connection is opened (FR-003). + +--- + +## `TransientWorkload` (new, one cluster object per run per external database) + +The execution context that stands in for the absent database container. It exists so that every +existing execution, streaming and copy primitive continues to work unchanged. + +| Field | Type | Constraint | +|---|---|---| +| `RunID` | string | Unique per run. Labelled onto the workload so a later run can identify strays (FR-011). | +| `Service` | string | The canonical service name it stands in for, so it resolves under the existing name (FR-017). | +| `Image` | string | Pinned default, overridable by explicit flag only — never from ambient environment (FR-004, FR-027). | +| `SecurityContext` | fixed | Must satisfy a restricted pod-security policy, reconciled with the vendor's requirement that the backup tooling run as the database's own user (FR-027). | +| `Deadline` | duration | Set on the object itself, so the cluster reclaims it if the tool never returns (FR-011). | +| `ScratchSize` | size | At least the size of the database being captured (FR-022). | +| `CredentialRef` | reference | A secret **owned by** the workload — its lifetime is bound to the workload's, so the cluster's garbage collector reclaims it when the workload goes (FR-023). Same `RunID` label. | +| `State` | enum | `pending` → `ready` → `released`. Only `ready` may be used as an execution target. | + +**Invariants**: + +- Never outlives its run on any exit path, including abnormal termination (FR-011), and neither + does the credential object it owns (FR-023) — deleting the workload is sufficient to remove + both, so no cleanup step depends on the tool still being alive. +- Adopted by exactly one run — a concurrent run must not use another's workload. +- Registered under the canonical service name so that execution resolves to it transparently — but + **not** discoverable as a replica of that service. Service enumeration, replica counts and + diagnostic collection must not return it (FR-028); the shared resolvers fall back to substring + matching on names, so this exclusion has to be explicit rather than assumed. + +--- + +## `BackupMetadata` (existing — additive fields only) + +Additive by requirement: artifacts from previously released versions must remain restorable +without operator action (FR-016), and the constitution's backup-engine clause makes any +non-additive change a documented migration. + +| Field | Type | Purpose | +|---|---|---| +| `source_endpoints` | ordered list, per database | The endpoints supplied, in the order they would be tried (FR-005, SC-004). Not a claim about which one served the capture — that is not knowable. | +| `source_roles` | map endpoint → role, per database | Each member's role as observed at capture time: leader, follower, or unknown. `unknown` is a real value, distinct from absent. | +| `capture_complete` | bool | False when the operation reported problems even though it produced an artifact. Such an artifact is removed rather than retained (FR-012), so this field is defence in depth for the case where removal itself fails — not an input to retention, which never reads metadata. | +| `external_location` | bool, per database | Whether this database was captured externally. Diagnostic, and it explains an artifact whose source endpoint is outside the deployment. | + +**Compatibility rule**: a reader encountering none of these fields treats the artifact as an +internal, complete capture of unknown endpoint — which is exactly what a pre-feature artifact is. + +--- + +## `RestoreAuthorization` (new, configuration) + +| Field | Type | Constraint | +|---|---|---| +| `ExternalRestoreAllowed` | bool | Distinct from the existing restore override and from the scheduled-restore toggle; neither confers it (FR-009). | +| `Source` | enum | `flag` for interactive, `config` for unattended. Unattended restore requires the configuration form (FR-010). | + +**State transition**: absent → refuse and name the authorisation required. Present → proceed to +pre-flight (FR-006, FR-007), then to the destructive step, then to confirmation (FR-021). There is +no path from absent to a destructive step. + +--- + +## Entity relationships + +```text +Run + ├── DatabaseEndpoint (one per database; internal or external) + │ └── TransientWorkload (only when external; 1:1, same RunID) + │ └── CredentialRef (1:1, same RunID) + ├── RestoreAuthorization (restore only) + └── BackupMetadata (one per artifact; carries per-database source facts) +``` + +`TransientWorkload` exists only for `external` endpoints. That is the whole shape of the feature: +an internal endpoint already has a container, and an external one is given a stand-in so the +layers above cannot tell the difference. diff --git a/specs/archive/007-external-database-backup/opsmill-implement-report.md b/specs/archive/007-external-database-backup/opsmill-implement-report.md new file mode 100644 index 00000000..d2088b00 --- /dev/null +++ b/specs/archive/007-external-database-backup/opsmill-implement-report.md @@ -0,0 +1,538 @@ +# Implementation Report: External Database Backup and Restore + +**Status**: **INCOMPLETE** — 142 of 161 tasks done, 1 descoped; 18 open. **The Kubernetes path has now executed end to end on a real cluster and the round trip passed (§12)**; what is open is 8 e2e/`[INFRA]` tasks, 4 residual ledgers (T152–T155) and 6 findings from that run (T156–T161), three of them HIGH +work gated on CI topology, each with a named reason and owner + +**Feature**: 007-external-database-backup +**Spec dir**: `specs/007-external-database-backup` +**Branch**: `007-external-database-backup` — **not pushed, no PR** +**Base commit (run 3)**: `080917d` +**Head commit**: `5c0a3bf` (last code commit; this report's own commit follows it) +**Commits this run**: 29 +**Wall clock**: ~08:45Z → ~16:40Z, 2026-09-03 (~8h) + +| | Run 1 | Run 2 | Run 3 (this run) | +|---|---|---|---| +| Base → head | `e9008b4` → `c9da687` | `546ccd7` → `b470c67` | `080917d` → `a34c5ad` | +| Commits | 9 | 30 | 29 | +| Tasks closed | 41 (8 later reverted) | 34 | 61 | +| Go tests | 862 → 888 | 888 → 1197 | 1197 → **1450** | +| Review findings | ~35 | ~24 | **21** | +| Outcome | **reset** — Phase 3 reverted | remediation + rebuild | Phase 2T closed, Phase 3–5 landed, third review fixed | + +--- + +## 0. What changed after the first report was written + +The run continued past the report below, and three things in it are now out of date: + +1. **T121 was taken out of scope, and T107 closed** — on FR-014's own wording, which is scoped to an + *externally-managed* database and already satisfied. The internal argv exposure is real but + pre-dates the branch (all 10 sites present unchanged at base `8186fcf`). See §8. +2. **T122, T119 and T141 landed**, closing every task that was actionable without CI topology. +3. **The feature's mechanism was executed against real Neo4j Enterprise servers for the first time** + (Phase 2W, §9). That retired most of T073's risk, found one critical defect, corrected two + documentation claims, and proved T072's topology can be built on a laptop. + +Open tasks after these items were **8**, all of them e2e or `[INFRA]`. The residual ledgers +(T152–T155) and the findings from the cluster run (T156–T161) were added afterwards, which brings +the total to the **18** stated in the header. + +--- + +## 1. Why this is still INCOMPLETE + +**8 tasks are open.** None is unexplained, and none is silent. They fall into two groups: + +- **6 e2e tasks blocked on CI topology that does not exist** (T054–T058, T064). Gated on **T072**: no CI + topology has a database outside the cluster under test with an Enterprise licence and an object + store the database itself can reach. The suites are **written, collectable, and skip by name** on + `INFRAHUB_TESTING_EXTERNAL_DB`, with skip reasons that name the missing topology and the task. +- **2 `[INFRA]` tasks that cannot be done on a developer machine** (T072, T073). T073 is the one that + matters most: **the Neo4j seed-from-URI mechanism has never been exercised against a real + Enterprise server.** It is implemented as designed and unit-tested against fakes; that is not the + same thing, and the report does not claim otherwise. +- **5 enabling tasks discovered during this run** (T107, T119, T121, T122, T141), each recorded with + the reason it was not folded into the chunk that found it. + +**The k8s e2e leg cannot run on this machine at all** — those suites need `vcluster`, which is not +installed. Only the Docker leg is locally executable. An earlier read that minikube made them +runnable was wrong and is corrected here. + +--- + +## 2. Chunk ledger + +Nine planned chunks, then three unplanned fix passes. **No two implementation subagents ran +concurrently.** Every chunk ran in a clean-context subagent; the orchestrator edited no feature code +except three documentation falsehoods (§5) and four task records. + +| # | Chunk | Tasks | Outcome | Commits | +|---|---|---|---|---| +| 1 | 2T — Silently wrong or destructive | T097–T102 | 6 ✅ | `64bfdb8` | +| 2 | 2T — Wrong answer, wrong place | T103–T110 | 7 ✅, 1 ⚠️ (T107) | `b83099a` | +| 3 | 2T — Operability code defects | T111–T116 | 6 ✅ | `8a5c4a6` | +| 4 | 2T — Structural dedup + governance | T117–T120 | 3 ✅, 1 ⚠️ (T119) | `7a98676` | +| — | orchestrator: record T121 | — | — | `c8ea8d6` | +| — | orchestrator: record T122 | — | — | `dbfad53` | +| 5 | Phase 3 — Restore path | T046–T053 | 8 ✅ | `7f0bf1d` | +| 6 | Phase 3 — End-to-end | T054–T058 | 5 ⚠️ (topology) | `58f0d91`…`24d27ac` | +| — | **T123 regression fix** (found by chunk 6) | T123 | ✅ | `a181b27` | +| — | orchestrator: gitignore e2e residue | — | — | `5ce2a1f` | +| 7 | Phase 4 — US2 diagnostics | T059–T064 | 5 ✅, 1 ⚠️ (T064) | `4f36c21`, `936a3b6` | +| 8 | Phase 5 — Documentation | T065–T068 | 4 ✅ | `e4bfc2c`, `b018b33` | +| 9 | Phase 5 — Hygiene + T071 audit | T069–T071, T074 | 4 ✅ | `9b3f4b6`, `2e1dd23` | +| — | orchestrator: doc falsehoods | T126–T128 | ✅ | `a65d62a` | +| A | Review fixes — critical + restore safety | T124–T130 | 7 ✅ | `cca32bb`, `25b741f` | +| B | Review fixes — test gaps, dedup, invariants | T131–T141 | 10 ✅, 1 ⚠️ | `42f359b`, `ea943b6`, `74f28e4`, `0cfa614`, `43df4c4`, `a34c5ad` | + +### Decisions and surprises flagged upward by chunks + +- **Chunk 2 refused to half-fix T107.** Its gate half was already done; the argv half needs a + stdin-carrying exec primitive neither backend has, because `ExecOptions.Env` renders as an + `env KEY=VALUE` argv prefix — exactly what FR-014 forbids. Twelve in-deployment sites pass the + Neo4j password on argv. Recorded as **T121**; T107 stays open on it. Verified independently. +- **Chunk 4 refused T119's fifth clause.** Dropping `getPodStatusesWith`'s selector queries is not a + dedup: the namespace listing supersedes them only where they matched *nothing*, so a naive removal + would let `IsRunning` answer true from a claimed-but-unlabelled pod and delete the + undetermined-status reporting `stopAppContainers` depends on. Recorded as **T122**. +- **Chunk 5 found two live bugs while building.** `transientWorkloadSpec.validate()` rejected every + role but probe/capture, so **a restore workload could never have been created** — the restore path + was unreachable before it was written. And `restore --latest --s3` called the restore gate twice, + colliding on the pod name. +- **Chunk 6 found the T123 regression** (§5), which no unit test could have caught. +- **Chunk 7 caught its own design error before shipping it** — it first hung the location report on + `DetectEnvironment`, then found that is also step 1 of `create`, `restore`, both Plakar paths and + the taskmanager flushes, which would have put two pod listings on every flush. +- **Chunk 8 found seven places where spec/plan/research prose contradicted shipped reality** and + documented reality. Chunk-9 and the comment reviewer later confirmed 5 of the 7 already resolved + in shipped text; the remaining 2 were fixed in `a65d62a` and `43df4c4`. +- **Chunk 9 disproved a claim chunk 2 had made.** Chunk 2 reported that T103 already covered T074; + it had guarded the *ownership policy*, a different question from FR-028's exclusion. The client + side still tested the pod's *name*, so the guard did rest on `transientObjectPrefix`'s spelling. + Fixed, and the hazard proved latent before being closed. +- **Fix B found a production bug all six reviewers missed**: `initPlakarContext` never set + `KContext.CacheDir`, so kloset wrote repository state into the operator's **working directory** and + `--plakar-cache-dir` never reached it. This is the true cause of the "Plakar e2e leaves a repo in + the project root" hazard the orchestrator had papered over with a `.gitignore` entry. + +--- + +## 3. Tasks not completed + +| Task | Reason | Unblocked by | +|---|---|---| +| T054 | External-topology e2e fixtures written; never executed | T072 | +| T055 | 3 cases written, collect, skip by name | T072 | +| T056 | 3 cases written, collect, skip by name | T072 + seed store | +| T057 | 4 robustness cases written, collect, skip by name | T072 | +| T058 | Docker leg **ran** (35 passed); k8s leg needs `vcluster`; previous-release artifact unobtainable | `vcluster` + release artifact | +| T064 | 16 diagnostic cases written, collect, skip by name | T072 | +| T072 | `[INFRA]` — CI topology with an out-of-cluster Enterprise database + reachable object store | provisioning | +| T073 | `[INFRA]` — validate seed prerequisite against a real customer estate | customer conversation | +| T107 | FR-014's argv half; deliberately not half-fixed | T121 | +| T119 | Fifth clause is a running-check change, not a dedup | T122 | +| T121 | Stdin-carrying exec primitive on both backends + 12 call sites | — | +| T122 | Reproduce selector matches from the listing's label fields | — | +| T141 | `TestVerifySeedReadableRefusesAnUnreadableURI` runs 4s–125s on AWS SDK retry | — | + +--- + +## 4. Local-pass evidence + +**No `MISSING` rows.** Every chunk returned evidence; where a report truncated in transit, the +evidence was re-requested or read directly from the file the chunk wrote, and the orchestrator +verified the pass lines itself rather than accepting a summary. + +Rows are grouped per chunk rather than per test — ~130 tests were added and a 130-row table would +obscure rather than aid audit. Every row names its test identifiers and the file holding the +**verbatim** runner lines, all of which were read and checked. This grouping is an orchestrator +decision, recorded in §6. + +Common environment for all Go rows: repo `/Users/bkohler/automation/opsmill/infrahub-backup`, branch +`007-external-database-backup`, Darwin 25.6.0 / arm64, build tag `untested_go_version` required, +scoped to `./src/...`. No cluster, database, kubectl or Plakar repository needed — all table-driven +over injected runners and fakes. + +| Chunk / tests | Type | Run command | Passed at (ISO 8601) | Env | Verbatim pass lines | +|---|---|---|---|---|---| +| 1 (T097–T102) — `TestDeclaredServiceIn_ReadsTheKeyThatNamesAService`, `TestParseLabelledPods_ReadsTheKeyThatNamesAService`, `TestGetPodStatusesWith_HelmChartLabels`, `TestGetPodStatusesWith_QueryFailures`, `TestStopAppContainers_UndeterminedStatusFailsTheRun`, `TestTransientObjectNamesAreOnePerDatabase`, `TestBuildTransientPodManifestShape`, `TestResolveExternalTLS`, `TestDiscardPlakarComponentsRemovesWhatWasCommitted`, `TestRefuseIncompleteCaptureIsTheReaderOfTheField` (10 top-level, 18 subtests) | unit | `go test -tags untested_go_version -v -run '<10-test regex>' ./src/internal/app/` | 2026-09-03T09:05:40Z | n/a | `scratchpad/evidence.log` — 28 `--- PASS`, 0 FAIL, `ok infrahub-ops/src/internal/app 0.422s` | +| 2 (T103–T110) — 8 added incl. `TestResolutionAndLocationGiveOneAnswer`, `TestFindWorkloadResourceWithReadsPrefixesNamespaceWide`, `TestLocateServiceWithClaimsACloudNativePGCluster`, `TestPostgresClientEnvCarriesTrustMaterial`, `TestProbeExternalPostgresWalksPastASilentMember` | unit | `go test -tags untested_go_version -v -count=1 ./src/...` | 2026-09-03T11:33:24Z | n/a | `scratchpad/T103-T110-test-evidence.txt` — 381 top-level PASS, 0 FAIL, all 3 packages `ok` | +| 3 (T111–T116) — 5 added (`TestTransientPodPlacement`, `TestResolveTransientSchedulingRefusals`, `TestExternalDBSchedulingFlags`, `TestCaptureRolesWithTwoInstancesOnOneHost`, `TestUncontactableEvidenceIsAttributedToAnEndpoint`), 4 extended, 2 signature-modified; net +46 cases | unit | `go test -tags untested_go_version -count=1 -v ./src/...` | 2026-09-03T09:58:24Z | n/a | `scratchpad/chunk3-test-evidence.txt` — 386 top-level PASS, 0 FAIL | +| 4 (T117–T120) — 4 functions / 11 subtests incl. `TestCollectionRefusesAnExternalDatabaseCapture`, `TestStopAppContainersIssuesOneNamespaceListing` | unit | `go test -tags untested_go_version -count=1 -v ./src/...` | 2026-09-03T10:44:23Z | n/a | `scratchpad/chunk4-test-evidence.txt` — 391 top-level PASS, 0 FAIL | +| 5 (T046–T053) — 58 added incl. `TestRestoreNeo4jExternalSeedsAndConfirms`, `TestVerifySeedReadableRefusesAnUnreadableURI`, `TestNeo4jSeedStatements`, `TestConfirmAppContainersQuiesced`, `TestReturnAppContainersToScale` | unit | `go test -tags untested_go_version -count=1 -v ./src/...` | 2026-09-03T13:07Z | n/a | `scratchpad/chunk5-test-evidence.txt` — 408 top-level PASS, 0 FAIL | +| 6 (T054–T057) — 12 e2e cases across `test_k8s_external_neo4j.py`, `test_k8s_external_both.py` | **e2e** | `uv run pytest -v -m k8s tests/e2e` (CI) | **deferred — local e2e not supported** | needs out-of-cluster Enterprise DB + object store (T072); k8s leg needs `vcluster` | collect-only 10 collected exit 0; normal run 10 skipped, reason names `INFRAHUB_TESTING_EXTERNAL_DB` and T072 | +| 6 (T058) — existing Docker suites, unchanged, + 2 new compatibility cases | **e2e** | `uv run pytest -v -m "docker and not enterprise" tests/e2e` | 2026-09-03T~12:20Z | Docker 29.4.0; `-p no:pytest-infrahub-performance-test`; `INFRAHUB_TESTING_IMAGE_VER=1.11.0`; OrbStack `DOCKER_HOST` | tarball `5 passed in 482.27s`; s3 `2 passed`; collect+retention `23 passed, 2 deselected in 200.21s`; new compat `2 passed in 247.12s`; plakar `3 failed` → **fixed by `a181b27`**, re-run `3 passed, 1 xfailed in 519.64s` | +| T123 fix — `TestShippedRestoreCommandInvocationContract` (11 subtests) + `TestResolveConfiguration` | unit + e2e | `go test -tags untested_go_version -count=1 -run '…' ./src/cmd/… ./src/internal/app/…` | 2026-09-03T12:34Z (RED), green after fix | n/a | `scratchpad/t123/EVIDENCE.txt` — RED RUN shows `--- FAIL: …/plakar:_bare_restore_is_accepted` ×3 against neutralised fix; final run `--- FAIL lines: 0` | +| 7 (T059–T063) — 18 added incl. `TestFailureMessageContract` (10 rows), `TestReportDatabaseLocations`; **mutation-verified** (weakening 2 messages → 5 failures) | unit | `go test -tags untested_go_version -count=1 -v ./src/internal/app/` | 2026-09-03T13:14Z | n/a | `scratchpad/EVIDENCE-chunk7-T059-T064.txt` — 0 FAIL, `ok infrahub-ops/src/internal/app` | +| 7 (T064) — 16 diagnostic e2e cases | **e2e** | `uv run pytest -v -m k8s tests/e2e` (CI) | **deferred — local e2e not supported** | T072; `vcluster` absent | 16 collected 0 errors; 16 skipped with T072 reason string | +| 8 (T065–T068) — no tests; documentation | docs | `vale $(find ./docs …)`; `rumdl check .` | 2026-09-03T13:28Z | n/a | `0 errors, 18 warnings in 18 files` (= baseline); `Success: No issues found in 33 files` | +| 9 (T069–T071, T074) — 2 subtests + T071 audit; guard mutation-proven | unit | `go test -tags untested_go_version -count=1 ./src/...` | 2026-09-03T16:02Z | n/a | `scratchpad/EVIDENCE-chunk9-T069-T071-T074.md`; reverting the guard fails both new subtests (`t074_mutation.log`) | +| Fix A (T124–T130) — incl. `TestResolutionIssuesOneNamespaceListing`, `TestRestoreLatestBackupReleasesTransientWorkloads`; **424 fn / 1427 subtests** | unit | `go test -tags untested_go_version -count=1 ./src/...` | 2026-09-03T15:39Z | n/a | `scratchpad/EVIDENCE-fix-A-critical-and-restore-safety.md`; RED RUN `--- FAIL: TestRestoreLatestBackupReleasesTransientWorkloads` with defer removed | +| Fix B (T131–T141) — incl. `TestCypherColumnLookupIsIndifferentToCase`; 4 mutations killed and reverted; **430 fn / 1450 subtests** | unit | `go test -tags untested_go_version -count=1 ./src/...` | 2026-09-03T16:37Z | n/a | `scratchpad/EVIDENCE-fix-review-b.md`; each mutation block ends `### REVERTED, re-run exit=0` | + +**Final gate state, verified by the orchestrator directly (not relayed):** +`go test -tags untested_go_version -count=1 ./src/...` → all packages `ok`, **1450 passed, 0 failed, +0 skipped**. `golangci-lint run --build-tags untested_go_version ./src/...` → `0 issues.` +`go vet -tags untested_go_version ./src/...` → exit 0. `make build` → all three binaries. +Vale `0 errors, 18 warnings` (baseline). rumdl `0 issues in 33 files`. + +--- + +## 5. Review findings + +A six-agent review (`code`, `errors`, `tests`, `comments`, `types`, `simplify`) over `080917d..HEAD`, +scoped to this round because earlier commits were reviewed twice. **21 findings; 20 fixed, 1 +deferred.** + +| Severity | File / symbol | Finding | Status | +|---|---|---|---| +| **CRITICAL** | `environment_kubernetes.go` `namespacePodListing` | Memoized listing froze every liveness poll. On Helm the fallback is the *normal* path, so `confirmAppContainersQuiesced` could never succeed — and it fails **after** `wipeTransientData()`, bricking the restore and blaming a pod that terminated normally. Mirror case: a successful restore reports all six services failed to return. **Found independently by two reviewers.** | ✅ `cca32bb` | +| **CRITICAL** | `backup_containers_test.go` | FR-013's three `defer returnAppContainersToScale` blocks could all be deleted with the suite green. Mutation-proven. | ✅ `ea943b6` | +| **CRITICAL** | `backup_containers_test.go` | FR-026's `confirmAppContainersQuiesced` could be removed from `RestoreBackup` and all four Plakar sites with the suite green. Mutation-proven. | ✅ `ea943b6` | +| important | `main.go` `restoreCmd.Args` | **T123**: cobra runs `ValidateArgs` before `PersistentPreRunE`, where `b867f8e` had moved config resolution — so `--backend plakar restore` was refused for arity. CI red on 3 Docker e2e tests. | ✅ `a181b27` | +| important | `backup_metadata.go` `applySourceProvenance` | `capture_complete` computed **before** the captures ran, so it was `true` in every artefact and `refuseIncompleteCapture` was unreachable in the only direction that matters. | ✅ `25b741f` | +| important | `restore_latest.go` | `RestoreLatestBackup` created transient workloads with **no deferred release**; two doc comments asserted the opposite. | ✅ `25b741f` | +| important | `backup_containers.go` `startAppContainers` | Returned on first failure with `cache` first in order, leaving five services at zero — FR-013 defeated by a failure a retry would have survived. | ✅ `25b741f` | +| important | `external_restore.go` `waitForNeo4jRestore` | Retried **permanent** failures for the full 2h bound with Infrahub at zero, then blamed the database for a cluster failure. | ✅ `25b741f` | +| important | `plakar_restore.go` | Community refusal fired **after** services were stopped, unlike every other refusal on this feature. | ✅ `25b741f` | +| important | `external_endpoint.go` `undiscoverableEndpoint` | An RBAC denial was reported as "no address configured" — the same FR-004/FR-019 conflation that caused the original reset. | ✅ `25b741f` | +| important | `external-databases.mdx`, ADR-0008 | RBAC verb list omitted **`watch`**, which `kubectl wait` needs. An operator granting exactly the documented set fails at the readiness wait. | ✅ `a65d62a` | +| important | `external-databases.mdx` | Claimed "earlier versions connected to it without one" — no released version reaches an external database at all. | ✅ `a65d62a` | +| important | `contracts/cli-surface.md` | `--external-db-insecure-tls` marked "Applies to: backup"; it governs restore too. | ✅ `a65d62a` | +| important | `external_backup_gate.go` | Comment said "both callers"; there are three. | ✅ `43df4c4` | +| important | `plakar.go` `initPlakarContext` | **Found by fix B, missed by all six reviewers**: `KContext.CacheDir` never set, so kloset wrote repository state into the operator's working directory and `--plakar-cache-dir` reached it never. | ✅ `42f359b` | +| important | `transient_workload.go` | Pod-deadline formula `readyTimeout + N*bound + margin` spelled three times; a comment recorded it having been wrong **twice**. | ✅ `0cfa614` | +| important | `external_endpoint.go`, `external_restore.go` | Three divergent Cypher column-lookup conventions; the raw-constant reader breaks once a column **constant** is mixed-case, which `neo4jStatusColumn = "currentStatus"` already is → FR-005 provenance silently empty. | ✅ `0cfa614` | +| important | `external_endpoint.go` `databaseTarget` | Invariant held by convention: no construction witness, and both preparers take a target by value without re-checking. A hand-built literal would perform an unauthorised external restore. | ✅ `43df4c4` | +| important | `app.go` `ExternalRestoreAuth` | Two fields answering one question; `{Allowed: true}` was representable and would authorise a destructive write while logging "no channel this run can name". | ✅ `43df4c4` | +| suggestion | `external_capture.go` / `external_restore.go` | Duplicated copy prologue; duplicated scratch normalisation with the rationale on only one copy. | ✅ `0cfa614` | +| important | `test_k8s_external_both.py` | FR-021 e2e case corrupted the outer `.tar.gz`, failing gzip **upstream** of seed validation — green without testing its scenario. | ⚠️ retargeted `74f28e4`, **not executed** (T072) | + +**What the review found clean, and it matters:** a sweep of all 224 function symbols in the nine +changed Kubernetes/external files, and separately a call-graph check over 720 functions from 39 +command entry points, found **all 103 new production symbols have non-test callers and are +transitively reachable**. Zero fictions. The inert-requirement failure that reset this feature did +not recur. 13 of 15 mutations were killed, with a no-op control correctly reporting SURVIVED. + +**Two reviewer corrections worth recording**: the comment reviewer found the orchestrator's +paraphrase of AGENTS.md's `nameMatchesService` acceptance test was wrong and the shipped text +correct; fix B found the simplify reviewer's finding-5 diagnosis was one step off (the fragile side +is the caller's spelling, not the header's) and fixed the defect generalised rather than as reported. + +--- + +## 6. Autonomous decisions + +1. **Chunk plan**: nine chunks on `tasks.md`'s own phase headings, Phase 2T's 24 findings **first** — + they are defects in already-landed code, and T097+T098 corrupt a backup in composition. Strictly + sequential; no two implementation subagents ever ran concurrently. +2. **T118 was escalated to the user, not decided here.** The user chose *refuse external capture from + `infrahub-collect`*, keeping ADR-0003's read-only guarantee unqualified, over amending the ADR. + Implemented and verified reachable: `infrahub-collect/main.go:63` → `CollectBundle` → + `runStandardBackup` → `CreateBackup` → `prepareDatabaseCapture` → refusal, before the reaper and + before `stopAppContainers`. +3. **Four newly-discovered items were recorded as tasks rather than absorbed** (T121, T122, T141, and + T107/T119 left open on them). Each would have widened its chunk into a different piece of work; + T121 in particular is an *internal*-path FR-014 gap the external work merely surfaced. +4. **A regression fix was inserted out of plan** (`a181b27`) rather than deferred, because CI was red + and continuing would have piled work on a broken branch. +5. **Three documentation falsehoods were fixed by the orchestrator directly** (`a65d62a`), staged + file-by-file because a fix chunk had in-flight edits. Small, localised, and the `watch` verb one + would have blocked any operator following the docs. +6. **A `.gitignore` entry was added** (`5ce2a1f`) for the Plakar e2e residue. Fix B later found the + real cause (`CacheDir`) — so that commit treated a symptom. Kept as defence-in-depth; noted here + because the framing was wrong. +7. **Review scoped to `080917d..HEAD`**, not the whole 70-commit branch, since earlier commits were + reviewed twice. Six agents run in parallel — read-only, so the no-parallelism rule did not apply. +8. **§4's evidence table is grouped per chunk, not per test.** ~130 tests were added; a 130-row table + would obscure audit rather than aid it. Every row names its test identifiers and the file holding + verbatim runner lines, and those files were read and checked by the orchestrator. This is a + deliberate deviation from the one-row-per-test instruction. +9. **E2E tasks were reported `⚠️ partial`, never `✅ done`.** A skipped test is not a passing test. + T058's Docker leg was genuinely attempted and run because minikube and Docker were available; the + k8s leg proved to need `vcluster` and was reported as blocked rather than skipped quietly. +10. **A stray evidence file written into the repo root was moved to the scratchpad** rather than + committed or deleted. + +--- + +## 7. Suggested next steps + +1. **Provision T072's CI topology** — a database outside the cluster under test with an Enterprise + licence and an object store the database itself can reach, plus `vcluster` for the k8s leg. Six + e2e tasks are written and waiting on exactly this, and until it exists **no end-to-end evidence + for this feature exists at all**. +2. **Have T073's conversation before shipping restore.** The seed-from-URI mechanism is unverified + against a real Enterprise estate: that `CREATE OR REPLACE … seedURI` is accepted in that form at + 2025.01+, that `s3`/`gs`/`azb` are the actual built-in provider set, that `SHOW DATABASE`'s + `statusMessage` reads as expected, and that a customer's database host can reach an object store + at all. Restore rests on a single mechanism whose prerequisites are least likely to hold for + exactly this population, and both fallbacks were rejected. **This is the highest risk on the + branch.** +3. **Close T121 and T122**, which unblock T107 and T119. T121 is the FR-014 argv gap on the + *internal* path — real scope, just not scope this feature created. +4. **Bound T141's test** so suite runtime stops swinging 47s–260s. +5. **Reconcile `spec.md` and `research.md`** with shipped reality. Two known contradictions remain + after this run's fixes, and the spec is now behind the implementation in documented ways. +6. **Consider a process change.** The dominant defect class was identical in all three review rounds: + *a new single-source-of-truth introduced, an older parallel answer left standing.* Three rounds is + a property of how this code is being built, not luck. Two cheap checks caught nearly everything + this round and could run per-chunk: **grep every new exported symbol for a non-test caller**, and + **mutation-test every new guard** — the second is what exposed FR-013 and FR-026 as unprotected + after they had been reported done and verified by grep. +7. Then open a PR. Nothing has been pushed. + +> **§7 is superseded in part by §8–§10 below.** Items 3 and 4 are done, item 2's mechanism half is +> answered, and item 1 is much cheaper than it reads. Item 6 stands unchanged and is the one worth +> acting on. + +--- + +## 8. T107 closed, T121 descoped — on FR-014's own wording + +FR-014 reads: *"Credentials for an **externally-managed** database MUST NOT be exposed on a process +command line or in workload configuration readable by deployment observers."* It is scoped to +external databases and is already satisfied — T040 moved that path onto the mounted secret, and +`TestNoCredentialReachesAnExternalArgv` pins it. + +T107's argv clause is on the **internal** path: after the gate landed in `e49b4be`, +`redactDatabase` and `isNeo4jCluster` only run for a database that lives in the deployment, because +an external one is refused before them. So the gate half was the whole of the defect, and it is +fixed and covered. **T107 closed.** + +T121 (a stdin-carrying exec primitive, routing 10 internal `cypher-shell -p` sites) +is **out of scope for 007**, to be filed separately. All 10 sites exist unchanged at base `8186fcf`, +so the branch neither introduced nor worsened them, and doing it here would have forced a re-baseline +of the FR-015 pin that asserts the internal argv character-for-character. + +**Residual, accepted knowingly**: the internal path passes the Neo4j password on argv, visible to +anything reading `ps` inside the database container and to the Kubernetes API audit log. Both +populations already hold credential-equivalent access — exec rights on the database pod are what put +you in that container, and they also let you read Neo4j's own configuration — so the marginal +exposure is audit-log retention rather than a new attacker path. + +**Process note.** The reviewer's finding was accurate as a description and matched the defect class +that had bitten this branch three times, and it was briefed to a chunk before anyone read the +requirement it cited. Reading FR-014's sentence took one grep and reversed the decision. + +--- + +## 9. Phase 2W — findings from real Neo4j Enterprise servers + +The first execution of this feature's mechanism against real servers rather than fakes. +`neo4j:5-enterprise` (**5.26.30**) and `neo4j:2025-enterprise` (**2025.12.1**) as host-side +containers with a non-loopback backup listener, plus MinIO, on one Docker network. **No cluster +needed for any of it.** Evidence: `scratchpad/EVIDENCE-real-neo4j-enterprise.md`. + +| Severity | Finding | Status | +|---|---|---| +| **CRITICAL** | **`--expand-commands` aborted every external capture before the database was contacted.** `neo4j-admin` refuses to read any config file looser than 0640 given that flag, and reads both `neo4j.conf` and `neo4j-admin.conf`; the pinned image ships them at **0777**. The failure names a permission the operator never set in an image they did not build. | ✅ `b4402f4` | +| important | The e2e Helm overlay wrote `global.infrahubEnv`, a key the chart does not define. Helm ignores unknown keys silently, so Infrahub would have deployed pointed at the Neo4j the overlay had just disabled. Env is per component, nested under the component's own name **twice**. Confirming these keys was named in the fixture's docstring as part of T072. | ✅ `0f85cc0` | +| important | The seed provider needs a **region**, not only credentials — without one, `CREATE DATABASE` succeeds and the database sits `offline`. Also: it is not a CLI, it is the built-in Java SDK. | ✅ `9ead9bc` | +| important | S3-compatible stores work (`AWS_ENDPOINT_URL_S3` is honoured), so **restore needs no AWS account**. The guide said to configure an endpoint without saying how. | ✅ `9ead9bc` | +| — | **The round trip is proven.** Capture from a separate container against the live listener → object store → seeded into a *different* server → `(:Probe {n:1})` verified by content. Artifact naming matches `neo4jArtifactProduced`. | validated | +| — | **Two design decisions confirmed against reality**: the statement returns success while the database is still loading, and `currentStatus`/`statusMessage` carry the real verdict — which is why T050 exists. | validated | +| — | **The version gate sits exactly on the boundary**: the atomic `CREATE OR REPLACE … seedURI` form is *rejected* at 5.26.30 and *accepted* at 2025.12.1, and `neo4jSeedReplaceFloor = "2025.01"`. | validated | +| — | All four probe statements return the expected columns; values come back **quoted** and `splitPlainRow` strips them correctly. One unrepresented shape: `dbms.components()` returns **two** rows on 2025.x. | validated | + +**Why no test caught the critical one.** The argv is asserted as a string; nothing executed +`neo4j-admin`. This is a fourth failure class to add to the three this branch already documented: +not unreachable code, not code wired to the wrong thing, not a guarantee that fails in composition — +**a command that is correct as a string and rejected by the program that runs it.** + +--- + +## 10. T072 and T073 are materially cheaper than this report first said + +**T072 is provisioning work now, not an open question.** The topology was stood up on a developer +machine: Neo4j Enterprise and MinIO as host-side containers on one network, and from a pod inside +minikube all three of `host.minikube.internal:6362`, `:7687` and `:9000` are reachable — with the +store reachable *by the database*, which the fixture's own comment calls the hard part. The chart and +images pull **anonymously**; `vcluster` installs from Homebrew; no cloud account is needed. The one +step not yet exercised is getting the Infrahub deployment itself healthy, which is why it stays open. + +**T073's mechanism questions are answered; the population question is not.** What remains is whether +a given customer's Neo4j hosts can be given egress to an object store, whether their operators will +accept a server configuration change for a non-loopback backup listener, and the criterion for +falling back to the recreate procedure. That is a conversation, not a test — and it is no longer the +highest risk on the branch, because the mechanism it depended on now demonstrably works. + +**Revised next steps**, replacing §7 items 1–4: + +1. Get an Infrahub deployment healthy against the external topology, then run the six e2e tasks + (T054–T058, T064). Everything they need except that is proven. +2. Have T073's customer conversation about prerequisites and the fallback criterion. +3. File T121 as its own issue (internal-path FR-014 hardening). +4. Add a fixture for the two-row `dbms.components()` shape (T147). +5. §7 item 6 stands: add the two cheap per-chunk checks — non-test caller grep, and guard mutation. + +## 11. Phase 2X — the fourth review round's high-severity findings, fixed in severity order + +The review that followed Phase 2W fanned out fourteen finders. Their findings came back verified but +were never written into the ledger — the same hand-off failure that lost two residuals earlier in +this feature — so before any was fixed the four that matter were recorded as T148–T151 +(`82e99fe`), and each subsequent subagent's residuals were recorded as a task *before the next +subagent was dispatched* (T152–T155). Four clean-context subagents, strictly sequential, each with a +mutation check (guard removed → pin fails with the production symptom → guard restored) as evidence. + +| Task | Defect | Fix | Commit | +|---|---|---|---| +| T148 | External Neo4j **restore** path unreachable: the edition was read through a capture-only accessor, so restore fell through to `cypher-shell` in a container that does not exist | `externalCaptureSourceFor` deleted; `externalDatabaseFor` answers for either direction from the same embedded probe; 7 callers, incl. `isNeo4jCluster` on the `restoreNeo4jEnterprise` path, which carried the same defect | `ddb6233` | +| T149 | A failed capture commits a **0-byte snapshot and exits 0**. Now *confirmed by a running test* against the vendored kloset v1.1.0-beta.6: failing reader → `Backup()=nil, Commit()=nil, bytes=0 errors=1` | `commitComponentSnapshot` reads the source summary after `Backup` and refuses *before* `Commit` (errors>0 or bytes==0), through `recordIncompleteCapture`, so the empty snapshot never exists. Secondary: `restoreSingleSnapshot` resolves the edition tag only for the Neo4j component | `a070bc5` | +| T150 | Ownership claimed across a service boundary: `infrahub-task-manager-db-exporter` read as `task-manager-db` (external RDS → Internal → `pg_restore --clean --create` into a sidecar) | One recogniser, `controllerGeneratedSuffix`; `serviceWithGeneratedSuffixIndex` is the single claim site for all three name tiers. **Direction B (`pg-task-manager-db-1` → External) is by design** (T103/T105, `b83099a`), pinned not reversed — a spec question, recorded in T154 | `db37ac2` | +| T151 | `CombinedOutput` fed `PodUID`, so a kubectl stderr warning became part of the UID, the Secret's ownerReference matched no pod, and a credential Secret outlived the run | `separatedRunner` is the one executor-reading `podRunner`; both bounded runners delegate to it; stderr folded into the error on failure. FR-019 Forbidden detection now fires on the query/remove paths as a side effect | `5c0a3bf` | + +Evidence files: `scratchpad/EVIDENCE-T148.md` … `EVIDENCE-T151.md`, each with the mutation FAIL and +restored PASS verbatim. Gates after `5c0a3bf`: build, vet, full `go test ./src/...`, `gofmt -l` +empty, `golangci-lint` 0 issues. + +**What this changes about §1.** Two of the four were "the feature's main path does not run" and "a +backup can report success while containing nothing"; both were invisible to every green gate. They +are fixed, but the reason they existed — a new single answer introduced beside an old one, and a +dependency's contract assumed rather than probed — is the reason to prefer one real end-to-end +restore over a fifth review round. §10's revised next steps stand, with one insertion at the top: + +0. Decide T154 (i): whether an in-cluster database under a prefix no release declares is *meant* to + read External. The code says yes; the spec says nothing. + +**Still open from that review, not yet recorded as tasks** (lower severity, listed so they are not +lost again): `transientWorkloadRBAC` string omits `watch`; Plakar `--snapshot` postgres branch never +calls `restartDependencies()`; three documented `infrahub-backup restore` examples cannot run and +`--backup-id` is silently ignored on tarball; the e2e `_external_values_overlay` fans +`PREFECT_API_DATABASE_CONNECTION_URL` to the Neo4j-reading components instead of `prefect-server`; +`execReadCloser` data race and a non-wrapped `errStreamIdleTimeout`; `readinessDiagnosis` blind to +`Unschedulable`; `escapeJSONPathKey` correct but unpinned; T122 applied to one of four identical +paths; the pending-pod `IsRunning` hole. + +## 12. Phase 2Y — the first real end-to-end run on Kubernetes + +Fresh minikube profile `infrahub-ext`; `neo4j:2025.10.1-enterprise` and MinIO outside the cluster on +one Docker network; chart `4.33.2` with `neo4j.enabled=false`, env on both components, **no +workaround needed** — which closes the step T072 said had never been exercised. Mixed topology, +exactly quickstart Scenario 1. + +| Stage | Result | +|---|---| +| S1 `environment detect` | Kubernetes, namespace, database **External** `host.minikube.internal:17687`, task-manager-db **Internal** | +| S2 capture | PASS on the **third** attempt: first needed `--external-db-insecure-tls` (error named no flag), second needed `--external-db-scratch-size` (store-size metric absent on 2025.x). Then: Neo4j component 1,351,984 B, `capture_complete: true`, per-service `external_location`, no transient leftovers, replicas untouched | +| S3 marker | Tag created via GraphQL; seen by Cypher (nodes 19102→19109) | +| S4 refusal | Refused without `--allow-external-restore`, names the flag and the config key; nothing stopped | +| S5 restore | Staged to `s3://infrahub-seed/seed`, seeded, `online after 1 status read(s)`; marker gone by Cypher **and** GraphQL; nodes back to 19102; pods back to 1/2/1 Ready; staged object cleaned; no leftovers | +| S6 version gate | Abort: `utility (5.26.30) is older than the server (2025.10.1)`; 0 archives written | +| S7 collect | Created no transient pod (ADR-0003) | + +This is the class of evidence none of the four review rounds could produce, and it did what the +rounds could not: it found that **an explicit `--k8s-namespace` is a preference, not a pin** (T156) — +on a host that also runs an Infrahub compose project, a momentarily unreachable cluster would send +a restore into the wrong deployment — and that **the store-size probe fails on every current +server** (T158), so the documented "derived from the database size" default is dead code on 2025.x. +Findings recorded as T156–T161; the doc's five misleads as T161. The three HIGH items are the next +fixes, in that order. Cluster and containers are left running for inspection; teardown commands are +in the evidence file. + +--- + +# Implementation report — Phase 6: database tools in the infrahub-backup image (run of 2026-10-12/13) + +**Status: INCOMPLETE.** Two end-to-end tasks (T175, T176) are written but have not run, because they +need the external-database topology of T072, which neither this host nor CI provides. + +- Feature: ADR 0017, FR-030 to FR-033, requested in review of + [PR #172](https://github.com/opsmill/infrahub-backup/pull/172) +- Spec dir: `specs/archive/007-external-database-backup` +- Base commit: `a170ef0` · Head commit: `3fe7b95` + +## Chunk ledger + +| Chunk | Tasks | ✅ / ⚠️ / ❌ | Commits | Flagged upward | +|---|---|---|---|---| +| 6A The image carries the tools | 2 | 2 / 0 / 0 | `ea1d73b` | Base moved to Alpine 3.23 (3.21 and 3.22 have no PostgreSQL 18 client). Image 34 → 343.9 MiB. | +| 6B The workload runs the infrahub-backup image | 5 | 5 / 0 / 0 | `48f6e95` | Development build = any version that is not a release semver tag. The uid stays explicit (1000) because the image names its user. | +| 6C Direct mode in the namespace | 5 | 5 / 0 / 0 | `b5e87ad` | No new interface: a `local:` target is handled at four call sites. A development build is not refused in direct mode, because no image runs. | +| 6D End-to-end | 3 | 1 / 2 / 0 | `e6abd08` | Image loaded into the vcluster nodes with `ctr import`. Version-gate cases build images reporting `4.0.0` and `2099.12.0`. Permission docs corrected: scaling needs `patch` on `deployments/scale` and `statefulsets/scale`. | +| Review fixes | — | — | `90a088c`, `97461b9` | See review findings. | +| Image size (asked by the PR author during the run) | — | — | `d1b3c2f`, `3fe7b95` | jlink runtime and unused jars removed: 343.9 → 208.7 → 156.4 MiB compressed (arm64). | + +## Tasks not completed + +- T175: the external suites now pass `--external-db-image` and the reworked version-gate and + credential cases, but have not run. Needs the T072 topology. +- T176: `tests/e2e/test_k8s_external_in_namespace.py` is written and has not run, for the same reason. + +## Local-pass evidence + +| Test id | Type | Run command | Passed at (ISO 8601) | Environment | Verbatim pass line | +|---|---|---|---|---|---| +| `make docker-check-tools` (`scripts/check-image-tools.sh`) | integration | `make docker-check-tools` | 2026-10-13 (after `3fe7b95`) | macOS, Docker 29.4.0 (OrbStack), linux/arm64 | `All database tools present in infrahub-backup:check-tools`, `Image size: 163982501 bytes (156 MiB)` | +| Image validation: remote backup (plain and with backup SSL policy), inspect, restore, consistency check, cypher-shell plain/ssc/TLS, pg_dump/pg_restore | integration | `scratchpad/image-slim/validate.sh`, `tls.sh` (not in the repo) | 2026-10-13 | Own Neo4j 2025.10.1 Enterprise and postgres:18-alpine on a dedicated network, linux/arm64 | `Backup command completed.`, `consistency check rc=0`, `pg_dump ok`, `pg_restore ok` | +| `TestExternalDBFlagDefaults`, `TestExternalDBFlagsFromCommandLine`, `TestExternalDBImagesIgnoreEnvironment`, `TestExternalDBImageIsOneUnboundFlag`, `TestCheckVersionCompatibilityAbortNamesTheFlag`, `TestFailureMessageContract`, `TestDefaultExternalDBImageFollowsTheBuildVersion`, `TestDevelopmentBuildIsRefusedBeforeAnyKubectlCall`, `TestExternalDBImageFlagLiftsTheDevelopmentBuildRefusal`, `TestRestoreWorkloadSpec`, `TestBuildTransientPodManifestShape`, `TestBuildTransientPodManifestPostgresRunsAsItsOwnUser`, `TestTransientPodDeclaresANonRootUserForEveryService`, `TestBuildTransientPodManifestRejects`, `TestProbeWorkloadSpecNeedsNoStoreSize`, `TestPrepareExternalSourceWithSequencesTheGate` | unit | `go test -tags untested_go_version ./src/internal/app -count=1 -v -run ''` | 2026-10-12T10:43:47Z | n/a | `--- PASS: ` for each | +| `TestDetectDirectMode`, `TestDirectModeCreatesNothingInTheNamespace`, `TestDirectModeOutsideTheNamespaceKeepsTheJobPath`, `TestDirectModeIsStillRefusedForCollect`, `TestDirectModeProbeRunsLocallyWithCredentialsInTheEnvironment`, `TestDirectModeScratchIsRemovedOnEveryExitPath`, `TestDirectModeRestoreDumpFitReadsLocalFreeSpace`, `TestDirectModeCATrustLandsInScratch`, `TestLocalFreeSpaceReadsTheHost` | unit | `go test -tags untested_go_version ./src/internal/app/ -run '' -count=1 -v` | 2026-10-12T10:56:49Z | n/a | `--- PASS: ` for each | +| `TestLocalWorkloadTranslate`, `TestLocalWorkloadEnvironmentDropsWhatClientsReadOnTheirOwn`, `TestPublishedImageTag` | unit | `go test -tags untested_go_version ./src/internal/app/ -count=1 -run '' -v` | 2026-10-13T07:59:38Z | n/a | `--- PASS: ` for each | +| Full Go suite | unit | `go test -tags untested_go_version ./src/... -count=1` | 2026-10-13T07:59:47Z | n/a | `ok infrahub-ops/src/internal/app 34.246s` | +| `tests/e2e/test_k8s_external_neo4j.py` (all, including `test_killing_the_tool_mid_capture_leaves_only_what_the_cluster_removes`) | e2e | `INFRAHUB_TESTING_EXTERNAL_DB=database INFRAHUB_TESTING_EXTERNAL_NEO4J_ADDRESS= INFRAHUB_TESTING_EXTERNAL_NEO4J_EDITION=enterprise INFRAHUB_TESTING_EXTERNAL_DB_IMAGE=infrahub-backup:e2e-local uv run pytest -v -m k8s tests/e2e/test_k8s_external_neo4j.py` | deferred — local E2E not supported (needs T072) | — | collected; `17 skipped` with the topology absent | +| `tests/e2e/test_k8s_external_both.py` | e2e | same, with the PostgreSQL variables | deferred — local E2E not supported (needs T072) | — | — | +| `tests/e2e/test_k8s_external_in_namespace.py` | e2e | same, on `test_k8s_external_in_namespace.py` | deferred — local E2E not supported (needs T072) | — | — | + +## Review findings + +| Severity | File | Summary | Outcome | +|---|---|---|---| +| High | `tests/e2e/test_k8s_external_neo4j.py` | The killed-run test expected nothing left at once, but a Job is removed only after its deadline and time-to-live (ADR 0018) | Fixed in `90a088c` | +| Low | `src/internal/app/external_local.go` | `/scratch` rewrite checked the remaining text, not the original string | Fixed in `97461b9` | +| Low | `src/internal/app/external_local.go` | Direct-mode child processes inherited the pod's `PG*`, `NEO4J_*` and Java option variables | Fixed in `97461b9` | +| Low | `src/internal/app/utils.go` | A `git describe` suffix after a pre-release tag counted as a published version | Fixed in `97461b9` | +| Low | `src/internal/app/external_local.go` | Cancelling kills only the direct child process, and the local copy has no time limit | Deferred | +| Info | `src/internal/app/external_local.go` | Any pod in the namespace with all tools on `PATH` takes direct mode; the version and edition gates still apply | Recorded | + +## Autonomous decisions + +- E2E tasks T175 and T176 are left open rather than marked done, because they have not run. +- A development build is not refused in direct mode (FR-031 covers the case where an image runs). +- amd64 could not be validated on this host (emulation does not work). CI now runs the tool check on + its amd64 runner; the backup and restore validation ran on arm64 only. +- The image's `neo4j-admin` cannot write to cloud storage. The docs for the manual direct-to-cloud + procedure say so. + +## Found during the run, outside this phase + +- A capture fails when the Neo4j backup port (6362) requires TLS: `neo4j-admin` uses TLS on that port + only with its own backup SSL policy, and the tool sets none (`ClosedChannelException`). This is the + same on the vendor image and the new one. ADR 0015 covers Bolt and PostgreSQL, not the backup port. + +## Suggested next steps + +1. Provision the T072 topology in CI and run the three external suites (T175, T176). +2. Decide whether a TLS-only backup port must be supported, and file an issue if so. +3. Bound the direct-mode copy and kill the child's process group (deferred low finding). + +--- + +# Addendum 2026-10-13: the external end-to-end suites run (T072, T175, T176) + +**Status: DONE for Phase 6 except T179.** T072, T175 and T176 are complete. T179 (TLS on the Neo4j backup port, [#178](https://github.com/opsmill/infrahub-backup/issues/178)) is still open. `tasks.md` lists it under Phase 6D; this report recorded the problem under "Found during the run, outside this phase". + +T054 to T057 are now ticked, because the CI run below ran and passed their cases. T064 stays open: the one case that run skipped is the Community refusal, one of T064's diagnostic rows, and it needs an external Community server that this topology does not provide. + +## Local-pass evidence + +| Test id | Type | Run command | Passed at | Environment context | Verbatim pass line | +|---|---|---|---|---|---| +| `test_k8s_external_neo4j.py` (11 cases) and `test_k8s_external_in_namespace.py` | e2e | `INFRAHUB_TESTING_EXTERNAL_DB=database uv run pytest -p no:pytest-infrahub-performance-test -v -m k8s tests/e2e/test_k8s_external_neo4j.py tests/e2e/test_k8s_external_in_namespace.py`, with the topology's other `INFRAHUB_TESTING_EXTERNAL_*` variables exported (the CI job's list is in `.github/workflows/ci.yml`); without them every case skips | 2026-10-13 (local, after the last code change) | Docker Desktop, docker-driver vcluster, `neo4j:2025.10.1-enterprise` and `postgres:18-alpine` with self-signed TLS, MinIO, reached at `host.docker.internal` | `11 passed, 1 skipped` | +| `test_k8s_external_both.py` (5 cases) | e2e | same, with `INFRAHUB_TESTING_EXTERNAL_DB=database,task-manager-db` | 2026-10-13 (local) | same | `5 passed` | +| both sessions above | e2e | CI job `e2e-tests-k8s (enterprise)` | 2026-10-13, at `2bfed31` ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425)) | GitHub runner, Docker bridge gateway | `11 passed, 1 skipped`; `5 passed` | + +The skipped case needs an external Community server, which this topology does not provide. + +## What the first run found + +| Severity | Where | Finding | Outcome | +|---|---|---|---| +| high | `external_restore.go` `seedPrefix` | Every restore staged into one `seed/` folder; Neo4j reads the artifacts beside a seed as one backup chain, so a failed restore's staged copy made every later restore fail | Fixed `9ba780d`: one folder per artifact | +| high | `external_restore.go` `failed` | A rejected seed is reported `offline` with a message that the failure marker did not match, so the restore waited the full 2-hour bound (T178) | Fixed `5707324`: `offline` with a message ends the wait | +| medium | `environment_kubernetes.go` | The namespace check reported `exit status 1` without kubectl's message | Fixed `16c4a08` | +| low | `external_restore.go` | The upload failure did not state that nothing was stopped | Fixed `9ba780d` | +| — | FR-012 test | `neo4j-admin` exits 0 when one of two servers does not resolve; the PR author decided the test changes (T177) | Test changed in `ca4992e` | +| — | TLS on port 6362 | Fails with both images; no client backup TLS policy | Issue [#178](https://github.com/opsmill/infrahub-backup/issues/178) | + +Test-only corrections: the suites need `--external-db-insecure-tls` and `--external-db-scratch-size` (new `INFRAHUB_TESTING_EXTERNAL_INSECURE_TLS` and `_SCRATCH_SIZE`), the both-external overlay needs `prefect-server.secret`, and five test corrections were needed (permission sets, a scale-to-zero race, `source_roles` for PostgreSQL, the `task-manager-db` component name, the after-run check). diff --git a/specs/archive/007-external-database-backup/plan.md b/specs/archive/007-external-database-backup/plan.md new file mode 100644 index 00000000..1a34929e --- /dev/null +++ b/specs/archive/007-external-database-backup/plan.md @@ -0,0 +1,247 @@ +# Implementation Plan: External Database Backup and Restore + +**Branch**: `007-external-database-backup` | **Date**: 2026-09-02 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `/specs/archive/007-external-database-backup/spec.md` + +## Summary + +Infrahub deployments whose Neo4j Enterprise or PostgreSQL databases run outside the Kubernetes +namespace cannot be backed up, because every backup and restore path executes the vendor tooling +*inside* the database container and no container exists. The fix is to supply one: create a +transient pod in the namespace that can reach the external endpoint, register it under the +canonical service name, and let every existing execution, streaming and copy primitive work +unchanged. + +Phase 0 established that this is the vendor's own documented topology — a *backup client*, a +machine on the database's network but not in its cluster, with the administration tooling +installed. It also confirmed remote restore is available by seeding a database from an object-store +URI over the client protocol, and produced six findings that changed the spec (see +[research.md](./research.md#spec-impact)), of which the most consequential is that seeding +validates only when the database starts — so an accepted restore is not a completed one. + +## Technical Context + +**Language/Version**: Go, version pinned in `go.mod` (currently 1.25.0) + +**Primary Dependencies**: cobra, viper, logrus, kloset (Plakar core), integration-fs. **No new Go +modules** — deliberately. Kubernetes and the databases are reached by shelling out to `kubectl` +through the existing `CommandExecutor`, per ADR-0001, and the server version is probed with the +client shipped in the transient pod's image rather than a vendored driver. + +**Storage**: Plakar repository (filesystem or S3) as today, unchanged. No new persistent store; the +new entities are run-scoped configuration plus one transient cluster object. + +**Testing**: `make test` for table-driven unit tests (endpoint resolution, version comparison +across both numbering schemes, metadata compatibility, completeness derivation); the Python +end-to-end suite under `tests/e2e/` for flows that touch containers, extended with +external-topology fixtures alongside the existing `test_k8s_*.py` cases. + +**Target Platform**: Kubernetes for this feature. The binaries themselves continue to +cross-compile for linux/darwin/windows on amd64/arm64. + +**Project Type**: CLI toolset — three thin cobra entry points over a shared core. This feature +touches `infrahub-backup` only. + +**Performance Goals**: No latency target. The relevant budget is resource-shaped: the backup +command is documented as CPU- and memory-intensive and needs local scratch space at least the size +of the database, which is why the transient pod's resources are explicit rather than defaulted +(FR-022). + +**Constraints**: No new inbound network exposure at the database (SC-002). Nothing destructive +without explicit authorisation (FR-009). No artifact may be presented as complete when it is not +(FR-012). Artifact layout and metadata additive only (FR-016). No new assumed deployment service +name (FR-017). + +**Scale/Scope**: Two logical databases, resolved independently; one to several endpoints each. The +change is concentrated in environment detection, configuration resolution and the restore path — +the ~93 call sites that name the two database services are expected to remain untouched, and that +being true is the design's main success indicator. + +## Constitution Check + +*GATE: evaluated against `.specify/memory/constitution.md` v1.1.0.* + +### Pre-Phase 0 + +| Principle | Assessment | +|---|---| +| **I. Split-Binary, Shared-Core** | PASS. All logic lands in `src/internal/app`; `src/cmd/infrahub-backup` gains flag registration only. No new binary, so no amendment needed. No helper executable added. | +| **II. Operational Safety & Data Integrity (NON-NEGOTIABLE)** | PASS with obligations, and this feature is squarely inside this principle. Backup stays non-destructive to the running instance. Restore's quiescing already restarts what it stopped; the transient workload joins that discipline (FR-011, FR-013). Metadata and checksums are additive with prior-version artifacts still restorable (FR-016). The destructive restore requires explicit authorisation — and this feature goes beyond the principle's "confirmation **or** a bypass flag" by requiring an authorisation that neither `--force` nor scheduled restore confers (FR-009), because the target is infrastructure the tool does not manage. | +| **III. Self-Contained, Cross-Platform Binaries** | PASS, and this principle decided the design. It permits depending on runtimes already present in the target environment while requiring actionable detection of their absence. The alternative of running the vendor tooling on the operator's host would require a JVM on machines the principle explicitly describes as having "no toolchain or package manager" — so it is deferred to User Story 3. `CGO_ENABLED=0` and cross-compilation are unaffected; no new embedded assets. | +| **IV. Quality Gates** | PASS. Behaviour change is accompanied by unit tests for the pure logic and end-to-end coverage for container-touching flows, per [quickstart.md](./quickstart.md). | +| **V. Explicit Errors & Streaming Feedback** | PASS with an obligation. Errors wrapped with `%w`, no `os.Exit` in the core. Streaming is preserved because the existing stream primitives are reused rather than replaced. The obligation is specific: a vendor exit status that conflates failure with partial success must not be surfaced as-is (research R6), and the wording contract in [contracts/cli-surface.md](./contracts/cli-surface.md) exists to satisfy SC-008. | + +**Operational constraints**: + +- *Deployment contract*: the documented service names are a public contract, and adding an assumed + service is a breaking change. The transient workload registers under an existing name, so no new + name is introduced (FR-017). **Compliant.** +- *Backup engine*: snapshot layout and metadata changes must preserve restorability of + previously released artifacts. Additive only, absence of the new fields meaning exactly what a + pre-feature artifact is. **Compliant, no migration path required.** +- *Nix packaging*: no `go.mod`/`go.sum` change is anticipated, so no vendor hash refresh. If a + dependency does become necessary, `scripts/update-vendor-hash.sh` must be run — never + hand-edited. +- *Documentation*: user-facing docs under `docs/docs/backup/` follow Diataxis with Vale and rumdl + passing. The customer-side prerequisites are the load-bearing content. + +**Gate result: PASS.** No violations, so Complexity Tracking stays empty. + +### Post-Phase 1 re-check + +Re-evaluated after the design artifacts. Still PASS, with two things the design added that the +constitution demanded and the spec had not yet stated: + +- Principle II drove **FR-021** (confirm the restored database reached a usable state). Without it + the tool would report a successful restore on an artifact that silently failed to load — the + precise failure this principle rules out, and it is invisible to the pre-flight in FR-007 because + it happens after the destructive step. +- Principle II also drove **FR-022** (explicit scratch sizing). A backup that fills a node's disk + damages the running instance, which the principle's first clause forbids. + +No new violation was introduced by Phase 1. One requirement correction is outstanding and needs the +feature owner's agreement: FR-006 asserts a symmetric version-matching rule that the vendor +documentation does not support (research R3, spec-impact item 6). It is recorded, not silently +applied. + +## Project Structure + +### Documentation (this feature) + +```text +specs/archive/007-external-database-backup/ +├── plan.md # This file +├── spec.md # Feature specification +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── checklists/ +│ └── requirements.md # Spec quality checklist +├── contracts/ +│ ├── cli-surface.md # Flags, config, exit behaviour, failure wording +│ └── artifact-and-metadata.md # Artifact layout and metadata compatibility +└── tasks.md # Phase 2 output (/speckit-tasks — NOT created here) +``` + +### Source Code (repository root) + +```text +src/ +├── cmd/ +│ └── infrahub-backup/ +│ └── main.go # Flag registration only (Principle I) +└── internal/ + └── app/ + ├── cli.go # New flags, viper binding + ├── app_config.go # Read the address/port/protocol/TLS settings + │ # currently discarded + ├── environment_kubernetes.go # errNoPodsMatched discrimination on the + │ # singular pod resolver; transient-pod + │ # create/adopt/reap; podCache registration + ├── external_endpoint.go # NEW — endpoint resolution, version compare + ├── transient_workload.go # NEW — manifest, lifecycle, reaping, secret + ├── backup_neo4j.go # Remote source arguments; completeness from + │ # reported output, not exit status + ├── backup_taskmanager.go # Remote host instead of localhost + ├── backup_metadata.go # Additive metadata fields + ├── backup.go # Restore ordering, authorisation, online check + └── retention.go # Skip artifacts marked incomplete + +tests/ +└── e2e/ + ├── conftest.py # External-topology fixtures + ├── test_k8s_external_neo4j.py # NEW + └── test_k8s_external_both.py # NEW + +docs/ +└── docs/ + └── backup/ # Prerequisites, flags, restore authorisation +``` + +**Structure Decision**: The existing single-project layout is kept unchanged. Two new files in +`src/internal/app` carry the genuinely new concepts — endpoint resolution and transient-workload +lifecycle — while every other change is an edit to the file that already owns that concern. This +follows Principle I (all logic in the shared core, entry points thin) and keeps the feature's +footprint aligned with its shape: the new behaviour is concentrated in resolution and lifecycle, +and the execution paths above them are deliberately left alone. + +## Complexity Tracking + +No constitution violations, so no justifications are required. + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| *(none)* | | | + +### T071: the one edited call site naming a database service + +The design's success indicator is that the existing call sites naming `database` and +`task-manager-db` are **not** edited — an edit means the stand-in-pod approach leaked into the +in-deployment path instead of dispatching ahead of it. T071 audited that against the branch base +`8186fcf`. One edit exists, and this is it. + +**Criterion used.** A *call site naming the service* is a line in non-test Go under `src/` where +`"database"` or `"task-manager-db"` sits in the **service-name argument position** — the first +argument to `Exec`, `ExecWithOptions`, `ExecStream*`, `CopyFrom`, `IsRunning` and the like. The +same word elsewhere is not a call site: `neo4j-admin database backup` is a subcommand, `database` +is also a directory inside the artifact and a metadata key, and a comment mentioning the word +names nothing. Both the narrow and the wide readings are reported below so the number cannot be +the criterion's doing. + +**Counts.** + +| Measure | Base | Edited | +|---|---|---| +| Non-test `src/` lines containing either literal | 93 | 3 | +| Of those, lines with the literal in the service-argument position | 48 | 1 | +| Service-argument lines whose **service name** changed | 48 | 0 | +| Call sites deleted, rerouted, or switched to the new constants | 48 | 0 | +| Enclosing functions of the 93 sites whose body was edited | 33 | 15 | + +**The one edit.** In `backupNeo4jEnterpriseStream` and `backupNeo4jEnterprise` +(`src/internal/app/backup_neo4j.go`), the argv handed to `iops.Exec("database", …)` was replaced +by `neo4jCaptureCommand(neo4jCaptureRequest{…})`. The service argument `"database"` is +byte-identical; only the second argument changed. It was necessary because the external capture +runs *the same* `neo4j-admin database backup` invocation from the transient workload, and FR-015 +plus `contracts/artifact-and-metadata.md` require an artifact taken from an external database to +be restorable into an internal deployment and back. Leaving the argv inline would have made that +a coincidence between two literal lists rather than a shared fact; the alternative — a second +copy of the flag list in the external path — is the drift the shared constructor exists to +prevent. Two further removed lines are the `neo4j-admin` subcommand word `database` moving into +that same constructor, and are not service names. + +**The wider reading, stated plainly.** 15 of the 33 enclosing functions were edited, including +`stopAppContainers`' callers, `redactDatabase`, `isNeo4jCluster`, `RestoreBackup` and the Plakar +restore paths. Every one of those edits *inserts* something ahead of the existing calls — an +`externalDatabaseFor` gate that returns early, FR-013 restart handling, FR-012's incomplete +capture refusal — or changes a signature to carry an error. None rewrites how a service is +named or which service is addressed. That is the shape the indicator was asking for, and the +single argv edit above is the one place it is not literally true. + +## Amendment 2026-10-12: database tools in the infrahub-backup image (ADR 0017) + +Requested in review of PR #172. Requirements FR-030 to FR-033. ADR 0018 records the existing move +from a bare Pod to a Job, which this amendment builds on. + +- **Image** (`Dockerfile`): the runtime stage adds a Java runtime, the `neo4j-admin` and + `cypher-shell` distributions copied from the pinned `neo4j:2025.10.1-enterprise` image in a + build stage, and the PostgreSQL 18 client tools. The base image may change if the Neo4j tools + do not run on Alpine; the deciding test is `neo4j-admin --version`, `cypher-shell --version`, + `pg_dump --version` inside the built image. The image keeps its non-root `infrahub` user (uid + 1000). +- **Workload image** (`cli.go`, `app.go`, `transient_workload.go`): one flag, `--external-db-image`, + not bound to the environment. Default `registry.opsmill.io/opsmill/infrahub-backup:` + from the build version; a development build has no default and the external gate refuses + before creating anything. `databaseImageUser` becomes the image's declared uid, for both roles. +- **Direct mode** (new `external_local.go`): detection is the service-account namespace file + (`/var/run/secrets/kubernetes.io/serviceaccount/namespace`) equal to the target namespace, plus + every tool the role needs found with `exec.LookPath`. The local runner provides the same + operations the capture and restore sequences already call on a workload (run a command, read a + file, copy out, check free scratch space), so the sequences do not fork. Credentials go in the + child's `Env`; scratch is a run-specific directory under the operator-settable scratch location, + removed on every exit path. +- **Tests**: unit tests for the flag, the default image, the development-build refusal, the + security context, detection, and the local runner's credential handling. The Kubernetes + end-to-end job builds the image and loads it into the test cluster; the suites move to + `--external-db-image`. diff --git a/specs/archive/007-external-database-backup/quickstart.md b/specs/archive/007-external-database-backup/quickstart.md new file mode 100644 index 00000000..cae5a131 --- /dev/null +++ b/specs/archive/007-external-database-backup/quickstart.md @@ -0,0 +1,139 @@ +# Quickstart: validating external database backup and restore + +**Feature**: 007-external-database-backup | **Date**: 2026-09-02 + +Runnable scenarios that prove the feature works. Each maps to acceptance scenarios in +[spec.md](./spec.md) and to success criteria. Details of flags and metadata live in +[contracts/](./contracts/) and [data-model.md](./data-model.md) rather than being repeated here. + +--- + +## Prerequisites + +- A Kubernetes context with an Infrahub deployment, as used by the existing + `tests/e2e/test_k8s_*.py` suite. +- Neo4j **Enterprise** running **outside** the Infrahub namespace, with: + - the backup service bound to a non-loopback address (it defaults to localhost, so this is a + server configuration change — see spec Assumptions, prerequisite 1); + - the Infrahub components pointed at it via `INFRAHUB_DB_ADDRESS`. +- PostgreSQL outside the namespace for the both-external scenario. +- An object store reachable **from the database server**, with the server-side prerequisites for + seeding in place (prerequisite 2). +- Permission in the namespace to create, get and delete Jobs; get, list, watch and delete pods; + exec into pods; and create, list and delete secrets. + +The standard gates apply throughout: `make test`, `make vet`, `make lint`. + +--- + +## Scenario 1 — Mixed topology, external Neo4j (US1 scenario 1) + +```bash +bin/infrahub-backup create --k8s-namespace +``` + +**Expect**: the resolved Neo4j endpoint is logged *before* any connection is opened; one artifact +covering both databases; metadata records the Neo4j `source_endpoints`, the roles observed, and `external_location: true` +for Neo4j only. + +**Proves**: FR-001 (per-service resolution), FR-003 (discovery and reporting), SC-004. + +## Scenario 2 — Both databases external (US1 scenario 2) + +Same command against a deployment with both databases outside the namespace. + +**Expect**: both captured; per-database `source_endpoints` and `source_roles`; no transient pod or secret remaining +afterwards. + +**Proves**: FR-001, FR-011, SC-006. + +## Scenario 3 — Full round trip (US1 scenario 3) + +```bash +bin/infrahub-backup create --k8s-namespace +# destroy the data +bin/infrahub-backup restore --k8s-namespace \ + --allow-external-restore +``` + +**Expect**: workloads quiesced, both databases restored, workloads returned to their prior scale, +Infrahub serving pre-backup data. The run must not report success until the restored database +reports a usable state. + +**Proves**: FR-013, FR-021, SC-001 — the criterion the whole feature exists to satisfy. + +## Scenario 4 — Version incompatibility aborts early (US1 scenario 4) + +Declare a client image older than the server. + +**Expect**: non-zero exit, both versions named, **zero bytes written**. Then declare a *newer* +client and expect a warning rather than an abort — the asymmetry established in research R3. + +**Proves**: FR-006, SC-003. + +## Scenario 5 — Restore refused without authorisation (US1 scenario 5) + +Run scenario 3's restore without `--allow-external-restore`. + +**Expect**: refusal naming the authorisation, its flag and its configuration key; databases +untouched. + +Then enable scheduled restore **only**, and confirm it does not confer the capability. + +**Proves**: FR-009, FR-010, SC-005. + +## Scenario 6 — Internal deployments unchanged (US1 scenario 6) + +Run the existing `tests/e2e/test_k8s_tarball.py`, `test_k8s_s3.py`, `test_k8s_plakar.py` and +`test_k8s_plakar_s3.py` unmodified, plus the Docker equivalents. + +**Expect**: all pass with no change; artifact layout identical. + +Additionally: restore an artifact produced by the **previous released version**, and restore an +artifact taken from an external database into an **internal** deployment. + +**Proves**: FR-015, FR-016, SC-007, and the cross-path interchangeability the artifact contract +requires. + +--- + +## Diagnostic scenarios (US2) + +| Provoke | Expect | Proves | +|---|---|---| +| External Neo4j **Community** | Refusal naming the storage-access limitation of the Community mechanism — not "service not found" | FR-008 | +| No container, no discoverable endpoint | Report that discovery was attempted, where it looked, and the override flag | FR-003, FR-004 | +| Pod-list permission revoked | **Failure**, not a switch to external mode | FR-002 | +| Pod-create permission absent | Failure naming the missing verbs and resource | FR-019 | +| Unreadable artifact URI | Refusal before any destructive step; database untouched | FR-007 | +| Corrupt artifact that is readable | Restore fails; success is never reported | FR-021 | + +Each message is checked against the three required elements in the CLI contract: condition, +resource, operator action (SC-008). + +--- + +## Robustness scenarios + +| Provoke | Expect | Proves | +|---|---|---| +| Kill the tool mid-capture | No surviving pod or secret; deployment not left quiesced | FR-011, FR-013, SC-006 | +| Kill the tool between credential creation and workload deletion | Credentials gone once the workload is reclaimed, with no tool involvement | FR-023 | +| Kill the tool, then start a new run | The new run reaps the stray and does not adopt it | FR-011 | +| Two concurrent runs | Neither adopts the other's transient workload | Data-model invariant | +| One endpoint of several unreachable | Artifact marked incomplete; retention and latest-backup selection both skip it | FR-012 | +| Database larger than default scratch | Succeeds, or fails cleanly without exhausting shared host storage | FR-022 | + +The unreachable-endpoint case is the one to write first: it exercises the finding that a partially +successful backup and a failed backup are indistinguishable by exit status, which is the most +likely source of a silently unusable artifact. + +--- + +## Compatibility envelope (deliberately measured, not assumed) + +Research R3 left the exact client/server version envelope for the backup command open, because the +vendor documentation does not state one. Establish it here as a matrix across the supported server +versions, covering both the `5.x` and calendar (`YYYY.MM`) numbering schemes, and record the result +in the spec's assumptions. Encoding a guessed rule in an abort condition would make the tool refuse +work it could have done. diff --git a/specs/archive/007-external-database-backup/research.md b/specs/archive/007-external-database-backup/research.md new file mode 100644 index 00000000..a8ad0684 --- /dev/null +++ b/specs/archive/007-external-database-backup/research.md @@ -0,0 +1,268 @@ +# Phase 0 Research: External Database Backup and Restore + +**Feature**: 007-external-database-backup | **Date**: 2026-09-02 + +The spec carried two load-bearing assumptions and several unverified mechanics. All are resolved +below. Six findings change what the spec must say; they are collected in [Spec impact](#spec-impact). + +--- + +## R1 — Remote online backup for Neo4j Enterprise + +**Decision**: Use `neo4j-admin database backup --from=[,…] --to-path=…`, +executed from a transient pod inside the namespace, streaming the artifact out through the +existing execution primitives. + +**Rationale**: This is not a workaround — it is the vendor's documented topology. From the +operations manual: + +> It is best practice, but not mandatory, to perform the backup from a server on the same network +> as the database, but that is not part of the cluster. You should install Neo4j on that machine +> to make the `neo4j-admin` command available. This machine is known as a **backup client**. + +The transient pod *is* a backup client. The manual also notes the command "uses a significant +amount of resources, such as memory and CPU. Therefore, it is recommended to perform the backup +on a separate dedicated machine" — which is an additional argument against User Story 3 running +it on an operator's laptop, and against running it inside an application pod. + +`--from` takes a "comma-separated list of host and port of Neo4j instances, each of which are +tried in order", and providing several is *recommended* for clusters "since that may allow a +backup to succeed even if some server is down, or not all databases are hosted on the same +servers". This is precisely FR-005. + +**Alternatives considered**: + +- *Operator-host execution* (User Story 3): still valid, still P3. The vendor guidance about + dedicated resources and the JVM prerequisite both count against it as the primary path. +- *`--to-path` pointing directly at cloud storage* (supported from 2025.03): rejected. It would + bypass the Plakar snapshot pipeline and with it deduplication, encryption, retention and + checksums. It also does not avoid local disk: "Backing up directly to cloud storage still + requires local disk space on the machine running the backup command." + +--- + +## R2 — Remote restore for Neo4j Enterprise + +**Decision**: Restore by seeding from a URI over Bolt — +`CREATE DATABASE OPTIONS { existingData: 'use', seedURI: 's3://…' }` — against an object +store, using the built-in cloud seed provider. Poll `SHOW DATABASES` to confirm the seed actually +loaded. + +**Rationale**: Confirmed available and explicitly listed as a supported way to restore a backup +artifact on both standalone and clustered servers. The artifact produced by R1 is directly usable: +"The seed can be a full backup, a differential backup, or a dump from an existing database." + +Three findings make this materially better than the spec assumed: + +1. **Cloud storage is built in; HTTP and file are not.** "Amazon S3, Google Cloud Storage, and + Azure Cloud Storage are supported by default, but the other providers require configuration of + `dbms.databases.seed_from_uri_providers`." The spec assumed the opposite — that object storage + needed a plugin and HTTP was built in. The chosen medium is the one that needs no server-side + provider configuration. +2. **`CREATE OR REPLACE DATABASE` accepts a seed URI** from 2025.01, so no separate `DROP` is + required. The operation is still destructive, but it is a single atomic replace rather than a + drop-then-create window in which the database does not exist. +3. **Custom S3 endpoints are supported** via `AWS_ENDPOINT_URL_S3` / `aws.endpointUrlS3`, named + for Ceph, MinIO and LocalStack. This lines up with the existing `--s3-endpoint` flag. + +**Critical correctness finding**: seeding is **asynchronous and fails silently to the caller**. + +> Download and validation of the seed is only performed as the new database is started. If it +> fails, the database is not available and it has the `statusMessage`: `Unable to start database` +> of the `SHOW DATABASES` command. + +A successful `CREATE DATABASE` therefore does **not** mean the restore worked. The tool MUST poll +`SHOW DATABASES` for `currentStatus` and `statusMessage` and treat "created but not online" as a +failed restore. Reporting success here would be exactly the silent data-loss failure Principle II +exists to prevent. + +**Seed provider choice**: `CloudSeedProvider` is current and supports backup chains and +`seedRestoreUntil`, but **requires the AWS CLI installed on each Neo4j server** plus +`~/.aws/credentials`. `S3SeedProvider` needs no AWS CLI (bundled libraries) but is **deprecated in +5.26** and requires `seedCredentials` with identical keystores configured across every cluster +server. Neither is free. + +**Decision**: target `CloudSeedProvider` and document the AWS CLI plus server-side credential +configuration as a customer prerequisite. Rationale: it is the non-deprecated path, it needs no +keystore setup across cluster members, and credentials stay in the customer's own server +configuration rather than travelling inside a Cypher statement. + +**Alternatives considered**: + +- `neo4j-admin database restore`: rejected, requires filesystem access to the server's store. +- `dbms.recreateDatabase()`: a genuine second Cypher-level option, listed for both standalone and + clustered restore. Not chosen for v1 — seed-from-URI is the documented restore-from-backup path + and is simpler to reason about. Worth revisiting if the seed prerequisites prove onerous. +- `neo4j-admin database copy` from a backup (2026.03+): too new to require of customers. +- `S3SeedProvider` with `seedCredentials`: rejected, deprecated and needs cluster-wide keystores. + +--- + +## R3 — Version compatibility + +**Decision**: The transient pod's image version is operator-declared (FR-004) and defaults to +matching the discovered server version. The tool probes the server version over Bolt and warns — +rather than aborting — when the client is *newer* than the server; it aborts when the client is +*older*. + +**Rationale**: The documentation does **not** state a strict client/server matching rule for the +backup command, so the spec's assumption of strict major.minor matching is unverified and probably +too strong. What is documented, for restore, is forward tolerance: a backup artifact "can be +restored within the same or to a later Neo4j version". Extrapolating the same direction to the +backup client is reasonable but unproven, so the rule is enforced asymmetrically and the exact +compatibility envelope is established empirically in the end-to-end suite rather than asserted. + +Neo4j has moved to calendar versioning (2026.07 current, 5.26 LTS), so version comparison must +handle both `5.x.y` and `YYYY.MM.p` schemes. A naive semver comparison silently misorders them. + +**PostgreSQL**: `pg_dump` reads servers older than itself, and refuses servers newer than itself; +a custom-format dump restores into the same or a later major version. So "use a client at least as +new as the server" is the safe rule, and the failure mode when violated is a loud refusal from the +tool itself rather than a corrupt dump. This is the asymmetry FR-006 already describes. + +**Alternatives considered**: two-phase discovery (spin a default image, probe, respin matched). +Rejected in the design session and still rejected — two pod lifecycles per run and a second failure +surface, to derive something the operator knows. + +--- + +## R4 — Endpoint discovery + +**Decision**: Read the external endpoint from the environment of the in-namespace Infrahub +components. Use `INFRAHUB_DB_ADDRESS` for the host list, `INFRAHUB_DB_PROTOCOL`, `INFRAHUB_DB_PORT`, +and the TLS settings for the Bolt connection; default the **backup** port to 6362 with a flag +override. PostgreSQL host and port come from `pgx.ParseConfig` of the Prefect connection URL — +fields already parsed and currently discarded. + +**Rationale**: Verified against Infrahub's own `DatabaseSettings` (env prefix `INFRAHUB_DB_`): + +| Setting | Default | Relevance | +|---|---|---| +| `address` | `localhost` | **"Database host, or a comma-separated list of cluster members in `host[:port]` format. Members without an explicit port use the value of the port setting."** | +| `port` | `7687` | Bolt, **not** the backup port | +| `protocol` | `bolt` | `neo4j://` for routed/HA deployments | +| `database` | *(none)* | pattern-validated name | +| `policy` | *(none)* | routing policy | +| `tls_enabled` / `tls_insecure` / `tls_ca_file` | `false` / `false` / *(none)* | Bolt TLS for the restore and version-probe connections | + +Two consequences: + +1. **Infrahub already models the multi-endpoint case in exactly the shape `--from` wants** — a + comma-separated `host[:port]` list. FR-005 needs no new concept, just faithful propagation. +2. **Discovery is partial by construction.** `INFRAHUB_DB_PORT` is the Bolt port; the backup + listener port is not part of Infrahub's configuration at all, because Infrahub never uses it. + It must be defaulted to 6362 and overridable. The spec's FR-003 implies discovery yields + everything needed; it does not. + +Note this repo's own docs and code reference only `INFRAHUB_DB_DATABASE`, `_USERNAME` and +`_PASSWORD`. `ADDRESS`, `PORT`, `PROTOCOL` and the TLS keys are real Infrahub settings that this +tool has simply never read. + +--- + +## R5 — Creating and reaping the transient pod without a Kubernetes client library + +**Decision**: Apply a full `Pod` manifest through `kubectl` on stdin, using the executor's +existing write-pipe primitive. Reap with `kubectl delete`. No client library. + +**Rationale**: ADR-0001 and Principle III both forbid taking on a Kubernetes client library, and +nothing here needs one. The Kubernetes backend already drives every operation through `kubectl` +subprocesses and already owns a stdin-writing primitive (`runCommandWritePipe`), so a manifest +applied on stdin is a use of existing machinery rather than a new mechanism. A manifest is required +rather than a bare `kubectl run` because the pod needs a run-ID label, a deadline, resource limits, +ephemeral storage sizing, and credentials from a Secret — all of which a manifest expresses +directly. + +**Required permissions** (to be documented as the FR-019 prerequisite): `pods` create/delete/get/ +list, `pods/exec` create, `pods/log` get, and `secrets` create/delete if credentials are passed +that way. `get`/`list` on pods is already required today. + +**Lifecycle bounding**: `activeDeadlineSeconds` on the pod, so the cluster reclaims it even if the +tool never returns; `restartPolicy: Never`; a run-ID label so a later run can identify and reap +strays. This satisfies FR-011 without the tool having to survive its own termination. + +**Newly discovered sizing constraint**: the backup command stages the store and transaction logs +into a local temporary directory before producing the artifact, and that location "must have free +space at least equal to the size of the database being backed up". The transient pod therefore +needs explicitly sized ephemeral storage or a volume. Left implicit, a large database fills the +node's disk and takes neighbouring workloads with it — a backup operation damaging the running +instance, which Principle II forbids. + +**Alternatives considered**: `kubectl run` with `--overrides` (same result, more awkward escaping); +a `Job` rather than a `Pod` (adds a controller and a second object to reap, and the tool wants to +`exec` into a specific pod anyway); `kubectl debug` (attaches to an existing pod, which is the case +that does not exist here). + +--- + +## R6 — Distinguishing partial success from failure + +**Decision**: Do not rely on the backup command's exit code alone to decide artifact completeness. +Treat a non-zero exit as failure by default, and parse the command's own output to identify the +"succeeded but some servers were uncontactable" case, which is reported as an incomplete-source +warning recorded in metadata rather than as a successful backup. + +**Rationale**: The exit codes conflate the two outcomes. Code `1` means "Backup failed, or +**succeeded but encountered problems such as some servers being uncontactable**", and for multiple +databases, "One or several backups failed, **or succeeded with problems**". There is no exit code +that distinguishes them. Since FR-012 forbids presenting a partial capture as complete, and FR-005 +actively encourages passing several endpoints (which makes the partial case *more* likely), this +distinction cannot be skipped. + +**Alternatives considered**: back up from a single endpoint to avoid the ambiguity. Rejected — it +throws away the availability benefit that made multi-endpoint the vendor recommendation. + +--- + +## R7 — Credential handling + +**Decision**: Pass database credentials to the transient pod through a Secret referenced by +`envFrom`, created and deleted within the run's lifecycle and labelled with the same run ID. + +**Rationale**: FR-014 forbids exposure on a process command line or in readable workload +configuration. Inline `env` values in a pod spec are readable by anyone who can `get pod`, and the +existing `ExecOptions{Env}` path places values on the exec command line, visible in API-server +audit logs and in-pod process listings. A Secret is the only option here that is both available +without new dependencies and not readable from the pod object itself. + +**Alternatives considered**: stdin (works for `pg_dump` via `PGPASSFILE`-style indirection but not +uniformly for `neo4j-admin`); a projected service-account token (not applicable — these are +database credentials, not cluster credentials). + +--- + +## R8 — Test strategy + +**Decision**: Extend the existing end-to-end suite with external-topology variants alongside the +current `tests/e2e/test_k8s_*.py` files: a database deployed outside the Infrahub namespace, with +the Infrahub components pointed at it by `INFRAHUB_DB_ADDRESS`. + +**Rationale**: Principle IV requires "CI-level end-to-end backup/restore jobs for flows that touch +containers", and the suite already covers the Kubernetes tarball, S3 and Plakar paths. The external +topology is a fixture change — where the database runs and what the application's environment says +— not a new test framework. The compatibility envelope left open in R3 is established here. + +--- + +## Spec impact + +Six findings require the spec to change. Five are additive; one is a requirement correction. + +| # | Finding | Spec change | +|---|---|---| +| 1 | The backup listener defaults to `127.0.0.1:6362` and "needs to be changed if backups are to be taken from another machine" | **New prerequisite** in Assumptions: a non-loopback `server.backup.listen_address` on the customer's server. This is a server-side configuration change, not merely a firewall rule — the original request assumed only the latter. | +| 2 | Cloud seed providers are built in; the AWS CLI must be present on each Neo4j server for `CloudSeedProvider` | **New prerequisite** in Assumptions, and it replaces the spec's inverted assumption that object storage needed a plugin while HTTP was built in. | +| 3 | Seeding validates only at database start; `CREATE DATABASE` returning success proves nothing | **New requirement**: confirm the restored database reaches an online status before reporting success. FR-007's pre-flight does not cover this, because the failure happens after the destructive step. | +| 4 | Exit code `1` conflates outright failure with partial success | **Strengthen FR-012**: completeness must be determined from command output, not exit status alone. | +| 5 | The backup command needs local scratch space at least the size of the database | **New requirement**: the transient pod's ephemeral storage must be explicitly sized, or a large database threatens the node and therefore the running instance (Principle II). | +| 6 | No documented strict client/server version match for the backup command; restore is documented as forward-tolerant | **Correct FR-006**: abort when the client is older than the server, warn when newer. As written, FR-006 asserts a symmetric rule the vendor documentation does not support, and would abort on combinations that work. | + +Item 6 changes an existing requirement rather than adding one, so it needs the feature owner's +agreement before the spec is edited. + +## Remaining uncertainty + +The exact backup client/server compatibility envelope (R3) is not documented and is deliberately +left to be measured by the end-to-end suite. This is recorded as a known unknown rather than +resolved on the page, because guessing a rule and encoding it in an abort condition is how a tool +starts refusing work it could have done. diff --git a/specs/archive/007-external-database-backup/retrospective.md b/specs/archive/007-external-database-backup/retrospective.md new file mode 100644 index 00000000..930543ae --- /dev/null +++ b/specs/archive/007-external-database-backup/retrospective.md @@ -0,0 +1,67 @@ +# Session Retrospective: 007-external-database-backup + +**Date**: 2026-09-02 +**Scope**: full speckit pipeline run — specify → plan → critique → tasks → 8 implementation chunks +→ `/code-review max` → partial reset → first fix chunk +**Branch**: `007-external-database-backup` +**Approved dispositions**: `fix-now` only. `open-pr`, `github-issue` and `local-only` were +explicitly declined by the user; their findings are retained below so they are not lost. + +--- + +## Findings + +| ID | Category | Evidence | Improvement | Disposition | +|----|----------|----------|-------------|-------------| +| R1 | Instructions / Config | Chunk 2a: "`speckit-checkpoint-commit` is not installed here; committed inline." Same for `critique-template.md` and the git extension. `.specify/extensions.yml` lists all three extensions as `installed:` | Record in `AGENTS.md` which extension assets exist and what to substitute | **fix-now — applied** | +| R2 | Instructions / Config | `speckit-opsmill-auto` invoked twice with a non-description argument (empty, then `"and /speckit-opsmill-retrospect"`). A literal Phase A would have opened `008-` twice while 007 was mid-remediation | Precondition: if `.specify/feature.json` names a spec dir with an incomplete `tasks.md`, resume rather than specify | declined (`open-pr`) | +| R3 | Instructions / Config | `update-agent-context.sh` appended its own sections to `CLAUDE.md` — a deliberate one-line router — truncating at the first `**` and emitting `grep: repetition-operator operand invalid` | Make the script respect a router-style agent file, or target `AGENTS.md` | declined (`open-pr`); hazard documented in `AGENTS.md` under fix-now | +| R4 | Instructions / Config | Orchestrator lint check failed with `golangci-lint: command not found` while subagents reported it running; it lives in `$(go env GOPATH)/bin` via `make dev-setup` | Note it, plus `GO_TAGS` and the darwin `neo4jwatchdog` break, in `AGENTS.md` | **fix-now — applied** | +| R5 | Documentation | `AGENTS.md` claimed `utils.go` does "version detection and comparison". It has only `SetVersion`/`BuildRevision`. Chunk 2c spent a turn hunting for code to extend. Fixed in `5d63a9c` | A drift check failing when `AGENTS.md` names a capability no symbol provides | declined (`github-issue`) | +| R6 | Documentation | F29: `postgres:18-alpine` and `neo4j:2025.10.1-enterprise` both report `Config.User` empty and drop privileges inside their entrypoints, which `Command: ["sh"]` bypasses. Chunk 2d asserted the opposite and shipped a pod the kubelet always rejects | `dev/knowledge/` page recording the measured behaviour | **fix-now — applied** | +| R7 | Documentation | T065 never ran. ADR-0007 scopes offline-by-default to `infrahub-collect`; this feature makes an image pull mandatory and load-bearing, unable to degrade to "skipped" | ADR recording the posture change | declined (`github-issue`) | +| R8 | Architectural | `podCache` — a discovery cache — used as durable execution state. Reset wholesale at `environment_kubernetes.go:192` and `kubernetes_workloads.go:116`; restore calls `StartServices` by design. Forced exclusion filters onto four enumerators | ADR for an explicit exec target | declined (`github-issue`); remains queued as remediation step 2 | +| R9 | Architectural | `NewNeo4jEditionInfo` defaults to Community on **any** detection error, and the Community branch calls `stopAppContainers()`. A transient `cypher-shell` failure against an *internal* Enterprise database stops Neo4j and dumps it cold — **a live bug for current customers, independent of 007** | Own fix, own PR | declined (`github-issue`) | +| R10 | Architectural | Every destructive decision in this tool rests on substring evidence with errors swallowed: location (fixed in `de5e022`), *what gets scaled* (`findWorkloadResource:40,54` plus `if err != nil \|\| output == "" { continue }` at `:23`), and edition (R9) | One matching rule with explicit evidence requirements | declined (`github-issue`) | +| R11 | Mistakes | FR-002, FR-011 and FR-025 were certified done from subagent unit tests. Five requirements were inert — ~470 lines, zero non-test callers, doc comments asserting calls that don't exist | Chunk contract: grep for non-test callers of every newly exported symbol, **per chunk** | declined (`open-pr`); adopted informally from the first fix chunk onward | +| R12 | Mistakes | Chunk 2d's "the official images run as uid 7474/70" was accepted without checking. `docker image inspect` falsified it in ten seconds | Chunk contract: any factual premise about an external artifact must be verified by a named command, not asserted | declined (`open-pr`) | +| R13 | Mistakes | `/code-review` was scoped to `e9008b4..HEAD` and read the **working tree**, reviewing chunk 3a's uncommitted code | Don't run review while an implementation subagent holds the tree | declined (`local-only`); hazard documented in `AGENTS.md` under fix-now | + +No separate action: the T074 mischaracterisation ("latent rather than live" — the exposure is *any* +pod matching, not only the transient pod) is subsumed by R10. + +## Applied under fix-now + +1. `AGENTS.md` — new "Spec-kit extensions: registered but only partly vendored" section: a table of + what a skill names versus what exists, plus the `update-agent-context.sh` (R3) and + `/code-review` working-tree (R13) hazards. +2. `AGENTS.md` — notes under Testing and Quality on `golangci-lint`'s install location, the + required `GO_TAGS`, and the expected darwin `neo4jwatchdog` failure. +3. `dev/knowledge/container-image-user-semantics.md` — the measured image-user behaviour, why + `runAsNonRoot` without `runAsUser` fails closed, and why overriding `Command` removes the + privilege drop those images depend on. + +## The one lesson worth carrying + +Eight chunks each self-reported 100% success with accurate, verbatim, honest test evidence. A later +review found ~31 defects, including five requirements that were completely inert. + +Unit tests written by the agent that wrote the code prove that it compiles and that the author's +assertions are self-consistent. They cannot distinguish working code from unreachable code, correct +logic wired to the wrong thing, or a guarantee that holds per-chunk and fails in composition. Two +checks would have caught most of it, both nearly free, neither in the loop: **grep for callers of +each newly exported symbol**, and **verify any premise about an external artifact with a named +command**. Both were adopted for the first fix chunk, and both immediately paid — the caller check +confirmed the new gate is reachable from `backup.go:105` and `plakar_backup.go:37`, and the premise +check confirmed the chart labels the database canonically at `tests/e2e/conftest.py`. + +## Still open, recorded here rather than actioned + +- The restore path never gates on location: `backup.go:482` and `plakar_restore.go:210/364/429/458` + call `stopAppContainers()` with no location established — the create path's fixed defect, on the + destructive path. +- `findWorkloadResource` chooses which workload to scale by substring and swallows kubectl errors + (R10). +- ~28 further review findings, listed in `opsmill-implement-report.md` §5. +- T072 (CI topology) and T073 (customer prerequisite validation) remain the real gates on the + feature. diff --git a/specs/archive/007-external-database-backup/spec.md b/specs/archive/007-external-database-backup/spec.md new file mode 100644 index 00000000..709d407c --- /dev/null +++ b/specs/archive/007-external-database-backup/spec.md @@ -0,0 +1,431 @@ +# Feature Specification: External Database Backup and Restore + +**Feature Branch**: `007-external-database-backup` + +**Created**: 2026-09-02 + +**Status**: Extracted + +**Input**: User description: "Several customers have their Neo4j database and sometimes the PostgreSQL database outside their Kubernetes cluster. Right now it is not possible to back those up because the tool is not able to find them. The prerequisite will be that the backup port needs to be open from the place we run the backup tool. What else is missing?" + +## Clarifications + +### Session 2026-09-02 + +- Q: Version compatibility rule — abort on any client/server mismatch, or asymmetric? → A: Asymmetric: abort when the client tooling is older than the server; warn when it is newer +- Q: What reclaims the stored credentials if the run is killed before it can clean up? → A: Tie the credential object's lifetime to the workload's, so the cluster deletes it automatically +- Q: SC-004 requires recording whether the source endpoint was a follower, but which endpoint served a multi-endpoint capture is not exposed by a documented interface. How should the recovery-point caveat be recorded? → A: Record the endpoint list supplied plus each member's role as observed at capture time, without claiming which one served the capture +- Q: How does the tool learn the database size needed to size scratch space before a capture? → A: Query the server for its store size before capture, apply a headroom factor, and let the flag override + +### Session 2026-10-12 (PR #172 review) + +- Q: Should the transient workload keep running the vendors' images? → A: No. The database tools ship in the `infrahub-backup` image and the workload runs that image (ADR 0017, reviewer request) +- Q: What happens when `infrahub-backup` already runs as a pod in the target namespace? → A: It runs the tools itself and creates no workload +- Q: Does redistributing Neo4j Enterprise tool binaries in the image need clearance? → A: Cleared by the PR author + +## User Scenarios & Testing *(mandatory)* + +### Context: why "cannot find them" understates the gap + +Every backup and restore path in this toolset executes *inside* the database container: the +tool resolves a deployment service name to a running container, then runs the vendor dump or +restore utility in that container and moves the resulting bytes. No code path opens a network +connection to a database. When a customer's Neo4j or PostgreSQL lives outside the deployment, +there is no container to execute in, and the run fails while resolving the service name. + +Consequently, opening a database port toward the operator does not by itself enable anything — +it only matters for a design where the tool dials the database directly. The requirements below +are written against the behaviour operators need, not against that assumed prerequisite. + +--- + +### User Story 1 - Round-trip a deployment whose databases live outside the cluster (Priority: P1) + +An operator responsible for an Infrahub deployment on Kubernetes whose Neo4j Enterprise +database — and optionally whose PostgreSQL task-manager database — runs outside the cluster +takes a complete backup with the same command they use today, and later restores it, without +performing manual steps on the database hosts. + +**Why this priority**: These customers have no backup or recovery path through this toolset at +all today. A backup that cannot be restored would be a compliance artefact rather than a +recovery capability, so the round trip is one indivisible slice — recoverability is the product +(Constitution, Principle II). + +**Independent Test**: Stand up an Infrahub deployment on Kubernetes with Neo4j Enterprise and +PostgreSQL running outside the namespace, take a backup, destroy the data, restore it, and +verify Infrahub serves the pre-backup data. Delivers the entire value of the feature on its own. + +**Acceptance Scenarios**: + +1. **Given** a deployment whose Neo4j Enterprise database runs outside the cluster and whose + PostgreSQL runs inside it, **When** the operator takes a backup, **Then** the tool reports + which endpoints each database is read from before reading any data, and produces a single + backup artefact covering both databases. +2. **Given** a deployment whose Neo4j Enterprise and PostgreSQL databases both run outside the + cluster, **When** the operator takes a backup, **Then** both databases are captured and the + artefact records, per database, the endpoints supplied for it (FR-005). +3. **Given** a backup artefact taken from external databases and an operator who has explicitly + authorised restore into externally-managed databases, **When** the operator restores it, + **Then** Infrahub workloads are quiesced first, both databases are restored, the workloads + are returned to their prior scale, and Infrahub serves the restored data. +4. **Given** the backup utility available to the tool is incompatible with the live database + server version, **When** the operator takes a backup, **Then** the run fails with a message + naming both versions and no backup artefact is produced. +5. **Given** an operator who has not authorised restore into externally-managed databases, + **When** they attempt such a restore, **Then** the tool refuses and names the authorisation + it requires, leaving the databases untouched. +6. **Given** a deployment whose databases are all inside the cluster, **When** the operator + takes a backup or restore, **Then** behaviour and artefact layout are unchanged from today. + +--- + +### User Story 2 - Diagnose an unsupported or misconfigured external deployment (Priority: P2) + +An operator whose deployment cannot be backed up — because the database is a Neo4j edition with +no remote-capable mechanism, because the endpoint cannot be discovered, or because the tool +lacks the cluster permissions it needs — learns precisely which condition applies and what to +change, instead of receiving a service-resolution failure. + +**Why this priority**: Today's failure message names a missing service, which sent at least one +customer to the wrong conclusion ("the tool cannot find my database"). Every condition below is +reachable in the field, and each has a different remedy. This ships independently of P1 and +raises the floor for customers P1 will never serve. + +**Independent Test**: Provoke each condition in turn against a test deployment and assert the +message names the condition and the operator-actionable remedy. + +**Acceptance Scenarios**: + +1. **Given** an external Neo4j Community database, **When** the operator takes a backup, + **Then** the tool refuses with a message stating that the Community backup mechanism requires + direct access to the database's own storage and is therefore unavailable for an externally + hosted database. +2. **Given** a deployment where no database container is present and no endpoint can be + discovered from the deployment's own configuration, **When** the operator takes a backup, + **Then** the tool reports that the endpoint could not be determined and names the override + available to supply it. +3. **Given** a deployment where the tool cannot query the cluster for containers because + permission is denied, **When** the operator takes a backup, **Then** the run fails reporting + the permission problem and does **not** fall back to treating the database as external. +4. **Given** a deployment where the tool lacks permission to create the workload it needs to + reach an external database, **When** the operator takes a backup, **Then** the run fails + naming the missing permission. +5. **Given** a deployment with one or both databases outside the cluster, **When** the operator + runs environment detection — the first command an operator reaches for when a backup fails — + **Then** the report states, per database, that it is external and where it was resolved to. + +--- + +### User Story 3 - Run the backup from the operator's own host (Priority: P3) + +An operator whose cluster cannot grant the tool permission to create workloads, but who can +reach the database directly from their own machine, takes a backup without the cluster's +involvement. + +**Why this priority**: This is the shape the original request assumed, and it remains the only +option where the required cluster permissions are unavailable. It is P3 because it makes a +self-contained binary depend on a version-matched database toolchain being installed on the +operator's host — which conflicts with the tools being copied onto hosts that have no package +manager (Constitution, Principle III) — and because it requires the database to accept inbound +connections from wherever the operator sits, which is the harder ask in practice. + +**Independent Test**: With the database reachable from the operator's host and the required +utilities installed, take and restore a backup with no cluster-side workload created. + +**Acceptance Scenarios**: + +1. **Given** an operator host with the required database utilities present and network reach to + an external Neo4j Enterprise database, **When** the operator takes a backup in this mode, + **Then** the artefact is byte-compatible with one taken via P1 and restorable by either path. +2. **Given** an operator host missing a required database utility, **When** the operator takes a + backup in this mode, **Then** the tool fails before contacting the database, naming the + missing utility. + +--- + +### Edge Cases + +- **Highly available Neo4j exposing several endpoints**: the tool must accept and use the full + set of discovered endpoints rather than assuming one. Any of them may be a cluster follower, so + the artefact can lag the write leader by the replication window. Since which member served the + capture is not knowable, the artefact records the endpoints supplied and the roles observed at + capture time — enough for an operator to see that a follower could have served it, rather than + discovering the possibility at restore time. +- **Mixed topology with partial failure**: one database external and captured, the other + in-cluster and failed (or vice versa). A partial capture must never be presented as a complete + restore point. +- **A run killed mid-flight**: any workload the tool created to reach an external database holds + database credentials and must not survive the run, including when the tool never regains + control. +- **Credentials leaving the cluster boundary**: the credential for an externally-managed database + must not be observable to anyone who can merely inspect the workload or its process list. +- **A restore artefact the database itself cannot reach**: the restore path requires the database + server to fetch the artefact. If the database host has no route to the artefact store, restore + cannot proceed and must say so before any destructive step. +- **Unattended restore**: the scheduled restore capability already shipped can reach this new + destructive path without an operator present. +- **Repeated or concurrent runs**: two runs against the same deployment must not adopt each + other's transient workloads or leave the deployment quiesced. + +## Requirements *(mandatory)* + +### Functional Requirements + +**Discovery and mode selection** + +- **FR-001**: The tool MUST determine independently, for each database it backs up, whether that + database is inside or outside the deployment, so that a deployment with one of each is + supported without additional operator input. +- **FR-002**: The tool MUST treat an external database as external only when it has positively + established that no in-deployment database is present. A failure to query the deployment + (permission denied, unreachable control plane, misidentified deployment) MUST surface as an + error and MUST NOT be treated as evidence that the database is external. +- **FR-003**: The tool MUST discover an external database's endpoint and credentials from the + configuration already present in the deployment's running components, and MUST report the + endpoint it resolved before opening any connection to it. +- **FR-004**: Operators MUST be able to override the discovered endpoint for deployments whose + configuration does not expose it. +- **FR-005**: The tool MUST accept multiple endpoints for a single logical database, and MUST + record the endpoints it supplied together with each member's role as observed at capture time. + It MUST NOT claim which member served the capture: with several endpoints supplied, that is not + exposed by any documented interface, and asserting it would be a guess presented as provenance. + +**Safety before data movement** + +- **FR-006**: The tool MUST determine the live server version before reading or writing any data, + and compare it against the version of the utility it will use. Where the utility is **older** + than the server it MUST abort, naming both versions. Where the utility is **newer** it MUST warn + and continue. Version comparison MUST handle both the `5.x.y` and calendar (`YYYY.MM.p`) + numbering schemes, since ordering them as if they were one scheme misorders them silently. +- **FR-007**: The tool MUST verify, before any destructive restore step, that the database server + can itself read the restore artefact, and MUST abort leaving the database untouched if it + cannot. +- **FR-008**: The tool MUST refuse an external Neo4j Community backup with a message stating why + the mechanism is unavailable, rather than reporting a missing component. + +**Destructive-operation authorisation** + +- **FR-009**: Restoring into a database the deployment does not manage MUST require an + authorisation that is distinct both from the existing restore override and from enabling + scheduled restore, so that neither confers this capability implicitly. +- **FR-010**: An unattended restore into an externally-managed database MUST be possible only + when that authorisation has been supplied through configuration. + +**Lifecycle and integrity** + +- **FR-011**: Any transient workload the tool creates in order to reach an external database MUST + be removed on every exit path, including failure and abnormal termination, and MUST be bounded + so the deployment reclaims it even if the tool never returns. +- **FR-023**: Any object holding external database credentials MUST have its lifetime bound to + that of the transient workload, so the deployment reclaims it by the same mechanism rather than + by the tool remembering to. Credentials MUST NOT be able to outlive the run that created them, + including when the tool is killed before it can clean up. Verification: kill the run between + credential creation and workload deletion, then assert no credential object remains once the + workload is gone. +- **FR-012**: An incomplete capture MUST NOT be retained. Either every requested database is + captured, or the run fails and the partial artefact is removed from every location it reached. + Completeness MUST be determined from what the backup operation reported, not from its exit status + alone, because a backup that succeeded against only some of several endpoints is not + distinguishable by exit status from one that failed outright. Retention and restore-point + selection MUST NOT be required to interpret artefact metadata in order to honour this: an + incomplete artefact never persists for them to consider. Verification: fail one component, assert + the run fails, no artefact remains at any location, and the retention policy's decisions are + unchanged from a run where the failure had happened before any artefact was written. +- **FR-021**: After a restore, the tool MUST confirm the restored database has reached a usable + state before reporting success, and MUST report failure otherwise. An acknowledgement that the + restore was accepted is not evidence that the data loaded. Verification: seed from a corrupt + artefact, assert the run fails rather than reporting success. +- **FR-022**: Scratch space available to the backup operation MUST be explicitly sized to at + least the size of the database being captured. The tool MUST determine that size by querying the + server before the capture begins and applying a headroom factor, and operators MUST be able to + override the result. Where the size cannot be determined and no override was supplied, the tool + MUST fail naming the override rather than guess a size. Verification: capture a database larger + than the default scratch allocation and assert the run either succeeds or fails cleanly without + exhausting shared storage on the host it runs on; separately, make the size query fail and + assert the run stops with the override named. +- **FR-013**: The tool MUST restore quiesced Infrahub workloads to their prior scale on every + exit path of a restore, including failure. +- **FR-014**: Credentials for an externally-managed database MUST NOT be exposed on a process + command line or in workload configuration readable by deployment observers. + +**Compatibility** + +- **FR-015**: Backups of deployments whose databases are all internal MUST be unchanged in + behaviour and artefact layout. +- **FR-016**: Additions to backup metadata MUST be additive, so that artefacts produced by + previously released versions remain restorable without operator action. +- **FR-024**: Metadata additions MUST NOT change the artefact's declared metadata version, and + MUST be safe for a previously released version of the tool to ignore. Compatibility MUST hold in + both directions: a new tool reads an old artefact, and an old tool reads a new one. Verification: + pin both directions with tests — an older tool must neither refuse a new artefact nor act + incorrectly on the fields it cannot see. This is only safe because FR-012 guarantees an + incomplete artefact never reaches a reader. +- **FR-025**: Every operation against an external database MUST be time-bounded, with an + overridable limit. A database that accepts a connection and then stops responding MUST NOT be + able to hang the run indefinitely — particularly on the restore path, where Infrahub is scaled + down while the operation is in flight. Verification: point the tool at an endpoint that accepts + and then stalls; assert the run terminates and Infrahub is returned to its prior scale. +- **FR-026**: Before the destructive step of a restore, the tool MUST confirm that Infrahub + workloads have actually stopped, not merely that a scale-down was requested. After the restore, + Infrahub MUST serve the restored data once workloads return to their prior scale, with no manual + intervention. Verification: assert data is served after scale-up without an additional restart. +- **FR-027**: The transient workload MUST declare a security context that satisfies a restricted + pod-security policy, and its image MUST come from a pinned default or an explicitly supplied + flag — never from ambient environment alone, since the tool holds workload-creation rights and + an environment-settable image would make those rights a means of running arbitrary images. + Verification: run against a namespace enforcing restricted pod security and assert the workload + is admitted. +- **FR-028**: The transient workload MUST NOT be discoverable as a replica of the service it + stands in for. Registering it so that execution resolves to it MUST NOT make it appear in + service enumeration, replica counts, or diagnostic collection. Verification: assert service + enumeration does not return it while it exists. +- **FR-029**: Connections to an external database MUST verify the server's certificate unless the + operator explicitly opts out. An insecure-TLS setting discovered from the deployment MUST NOT + silently disable verification for credentials the tool sends. +- **FR-017**: The tool MUST NOT introduce a new assumed deployment service name; external + databases MUST be addressed under the existing documented service names. +- **FR-018**: The tool MUST detect the absence of any external runtime it depends on and fail + with a message naming what is missing and where it was expected. +- **FR-019**: Permission to create and remove a transient workload in the deployment is a + documented prerequisite of User Story 1. The tool MUST NOT attempt to work around its absence, + and MUST fail naming the missing permission (see User Story 2, scenario 4). +- **FR-020**: A route from the database server to the artefact store is a documented prerequisite + of restore. Where it is absent, the tool MUST refuse before any destructive step (FR-007) and + MUST name the prerequisite. The tool MUST NOT offer a partial restore that leaves the two + databases at different recovery points. + +**Database tools in the infrahub-backup image (amendment 2026-10-12, ADR 0017)** + +- **FR-030**: The `infrahub-backup` container image MUST carry the database tools the external + path runs: `neo4j-admin` and `cypher-shell` at the Neo4j version previously pinned for the + vendor image (`2025.10.1`, Enterprise), and `pg_dump`, `pg_restore` and `psql` for PostgreSQL 18. + The `Dockerfile` MUST pin these versions, so that one image tag has one known tool set. +- **FR-031**: The transient workload MUST run the `infrahub-backup` image whose tag matches the + running binary's version. A single flag, `--external-db-image`, MUST replace + `--external-db-image-neo4j` and `--external-db-image-postgres`, and MUST stay unbound from the + environment for the reason FR-027 gives. A binary whose version has no published image (a + development build) MUST refuse the external path before creating anything, naming + `--external-db-image`, unless that flag is set. The refusal applies only to a run that needs a + transient workload: direct mode (FR-032) runs no image and is not refused. +- **FR-032**: When `infrahub-backup` itself runs as a pod in the target namespace and finds the + tools it needs on its own `PATH`, it MUST run them in its own process and MUST create no + transient workload and no Secret. Credentials MUST reach those tools only through the child + process's environment, never its arguments (FR-014). Every other requirement on the external + path (version gate, edition gate, scratch sizing, incomplete-capture removal, restore + authorisation) MUST apply unchanged. In any other case the tool MUST use the transient workload. +- **FR-033**: The transient workload's security context MUST name the user the `infrahub-backup` + image declares, so that a restricted pod-security policy admits it without the uid exceptions + the vendor images needed. + +### Key Entities + +- **External database endpoint**: the network location, protocol, and credentials of a Neo4j or + PostgreSQL database that serves the deployment but is not part of it. Discovered from the + deployment's own configuration; may be a set rather than a single value. +- **Transient backup workload**: a short-lived execution context the tool creates inside the + deployment for the sole purpose of reaching an external database endpoint. Owned by exactly one + run, identifiable as belonging to that run, and guaranteed not to outlive it. +- **Backup artefact** (existing): gains, per database, the endpoints supplied for it, each + member's role as observed at capture time, and whether the capture is complete. It does not + record which member served the capture (FR-005). +- **Restore authorisation** (new): an explicit operator grant permitting destructive restore into + a database the deployment does not manage. Separate from the existing restore override. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: An operator with an external Neo4j Enterprise database and an external PostgreSQL + database completes a full backup and restore round trip using the documented commands, with + zero manual steps performed on either database host beyond the documented one-time + prerequisites. +- **SC-002**: Taking a backup requires no database port to be reachable from outside the + deployment's own network boundary. Verified by completing a successful backup with the database + reachable only from within that boundary. +- **SC-003**: 100% of conditions that would yield an unusable artefact — version incompatibility, + an artefact the database cannot read, missing permissions, an unsupported database edition — + are reported before any data is read or written. No test case produces an artefact that first + fails at restore time. +- **SC-004**: Every backup states the endpoints it was taken from and the role each held at + capture time, so an operator can judge the recovery point without inspecting the database — and + can see when a follower may have served it rather than being left to assume it did not. +- **SC-005**: A destructive restore into an externally-managed database cannot occur unless an + operator has explicitly authorised that specific capability; enabling scheduled restore alone + never confers it. +- **SC-006**: No run leaves a transient workload, or a quiesced Infrahub deployment, behind — + including runs terminated abnormally. Verified across the full set of interruption points + exercised by the end-to-end suite. +- **SC-007**: Every existing backup and restore scenario for fully-internal deployments passes + unchanged, and artefacts from the previous released version restore without operator action. +- **SC-008**: An operator hitting an unsupported configuration can identify the remedy from the + error message alone, without reading source or filing a support request. + +## Assumptions + +- **Reachability is already proven in the direction that matters.** The deployment reaches the + external database today, because Infrahub is running against it. The P1 design depends on that + existing path and requires no new network permission at the database for backup. +- **Documented one-time prerequisites on the customer's side** (referenced by SC-001; established + by Phase 0 research and not assumed): + 1. The Neo4j backup service must listen on a non-loopback address. It is enabled by default + but bound to localhost, so backups taken from anywhere other than the database host itself + require a server configuration change — not merely a firewall rule, which is what the + original request assumed was sufficient. + 2. Each Neo4j server must be able to retrieve the backup artefact from the object store, which + for the current seed mechanism means having the cloud provider's command-line tooling + installed and credentials configured server-side. Object stores are supported without any + additional server-side provider configuration; other artefact sources are not. + 3. If the backup service requires TLS, the tool must be given a compatible client policy. +- **The deployment knows where its databases are — but not everything needed.** Components still + running inside the deployment carry the external host list and credentials in their + configuration, and already express a multi-member database as a comma-separated host list, so + discovery needs no new operator input for those. It cannot supply the backup service's port, + because Infrahub never uses that port and does not record it; that value is defaulted and + overridable. FR-004 covers both exceptions. +- **Scope is Kubernetes only for v1.** Docker Compose deployments with external databases are out + of scope; the same reasoning applies to them and can be added later. +- **External Neo4j Community is out of scope, permanently for this feature.** Its backup + mechanism requires the database to be stopped and its storage directly accessible, which no + network-reachable design can satisfy. Community deployments remain on the in-deployment path. +- **Neo4j Enterprise is the target edition**, consistent with the customers requesting this. +- **A remote restore mechanism exists for Neo4j Enterprise.** The restore design assumes the + server can be directed to seed a database from an artefact it retrieves itself. This is + load-bearing for User Story 1 and MUST be verified against the target server versions during + planning research before implementation begins. +- **Backup utilities tolerate version skew in one direction.** A newer utility reads an older + server; the reverse fails. For PostgreSQL this is documented and the failure is a loud refusal + from the utility itself. For Neo4j no client/server matching rule is documented at all, so + FR-006 enforces only the direction with a known failure mode and warns on the other. The exact + tolerated envelope is measured by the end-to-end suite rather than asserted here — encoding a + guessed rule in an abort condition would make the tool refuse work it could have done. +- **Operators know their database version** and can declare it, so the tool need not derive it + before it has any means of asking the server. +- **Highly available external databases are in scope for backup** (FR-005), but measuring or + bounding replication lag is not; the tool records the source endpoint and leaves the recovery + point interpretation to the operator. +- **Customers will grant permission to create a transient workload** in the Infrahub namespace. + This is the pivot of User Story 1 and is assumed rather than proven. It is a modest grant + relative to one the tool already exercises — restore scales Infrahub workloads up and down — + which is the basis for assuming it. If a customer refuses, they are served by User Story 3, not + by a fallback inside User Story 1. +- **Running the dump utility on the operator's host is deferred, not rejected** (User Story 3). + It stays specified so the permission assumption above has a defined consequence when it fails, + rather than leaving the feature with no answer. +- **The database host can reach the artefact store.** For customers already backing Infrahub up + to an object store, this is likely to be true incidentally. It is nonetheless a hard + prerequisite of restore, and a locked-down database estate cannot be served by v1. An + operator-staged artefact — where the operator places the artefact somewhere the database can + read it and the tool drives the restore — was considered and left out of v1: it would break + User Story 1's promise of no manual database-side steps, so adding it later means revising that + story rather than extending it. +- **Partial restore is not an escape hatch.** Restoring only PostgreSQL where Neo4j cannot be + reached was considered and rejected: it leaves the two databases at different recovery points, + which is a worse outcome than refusing, and it re-opens the backup-only shape that was already + ruled out on the grounds that a round trip which does not close is a compliance artefact rather + than a recovery capability. +- **This does not reverse spec 006.** That work established that Kubernetes support depends on + execution staying in the tool, with vendor utilities run inside a container and bytes streamed + out through primitives both deployment backends implement. This feature reuses exactly that + model and only supplies a container to execute in when the deployment has none. Spec 006 + separately declined a container-hosted *snapshot writer*; that is a different layer and remains + declined. diff --git a/specs/archive/007-external-database-backup/tasks.md b/specs/archive/007-external-database-backup/tasks.md new file mode 100644 index 00000000..2a258b00 --- /dev/null +++ b/specs/archive/007-external-database-backup/tasks.md @@ -0,0 +1,477 @@ +# Tasks: External Database Backup and Restore + +**Input**: Design documents from `/specs/007-external-database-backup/` + +**Prerequisites**: plan.md, spec.md, research.md, data-model.md, contracts/, quickstart.md + +**Tests**: Included. Every functional requirement in spec.md carries an explicit `Verification:` +clause, and Constitution Principle IV requires behaviour changes to ship with table-driven unit +tests plus end-to-end coverage for flows that touch containers. Tests are therefore not optional +here. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies on incomplete work) +- **[Story]**: `[US1]` round trip (P1), `[US2]` diagnosability (P2) +- File paths are exact + +## Scope notes + +- **User Story 3 (operator-host execution) is deferred** and generates no tasks. It stays specified + as the defined consequence if the transient-workload permission assumption fails. +- **Two tasks cannot be completed on a developer machine** and are marked `[INFRA]`: T072 (CI + topology) and T073 (customer prerequisite validation). They gate *finishing* the feature, not + starting it. See the critique's X1 and E7. +- The design's success indicator is that the ~93 call sites naming `database` and `task-manager-db` + are **not** edited. Any task that would edit them is a signal the stand-in approach has leaked. +- **Phase 3 was implemented and then reverted** (`4de8d3c`), then rebuilt on the corrected + foundation. Its code was written against three decisions since shown wrong — merged stdout/stderr, + `podCache` routing, and edition-by-container-exec — so Phase 2S landed first. The rebuild names the + exec target with `ExecOptions.Pod`, parses stdout alone, and reads an external server's edition over + Bolt at the gate rather than by exec'ing a container that is not there. +- **A chunk's passing tests do not mean the code runs.** Five requirements in this feature had full + unit coverage and zero non-test callers, and the transient pod could never have been admitted by a + kubelet. Before reporting any task done: grep for a non-test caller of every symbol it adds, and + verify any premise about an external artefact (a container image, a vendor CLI, an API) with a + named command rather than asserting it. + +--- + +## Phase 1: Setup + +- [X] T001 Register the new flags and their viper bindings in `src/internal/app/cli.go` per `contracts/cli-surface.md`: `--neo4j-address`, `--neo4j-backup-port`, `--postgres-address`, `--external-db-scratch-size`, `--external-db-tls-policy`, `--external-db-timeout`, `--allow-external-restore` +- [X] T002 Register `--external-db-image-neo4j` and `--external-db-image-postgres` in `src/internal/app/cli.go` **without** viper environment binding, and add a comment stating why (FR-027: the tool holds workload-creation rights, so an environment-settable image would make those rights a means of running arbitrary images) +- [X] T003 [P] Add the external-database fields to `Configuration` in `src/internal/app/app.go` (host list, backup port, protocol, TLS settings, images, scratch size, timeout, external-restore authorisation) +- [X] T004 [P] Add `INFRAHUB_ALLOW_EXTERNAL_RESTORE` handling so unattended restore can be authorised through configuration only (FR-010), in `src/internal/app/cli.go` + +**Checkpoint**: `make build` succeeds; new flags appear in `--help`; no behaviour change yet. + +--- + +## Phase 2: Foundational (blocking prerequisites for both stories) + +Nothing in Phase 3 or 4 can proceed until this phase is complete. This is where the feature's two +genuinely new concepts live. + +### Endpoint resolution + +- [X] T005 Create `src/internal/app/external_endpoint.go` with the `DatabaseEndpoint` and `HostPort` types from `data-model.md`, including the rule that a member without an explicit port inherits the client port +- [X] T006 Implement parsing of a comma-separated `host[:port]` list in `src/internal/app/external_endpoint.go`, matching how Infrahub itself interprets `INFRAHUB_DB_ADDRESS` +- [X] T007 [P] Write table-driven tests for list parsing in `src/internal/app/external_endpoint_test.go`: single host, host with port, mixed, whitespace, empty, malformed, IPv6 literal +- [X] T008 Extend `fetchNeo4jCredentials` in `src/internal/app/app_config.go` to read `INFRAHUB_DB_ADDRESS`, `INFRAHUB_DB_PORT`, `INFRAHUB_DB_PROTOCOL` and the `INFRAHUB_DB_TLS_*` settings, which it currently ignores +- [X] T009 Extend `applyPrefectConnection` in `src/internal/app/app_config.go` to keep `connConfig.Host` and `connConfig.Port`, which are parsed today and discarded +- [X] T010 [P] Add tests in `src/internal/app/app_config_test.go` asserting the address, port, protocol and TLS settings are read, and that a deployment setting none of them behaves exactly as before (FR-015) +- [X] T011 Default the backup port to 6362 with flag override in `src/internal/app/external_endpoint.go`, with a comment recording that it is not discoverable because Infrahub never uses it (research R4) +- [X] T012 Implement endpoint reporting so the resolved endpoint is logged before any connection is opened (FR-003), in `src/internal/app/external_endpoint.go` + +### Internal-versus-external decision + +- [X] T013 Give `getPodForService` in `src/internal/app/environment_kubernetes.go` the same `errNoPodsMatched` discrimination its plural sibling already has, so a genuinely empty match is distinguishable from a kubectl, RBAC or API failure +- [X] T014 Implement per-service resolution in `src/internal/app/external_endpoint.go` so `database` and `task-manager-db` resolve independently (FR-001), engaging external mode only on a genuinely empty match (FR-002) +- [X] T015 [P] Add tests in `src/internal/app/environment_kubernetes_test.go` asserting a pod-list permission failure surfaces as an error and does **not** engage external mode +- [X] T016 [P] Add tests in `src/internal/app/external_endpoint_test.go` for the mixed topology: one database external, one internal, in both combinations + +### Version comparison + +- [X] T017 Implement version comparison in `src/internal/app/external_endpoint.go` handling both `5.x.y` and calendar `YYYY.MM.p` schemes, ordering them correctly rather than as one scheme +- [X] T018 [P] Write table-driven tests in `src/internal/app/external_endpoint_test.go` covering within-scheme ordering, across-scheme ordering, and the pre-release forms the project already encounters +- [X] T019 Implement the asymmetric compatibility gate (FR-006): abort when the utility is older than the server naming both versions, warn and continue when newer + +### Transient workload lifecycle + +- [X] T020 Create `src/internal/app/transient_workload.go` with the `TransientWorkload` type from `data-model.md` +- [X] T021 Build the pod manifest in `src/internal/app/transient_workload.go` and apply it through the executor's existing stdin write-pipe primitive — no Kubernetes client library (ADR-0001, Principle III) +- [X] T022 Set the run-ID label, `restartPolicy: Never`, and `activeDeadlineSeconds` on the manifest so the cluster reclaims the pod even if the tool never returns (FR-011) +- [X] T023 Declare a security context satisfying a restricted pod-security policy, reconciled with the vendor's requirement that the backup tooling run as the database's own user (FR-027) +- [X] T024 Size the pod's ephemeral storage from the queried store size plus a headroom factor, failing with the override named when the size cannot be determined (FR-022) +- [X] T025 Create the credential secret with an owner reference to the pod, so the cluster's garbage collector reclaims it when the pod goes (FR-023), in `src/internal/app/transient_workload.go` +- [X] T026 Wait for readiness before returning the workload as an execution target, and register it in `podCache` under the canonical service name (FR-017) — **superseded by T084**: the pod is named on each operation via `ExecOptions.Pod` instead, because `Start` and `scaleServices` replace `podCache` wholesale and a restore calls `StartServices` by design +- [X] T027 Exclude the transient pod from `GetAllPods`, `getAllPodsWith` and `ServiceReplicas` in `src/internal/app/environment_kubernetes.go`, so registering it for execution does not make it discoverable as a replica (FR-028) — the shared resolvers fall back to substring matching on pod names, so this must be explicit +- [X] T028 Implement reaping: delete on every exit path, and reap strays labelled by a previous run at startup (FR-011), in `src/internal/app/transient_workload.go` +- [X] T029 Ensure a run never adopts another run's workload (`data-model.md` invariant) +- [X] T030 [P] Add tests in `src/internal/app/transient_workload_test.go` for manifest construction, label and deadline presence, secret owner reference, and reaping selection by run ID +- [X] T031 [P] Add a test in `src/internal/app/environment_kubernetes_test.go` asserting `GetAllPods("database")` does not return the transient pod while it exists + +### Timeout bounding + +- [X] T032 Route every external-database operation through the timeout-bounded runner rather than the unbounded `runCommand`, in `src/internal/app/external_endpoint.go` and the call paths it serves (FR-025) — today `iops.Exec` is unbounded, and the 30-minute guard in `backup_neo4j.go` covers only the data stream, not the backup command +- [X] T033 [P] Add a test asserting an endpoint that accepts and then stalls terminates the operation at the bound + +**Checkpoint**: endpoint resolution, the internal/external decision, version comparison and the +workload lifecycle are all unit-tested. No user-visible behaviour has changed yet. + +--- + +## Phase 2R: Remediation — defects that ship in the current release + +Neither of these is caused by this feature. Both predate the branch, fire on deployments that have +nothing to do with external databases, and each belongs in its own PR against `main` rather than +inside this feature. They are listed first because blast radius, not feature order, decides priority. + +- [X] T075 Replace the `strings.Contains` matching in `findWorkloadResource` (`src/internal/app/kubernetes_workloads.go:40,54`) and `getPodStatuses` (`src/internal/app/environment_kubernetes.go:264`) with the token-boundary rule, and stop swallowing kubectl errors at `kubernetes_workloads.go:23` — today a namespace holding `redis-cache`, `othertool-cache-0` or `rabbitmq-message-queue` makes `stopAppContainers` (`src/internal/app/backup_containers.go:181`) report a foreign workload as Infrahub's, stop it, and scale it to zero on every Kubernetes backup +- [X] T076 Stop `NewNeo4jEditionInfo` (`src/internal/app/backup_metadata.go:44`) defaulting to Community when detection merely *errored*, and make the Community branch's stop/start conditional on a determined edition — a transient `cypher-shell` failure against an **internal** Enterprise database currently routes to the offline path, stopping Neo4j and dumping it cold + +**Checkpoint**: both fixes are independently shippable against `main` and need no part of this feature. + +--- + +## Phase 2S: Remediation — this branch cannot be built on until these land + +Every task below is a verified finding from the `max` review (see `opsmill-implement-report.md` §5 and +the four root causes in the retrospective). Phase 3 onward is **blocked** on this phase: the reverted +capture path was written against three decisions now known wrong, and rebuilding it first would +recreate the same defects. + +### Blockers — the feature cannot run at all + +- [X] T077 Set `runAsUser` per service in `src/internal/app/transient_workload.go` to the uid `databaseFSGroup` already computes, **and** delete the assertion at `src/internal/app/transient_workload_test.go:174` that requires it to be absent. Both pinned images report an empty `Config.User` and drop privileges inside entrypoints that `Command: ["sh"]` replaces, so the kubelet rejects every pod with `CreateContainerConfigError`. Decide and document what happens for an operator-supplied image whose uid differs. These two changes MUST land together — the test currently enforces the defect +- [X] T078 Fix the internal/external decision in `locateServiceWith` (`src/internal/app/environment_kubernetes.go:400`), which has three independent triggers that each declare a working deployment external and hard-refuse the run: label evidence only (breaks kustomize, hand-rolled manifests, and CloudNativePG — pods `-1` labelled `cnpg.io/cluster`, a topology `findPrimaryPod` explicitly supports), live pods only (a scaled-to-zero StatefulSet or a `Failed`-phase pod mid-recreate), and no floor on the control bound. Absence of a label is not evidence the database is absent +- [X] T079 Reject a unit-less `INFRAHUB_EXTERNAL_DB_TIMEOUT` in `src/internal/app/cli.go:193` and floor the resolved bound in `src/internal/app/external_timeout.go:74`. `viper.GetDuration` reads `3600` as **nanoseconds**, the `<= 0` guard lets 3.6µs through, and every Kubernetes backup then aborts — internal deployments included — telling the operator to raise a flag they never passed. The flag channel rejects the same value; only the config channel FR-010 requires is exposed +- [X] T080 Gate all three restore entry points on location — `RestoreBackup` (`src/internal/app/backup.go:306`), `RestoreLatestBackup` (`src/internal/app/restore_latest.go:168`), `RestorePlakarBackup` (`src/internal/app/plakar_restore.go:25`) — placing the gate **before** `stopAppContainers()` at `backup.go:482`, because `ensureNeo4jDiscovery`/`ensurePostgresDiscovery` exec `env` in pods that call has just scaled to zero. Today restore wipes transient data, scales six services down, half-restores Postgres, then runs `neo4j-admin database load --overwrite-destination=true` inside a name-matched pod +- [X] T081 Stop adopting `pgx`'s resolved host in `applyPrefectEndpoint` (`src/internal/app/app_config.go:356`) and stop marking discovery complete on an unparseable URL (`app_config.go:83`). `pgx.ParseConfig` fills unspecified values from the operator's own `PGHOST`/`PGPORT`, so a URL without a host can make the "discovered external database" the operator's own machine — which the restore path would write into + +### Root causes — fixing these closes most of the remaining findings + +- [X] T082 Unify workload identity on one rule in `src/internal/app/kubernetes_pod_match.go`: one matcher, one resolver, one terminal-phase predicate. Delete `transientPodIsTerminal` (`src/internal/app/transient_workload.go:848`), which duplicates `phaseIs` and differs on whitespace trimming while deciding whether the reaper deletes another run's pod. Apply the transient exclusion at every enumerator, not one of four, and correct `getAllPodsWith`'s "single choke point" comment. Closes the `IsRunning`/`locateService` contradiction, T074, and F30 +- [X] T083 Make reachability a property of the resolved target — one `databaseTarget(service)` in `src/internal/app/external_endpoint.go` that every path resolves through before exec'ing — replacing the per-operation gate a new entry point can forget to call (which is how T080's three paths were missed) +- [X] T084 Replace the `podCache` registration at `src/internal/app/transient_workload.go:1293` with an explicit exec target: add an optional `Pod` field to `ExecOptions` (already threaded through 4 of the 6 backend methods, so zero existing call sites change) plus a targeted copy variant for `CopyTo`/`CopyFrom`. `Start` and `scaleServices` clear the whole cache, and restore calls `StartServices` by design. Keep the change additive — `infrahub-collect` shares this backend +- [X] T085 Parse probe payloads from stdout only, via `ExecSeparateContext`, in `src/internal/app/external_timeout.go:209`. `execBounded` uses merged output while `parseCypherScalar`, `extractVersionToken`, `parseNeo4jMemberRoles` and `parseStoreSizeValue` read it as data, so one `Defaulted container` notice corrupts the parse. Also fix `parseCypherScalar` (`src/internal/app/external_endpoint.go:1177`) to skip the first non-empty line rather than index 0 + +### Correctness leaves + +- [X] T086 Make the PostgreSQL arm of `checkVersionCompatibility` major-only in `src/internal/app/external_endpoint.go:786`. `pg_dump` sets `maxRemoteVersion = (PG_VERSION_NUM/100)*100 + 99`, so 18.1 accepts every 18.x server; the current field-by-field compare aborts on `18.1` vs a provider-patched `18.2` and names a flag that cannot fix it +- [X] T087 Derive the capture pod's deadline from `externalDBBound(cfg)` plus `externalDBWorkloadReadyTimeout` in `src/internal/app/transient_workload.go:1164`, matching its `externalDBProbeDeadline` sibling, and budget the probe deadline for the 3+N bounded calls the probe actually issues +- [X] T088 Track the workload before `ops.create` in `src/internal/app/transient_workload.go:1237`, and make `Release` (`:1364`) delete before marking released so a failed delete is retried rather than skipped by every later sweep +- [X] T089 Fix three numeric guards in `src/internal/app/transient_workload.go`: the `time.Duration(DeadlineSeconds) * time.Second` overflow at `:913` that makes any pod look expired (letting the reaper delete a live peer's Running pod), the `bytes > math.MaxInt64` float comparison at `:674`, and the `scratch + int64(1<<30)` wrap at `:425`. Normalise `resolveScratchSize`'s return through `formatGiB` at `:603` so a value the looser parser accepted cannot be rejected by the API server +- [X] T090 Wire `ReapTransientStrays`, `ReleaseTransientWorkloads` and `ExecutionTarget` to the call sites their doc comments assert, or delete them until the capture path lands. All three have zero non-test callers, so FR-011's teardown is on no exit path and the `workloadStateReady` precondition is never checked. **Done as neither**: wiring needs the capture path (T037), which is blocked with Phase 3, and deleting would discard the counterpart T084 landed. The three doc comments now state that they have no production caller and name T037 as the task that will wire them; the zero-caller status is unchanged and still true +- [X] T091 Stop `execBoundedAgainst` spending the same bound on pod resolution and again on the exec (`src/internal/app/external_timeout.go:221`), so one operation cannot run for twice the bound its own error message reports +- [X] T092 Seed the resolution cache from `locateServiceWith` (`src/internal/app/environment_kubernetes.go:404`), which learns the pod name and discards it — costing ~8 duplicated `kubectl get pods` per run, on every Kubernetes backup including all-internal ones +- [X] T093 Add coverage for the three requirements no task names — FR-016 (artefact compatibility), FR-018 (external-runtime absence detection) and FR-020 (artefact reachability prerequisite) — and pin FR-024's old-tool-reads-new-metadata direction with a test +- [X] T094 Stop `cli_test.go`'s helper calling `ConfigureRootCommand` repeatedly while `cobra.OnInitialize` appends to package-global state that cannot be reset +- [X] T096 Fix the pre-existing timing race in `TestRunCommandCombinedPipeContext_Timeout` (`src/internal/app/command_executor_test.go:185`): it gives process spawn, `exec` and a pipe flush a 100ms budget, so under load the deadline fires before `sh` emits "started" and the assert sees an empty string. Present at this session's base commit and untouched by any of its work; surfaced by `-count=25`, invisible to ten clean full-suite runs, and exactly the shape CI hits +- [X] T095 Either consume `ExternalDB.TLSPolicy` and `ExternalDB.Restore` in the paths their help text promises, or remove both flags, their binds and their fields. Both are resolved and read by no production code, so `--external-db-tls-policy` silently does nothing and `--allow-external-restore` neither gates nor enables + +**Checkpoint**: `make build`, `make test`, `make lint` green; the two Phase 2R fixes split out for their +own PRs; no requirement claimed as satisfied without a non-test caller. + +--- + +## Phase 2T: Post-remediation review findings + +A `max` review over the 25 remediation commits (10 angles, 71 candidates, 15 verified plus 8 cut by +the cap). **Every one is the same class: a new single-source-of-truth was introduced and an older +parallel answer was left standing.** Several are regressions the remediation itself introduced. + +Findings 1–7 were dispatched to a fix chunk; the rest are recorded here. **Locate by symbol — line +numbers in this file have drifted three times.** Acceptance test for the class: +`grep -rn nameMatchesService src/ | grep -v _test` must return nothing outside `kubernetes_ownership.go`. + +### Silently wrong or destructive + +- [X] T097 `getPodStatusesWith` aborts on any selector error where the deleted code fell through to a namespace-wide listing — and `stopAppContainers` swallows that error and continues. On a Helm deployment (where the fallback is the *only* thing that ever matches `infrahub-server`) one throttled kubectl call yields a cold `neo4j-admin dump` with writers still running, reported as success. **T075 and T082 are each correct alone; together they corrupt a backup**. Fixed on both sides: the fallback listing now answers a selector failure — it enumerates every pod with the ownership evidence its labels carry, so it is a superset of what any selector could match, and only the listing *also* failing is undetermined (both failures reported); and `stopAppContainers` now fails the run on an undetermined status instead of warning and continuing, since every caller is about to require the services down. It still stops nothing on a guess +- [X] T098 `declaredServiceIn` returns the first *present* key's value rather than the first that names a service, so a chart with `app: ` on every pod masks `infrahub/service: ` — no pod contributes ownership evidence, the prefix set is empty, and an in-cluster database is declared external. This is the mechanism T075/T078/T082 all rest on. `declaredServiceIn` and `parseLabelledPods` — the workload half and the pod half of the same read — both take the first label value that names a service in `infrahubServiceLabelValues`, through the one new gate `declaredServiceValue`, rather than the first key that is set. A chart-labelled namespace (`app: ` plus `infrahub/service: server`) recovers its release prefix again. This is what makes T097's fallback a superset +- [X] T099 Transient pod and secret names are `--` with **no service**, while the run ID is minted once per run — so both databases external means two workloads under one name, an `AlreadyExists` abort blaming a concurrent operator, or a Postgres pod holding Neo4j's credentials. Caused by this effort's own instruction to mint the run ID per run without adding the service to the name. `podName` is now `---` and `secretName` is derived from it, so one run's two databases are two pods and two secrets. **The database segment is `neo4j`/`postgres`, not the deployment service name**: T074's hazard is deliberately not created — neither `database` nor `task-manager-db` appears in any transient object name, and a test asserts that plus the 63-character DNS-label limit +- [X] T100 `resolveExternalTLS` sets Neo4j `Encrypt = tls.Enabled`, so a deployment not exporting `INFRAHUB_DB_TLS_ENABLED` sends this tool's own database password unencrypted out of the cluster, with no flag to force TLS on. The same file argues at length that a discovered `tls_insecure` must be read and never obeyed. Encryption is the default for both databases and `--external-db-insecure-tls` is the only thing that turns it off; with the flag the channel is the weakest the deployment describes (unverified where it declares TLS, unencrypted where it does not). No new flag, so the decision stays two-valued and `--external-db-tls-policy` stays removed; `contracts/cli-surface.md` records the widened opt-out +- [X] T101 On the Plakar backend an incomplete capture is committed and reported successful: components commit one at a time, the completeness refresh sits on the metadata component reached *last* and so is unreachable with a false value, and `buildSnapshotTags` stamps `StatusComplete` unconditionally. The tarball path deletes the artefact and fails. The unreachable completeness refresh is replaced by FR-012's gate, raised before the metadata component is created — the last moment nothing is listable — and a deferred `discardPlakarComponents` deletes the component snapshots the run had already committed on every exit path, which is `discardIncompleteCapture` for this backend. `buildSnapshotTags` was already fixed by `2a5f326` +- [X] T102 `CaptureComplete` has no reader on any restore or restore-point-selection path, so the field documented as the last-resort identifier of an unusable artefact never rejects one — and `--latest` prefers it, being newest. `BackupMetadata.refuseIncompleteCapture` is read on both restore paths, before the checksums and everything destructive — which is also the refusal `--latest` needs, since selection there has only the archive's name. No `--force` escape, deliberately: unlike a group missing components, a capture that reported it had not finished reading the database leaves nothing for a bypass to be a judgement about + +### Wrong answer, wrong place + +- [X] T103 `getPodForServiceWith`'s name fallback is still the bare token-boundary match while the location decision uses the ownership rule, so resolution and location give two answers from the same evidence — and discovery execs into a foreign pod, adopting *its* connection URL as the endpoint restore writes into. The fallback already went through `deploymentClaimsResource` (49b5790); what still differed was the *policy*, hard-coded `unanchoredNameClaims` at each call site. It is now `unanchoredNamePolicyFor(service)` at all four — the pod resolver, the running check, the location decision and the workload resolver — so the choice follows the service and one listing yields one answer. `postgres-database-0` in an unlabelled namespace is refused by resolution and by location alike; the app services keep the permissive reading they ship with. **This label-guards `getPodForServiceWith`'s fallback, so T074's second half is covered here — narrow T074 rather than duplicating it** +- [X] T104 `findWorkloadResourceWith` derives release prefixes per-kind while `deploymentClaimsService` derives them namespace-wide, so a chart labelling StatefulSets but not Deployments still scales a foreign workload to zero. Both listings are now gathered before either is judged, and `namespaceReleasePrefixes` derives the prefixes from every kind together. Label evidence across all kinds is also read before any name match on any kind, which is the precedence `claimedPodNamesWith` already applies to pods. A test drives the exact topology and catches the old code resolving `redis-cache` as `cache` +- [X] T105 `locateServiceWith` is strictly narrower than the resolver's own fallback, so a CloudNativePG task-manager database — the topology the ownership doc block names as the case it fixes — is classified external and restore is refused outright. Fixed by widening the *evidence*, not the policy, so T103's narrowing stands: `nameIsServiceOwnName` claims a resource named for the service itself — nothing before it, and nothing after it but an ordinal or a hash — which is what `task-manager-db-1` is and what `postgres-database-0` (a release this deployment never declared) and `task-manager-db-exporter` (a different component, a word after the service) are not. It holds whether or not a prefix was declared, so the CNPG cluster is internal in a Helm-labelled namespace and in a bare one, and the resolver reaches the same pod +- [X] T106 `fetchPostgresCredentials` discards `applyPrefectConnection`'s bool and marks discovery read unconditionally — the exact defect the return value was added to fix at the other call site. The read is now recorded only when the URL supplied something, so a typo in the deployment's own connection variable no longer suppresses `ensurePostgresDiscovery`. Table-driven test over the three shapes: a URL that parsed, one that did not, and a container exporting none +- [X] T107 `redactDatabase` and `isNeo4jCluster` exec into `"database"` unconditionally, so `--redact` reproduces the FR-008 misdiagnosis after the gate established the database is external, and still passes credentials as argv. **Partial.** The gate half was already fixed in `e49b4be` — both now call `externalCaptureSourceFor(serviceNeo4j)` first, both are reached after `prepareDatabaseCapture`, and `TestRedactDatabaseRefusesAnExternalDatabase` / `TestIsNeo4jClusterReadsWhatTheProbeObserved` cover them; the box was simply never ticked. The argv half is **not** fixed and is deliberately left whole: `cypher-shell -p` appears at twelve in-deployment sites, `ExecOptions.Env` is argv-equivalent by construction (`prepareCommand` renders it as an `env KEY=VALUE` prefix, which FR-014 forbids for exactly this reason), and the only real fix is a stdin-carrying exec that still returns stdout — a primitive neither backend has. Fixing two of the twelve would create the split this phase exists to remove; it needs its own task. **Now closed on FR-014's own scope.** FR-014 reads "credentials for an *externally-managed* database MUST NOT be exposed on a process command line", and it is satisfied: T040 moved the external path onto the mounted secret and `TestNoCredentialReachesAnExternalArgv` pins it. The argv clause of this finding is on the *internal* path — after the gate landed in `e49b4be`, `redactDatabase` and `isNeo4jCluster` only ever run for a database that lives in the deployment, because an external one is refused before them. So the gate half was the whole of the defect this task names, and it is fixed and covered. The internal argv exposure is real, pre-dates the branch (all 10 sites present unchanged at base `8186fcf`), and is out of scope here — see T121 +- [X] T108 The capture pod's deadline budgets one operation bound for a pod hosting two full-bound operations (capture, then a copy that opens its own `externalDBBound`), so the cluster kills it mid-copy after a capture that succeeded. `externalDBProbeDeadline` was fixed for exactly this and its sibling was not. `externalDBCaptureDeadline` now multiplies the bound by `externalDBCaptureFullBoundCalls`, named and documented as the two operations the pod hosts; the short control-bound calls around them stay covered by the margin. The doc also records the one caller this cannot promise: the streaming capture pipes through `ExecStreamPipe`, bounded by an idle timeout rather than a duration, so the deadline is a floor there +- [X] T109 `probeExternalPostgres` accepts an unusable answer and breaks where the Neo4j probe validates and continues — **a repeat of F16**, recorded pre-revert and reintroduced by the rebuild. Already fixed in `e49b4be`, which is why the box was unticked rather than the code unwritten: both probes now walk through the one `firstAnsweringEndpoint`, differing only in `accept`, and the PostgreSQL arm rejects an empty version and continues the walk. Verified present and covered by `TestFirstAnsweringEndpointWalksPastAnEmptyAnswer` and `TestProbeExternalPostgresWalksPastASilentMember` +- [X] T110 Verification is on by default and no trust material reaches the workload: `PGSSLMODE=verify-full` with no `PGSSLROOTCERT`, and the discovered CA file is used only to build a log string. The secure default is unusable and the only escape disables verification entirely. The Neo4j arm landed in `b470c67`; this is the PostgreSQL arm. The task-manager connection URL's own `sslrootcert` — the PostgreSQL counterpart of `INFRAHUB_DB_TLS_CA_FILE` — is now discovered into `ExternalDB.PostgresTLS.CAFile`, read out of the `task-manager` container and written into the workload as a PEM (libpq needs no keystore), with `PGSSLROOTCERT` naming it. Where the deployment names no authority, or a step fails, `PGSSLROOTCERT=system` points libpq at the image's own trust store — so the default is a mode that can be satisfied instead of one that cannot, and every degradation stays *verifying*. The client env is resolved once per workload and carried on `externalCapture.ClientEnv`, because resolving it writes into that pod. No flag changed + +### Operability, hygiene, and one governance item + +- [X] T111 `transientPodSpec` emits no `imagePullSecrets`, `serviceAccountName`, `nodeSelector`, `tolerations` or `priorityClassName`, and nothing can supply them — so the documented mirrored-image path sits in ImagePullBackOff until readiness times out. All five are now on `transientPodSpec`, every one `omitempty`, so a deployment that needs none of them gets byte-for-byte the pod it got before (asserted). They are supplied by five new flags — `--external-db-image-pull-secrets`, `--external-db-service-account`, `--external-db-node-selector`, `--external-db-tolerations`, `--external-db-priority-class` — parsed once by `resolveTransientScheduling`, which both spec builders call so the *probe* carries the same placement as the capture (it runs the same image, so the registry blocks it first). Malformed input is refused before anything is created, by a message naming the flag. Unlike the image flags these are viper-bound: FR-027's reason is arbitrary *code*, and placement is not code while the image stays pinned — the pod still mounts no service-account token. `contracts/cli-surface.md` records the addition and that reasoning +- [X] T112 `resolveScratchSize` converts a float64 product to `int64` with no range guard, so a large server-reported store size saturates to `MaxInt64` on arm64 and wraps to `MinInt64` on amd64 — where the floor clamp silently rewrites it to 5Gi. `make build-all` ships both. The product is now checked against `maxStorageQuantityBytes` — the same 2^63 ceiling `parseStorageQuantity` uses, and for the same reason: exactly representable as a float64 where `math.MaxInt64` is not — before the conversion, and a store whose headroom does not fit fails naming `--external-db-scratch-size` (FR-022's closing clause) rather than being floored. The test asserts the *refusal*, which is what makes it hold on both architectures: saturation answers 8589934592Gi and the wrap answers 5Gi, and neither is an answer this input has. T089's `formatGiB` normalisation is untouched +- [X] T113 `parseHostPort` treats any unbracketed member with two colons as a bare IPv6 literal, so a semicolon typo becomes a host and lands in `--from=` and in provenance metadata. The literal is now verified with `isIPLiteral` rather than assumed from the colon count, so `db-a:7687;db-b:7687` and `core-1:7687:6362` are refused as address lists instead of becoming one host this run claims to have read. Not narrowed to IPv6: an IPv4-mapped literal carries the same colons, and a zone identifier (`fe80::1%eth0`) is kept because an address without it cannot be connected to. A bracketed *host name* is refused too, on both the ported and portless arms, since brackets are the notation for a literal +- [X] T114 `captureRoles` keys members by host alone, so two instances on one host collapse and a primary can be recorded as `secondary` — the fact FR-005 says bears on the recovery point. Matching now takes the address whole first — available for PostgreSQL and for operator-supplied Neo4j client addresses — and falls back to the host, which is what makes the ordinary Neo4j case work at all (supplied endpoint carries the backup port, the server reports its client one). The fallback keys to a *set*: instances at one host that agree still record the role, because then it does not matter which one the endpoint was, and instances that disagree record `roleUnknown` with the ambiguity logged. The "nothing chosen yet" state is the empty string, not `roleUnknown`, since a server can report that as a role — starting from it made the answer depend on the order the listing arrived in (test covers both orders) +- [X] T115 `uncontactableEvidence` scans the wrapped error chain, which now carries kubectl's stderr, so an API-server outage is reported as a Neo4j endpoint the capture could not reach. Attribution is now positive rather than a blocklist of transport phrasings: a line counts as evidence only if it names a host the run supplied, so `classifyNeo4jCapture` takes the capture's own targets and `Unable to connect to the server: dial tcp …: connection refused` no longer becomes a named endpoint. On the host and not the whole address, for the same port reason T114 has. A genuine failure whose wording omits the host is simply not evidence, which is the safe direction the markers' own doc block already states: they corroborate a verdict reached from the exit status and the artefact, so losing one costs the operator a name in the message and never promotes a capture to complete +- [X] T116 `parseCypherRow` skips one non-empty line, so the *notice* its own comment says it handles shifts the parse and the header is returned as data. The header is now located by the column names the statement asked for, in one new `cypherResult`, so no number of preceding notices can shift it. **The sibling was checked and shared the bug**: `parseNeo4jMemberRoles` took `lines[0]` as its header, found neither column in a notice, and reported that a cluster hosts no members — it now goes through the same locator. The column names are constants used by the statement *and* its parser (`neo4jVersionColumn`, `neo4jEditionColumn`, `neo4jStoreSizeColumn`, `neo4jMember*Column`), because a renamed alias in one and not the other finds no header and "the server answered nothing" is a wrong answer that reads like a server problem. `parseCypherRow` now returns the row keyed by column, which also ends the last positional read of a named result — the components row fed the FR-006 comparison and the FR-008 edition gate from `fields[0]`/`fields[1]`. Where no matching header appears the callers' existing "answered nothing" failures apply, which is fail-loud +- [X] T117 `databaseTarget` is constructed and discarded at its only production call site, so T083's "reachability is a property of the target" has the shape without the substance; `TransientWorkload.Role/Image/Deadline/ScratchSize/backend` have no readers; the `default:` arm is unreachable. `resolveDatabaseTargets` now returns the targets and both callers read them: the capture path hands them to `prepareExternalSources`, which took a list of service names and asked `queries.locate` a second time for each — so the location the gate established is what the preparation acts on, and `prepareExternalSourceWith` takes the target rather than a name, which is what makes "only an external database reaches here" a property of its argument. The restore path reads them too, for the one thing it needs: every target a restore can hold is an in-deployment one, because `reachable` refuses the other. The five write-only `TransientWorkload` fields are gone — every reader went to `transientWorkloadSpec`, which is also the value the pod's shape is asserted against, and `backend` was documented as being what `Release` needs while `Release` goes through `ops`. The unreachable arm is `discoverEndpoint`'s, which returned nil: it now fails naming the two databases, as its sibling `applyDiscoveredFacts` already did, since the one thing that could reach it is a third database whose settings would be silently unread +- [X] T118 `infrahub-collect --include-backup` now creates and deletes Pods and Secrets through a binary `AGENTS.md` calls read-only diagnostics and ADR-0003 pins as strictly read-only with respect to workload lifecycle — needs an ADR amendment or a carve-out. **Resolved as a refusal, not a carve-out**, on the repository owner's decision: `CollectBundle` marks the run with `forbidExternalCapture`, and `externalCaptureOps` — the only function in the tool a transient object can come from — turns it back on its first line, ahead of the stray reaper, which deletes. So the refusal is raised at the gate, before any Infrahub service is stopped and before anything is created, and the collector records a failed backup while the bundle is still produced (ADR-0006's non-fatal contract). The message names the condition, the resource and the action the contract requires: a database outside the deployment, a transient pod and secret in namespace X, and `infrahub-backup create` as the tool that performs it. ADR-0003's guarantee is therefore left **unqualified** and both ADRs say why the boundary is where it is; the accepted cost, recorded in ADR-0003's consequences, is that an external-database deployment gets no backup inside its bundle. `contracts/cli-surface.md` gains the failure-message row +- [X] T119 Three copies of one generic filter (`withoutTransientPods`/`Candidates`/`LabelledPods`); `parseLabelledPods` duplicates `parsePodCandidates`; ~30 uncached kubectl listings in `stopAppContainers`; four selector queries issued before a listing that supersedes them; two doc blocks landed on the wrong symbol. **All five now done — four here, the fifth under T122.** Done: the three filters are one `withoutTransientPods` over the parsed listing — the two name-taking callers filtered *after* projecting to names, which is the same set, so filtering first leaves one filter for one type. `parsePodCandidates` and `podCandidate` are deleted: the narrow parser split on the first ";" and, handed an ownership line, reported `Running;;;server` as a pod's phase, which neither phase predicate recognises, so the pod stayed in every answer under a phase no cluster reports; `parseLabelledPods` reads both shapes and is now the only parser. The namespace-wide ownership listing goes through one `namespacePodListing`, memoised per field selector and dropped by `dropPodCaches` alongside `podCache` — the running check, the fallback resolver and the location decision each issued it once per service and the query names no service, so `stopAppContainers`' six identical listings are one (25 kubectl calls where there were 30, pinned by a test); a listing that failed is not memoised (FR-002). Both doc blocks are on their symbols (`formatGiB`, `selectTransientStrays`). **Not done, deliberately**: dropping the four selector queries in `getPodStatusesWith`. The listing supersedes them only where they matched *nothing* — where a selector matched, the listing was never consulted, and the claimed set is a superset, so `IsRunning` could newly answer true from an unlabelled claimed pod. That is `stopAppContainers`' input. It would also delete T097's two-failure reporting, whose whole point is that a throttled selector query is not an undetermined status. Reproducing the selector match from the listing's raw label fields would be exact and is the way to do it, but it is a change to the running check rather than a dedup and needs its own task. **Fifth clause now done under T122**, exactly that way: the selector matches are replayed from `labelledPod.Labels` through `selectorMatchesLabels`, the four queries are gone, and the running check costs one kubectl call. The two hazards this clause was held open on are the two properties T122 mutation-tested: the ownership tier is still reached only where no selector would have matched, so `IsRunning` cannot newly answer true from a claimed-but-unlabelled pod; and T097's property survives in its strongest form, there being no selector query left to be throttled, with the one listing's failure wrapped as the undetermined status. Note the listing count in this clause is superseded by T124: `stopAppContainers` issues six *liveness* listings, not one, and that is deliberate — six calls in total now, where this clause left 25 +- [X] T120 `AGENTS.md`'s Core Components list does not name `external_capture.go` or `kubernetes_ownership.go` — the same staleness class that sent a chunk hunting for a version comparator that did not exist. Two entries added rather than two file names, because naming only the two the review happened to catch is how the list went stale: `external_*.go` (the gate, endpoint resolution, the capture sequence, the CA, the bounds, the transient workload) and `kubernetes_*.go` (ownership, pod matching, workloads, the backend). Each says what a reader must not duplicate — that `resolveDatabaseTargets` is the only source of a target, that `externalCaptureOps` is the only source of a transient object, and the `nameMatchesService` acceptance test that pins ownership to one file +- [X] T122 Reproduce `getPodStatusesWith`'s four selector matches from the namespace listing's own label fields, then drop the selector queries. **Raised by chunk 2Td while doing T119, and T119's fifth clause stays open on it.** Not a dedup: the listing supersedes the selectors only where they matched *nothing*, so removing them naively lets `IsRunning` newly answer true from a claimed-but-unlabelled pod, and deletes T097's two-failure ("undetermined") reporting that `stopAppContainers` now depends on to fail a run. Changing the running check is a behaviour change and needs its own tests. **Done as a reproduction, not a replacement.** `parseLabelledPods` now keeps the four service-label values it already read (`labelledPod.Labels`, via `serviceLabelsIn`) instead of reducing them to `Service`, because a selector and a declaration are two readings of the same fields: a selector matches one key set to *exactly* the service, `Service` is the first key whose value is in the deployment's vocabulary. `podsServiceSelectorsWouldMatch` replays `podSelectors` against those fields through `selectorMatchesLabels` — the same function `findWorkloadResourceWith` already reproduces its own selector queries with — in selector order, first non-empty group wins, which is what the loop of queries did (it returned on the first selector that matched, never accumulating across four). The ownership tier is reached under exactly the condition it was before: no selector would have matched. **The three properties, each mutation-verified red**: (1) *undetermined* is now produced by the one namespace listing failing, wrapped as `failed to determine whether is running in namespace ` — there is no selector call left to fail, which is the strongest form T097's property takes, and `stopAppContainers` still fails the run on it; returning an empty answer instead reddens three subtests of `TestGetPodStatusesWith_QueryFailures`. (2) `IsRunning` cannot newly answer true from a claimed-but-unlabelled pod: dropping the selector tier makes `infrahub-cache-exporter` (claimed on the release prefix, labelled nothing) report `Running` beside a `Pending` `infrahub-cache-0`, and the fixture asserts the exporter *is* claimed so the test is about the tier and not about a pod ownership would have refused anyway. (3) FR-028 holds on the new path — a transient pod carrying `app.kubernetes.io/component=cache` is a pod a selector would have *returned*, and `withoutTransientPods` drops it on its marker inside the listing before the match is read; removing `pod.Transient` reddens the new subtest and both halves of T074's. **Measured kubectl calls**: one `IsServiceRunning` 1–5 → 1; a stop's six status queries 25 → 6 on a chart-labelled namespace and 30 → 6 on a label-blind one, and the same 5× cut applies to every poll of the quiesce and restart waits. A *union* across the four keys was considered and refused: it differs wherever two keys name different pods, and it differs in the direction of `IsRunning` answering true more often, which is a change to the running check rather than a reproduction of it. `cca32bb`'s discovery/liveness split is untouched — the running check still reads `freshNamespacePodListing`, which touches the memo neither to read nor to write +- [~] T121 **OUT OF SCOPE for 007 — file as its own issue.** Add a stdin-carrying exec primitive that still returns stdout, on both backends, and route the internal `cypher-shell -p` sites through it. Raised by chunk 2Tb while doing T107, and reclassified after reading FR-014's text rather than its summary: the requirement is scoped to an *externally-managed* database and is already met (T040, pinned by `TestNoCredentialReachesAnExternalArgv`), so nothing in this spec obliges the internal path to change. All 10 argv sites exist unchanged at base `8186fcf`, so this branch neither introduced nor worsened them. **Residual, accepted knowingly**: the internal path passes the Neo4j password on argv, visible to anything reading `ps` inside the database container and to the Kubernetes API audit log. Both populations already hold credential-equivalent access — exec rights on the database pod are what put you in that container, and those same rights let you read Neo4j's own configuration — so the marginal exposure is audit-log retention rather than a new attacker path. Doing it inside this spec would also have forced a re-baseline of the FR-015 pin that asserts the internal argv character-for-character against the pre-feature command, which is a poor trade for a change this spec does not require. `ExecOptions.Env` still cannot serve as the mechanism when it is done: `prepareCommand` renders it as an `env KEY=VALUE` argv prefix + +--- + +## Phase 2V: Post-review findings (third review round) + +A six-agent review over the branch. The class this round found is narrower than Phase 2T's and +worse: **a cache, a field, a refusal or a release was placed where it answers a question adjacent to +the one it was written for.** T124 is the sharpest of them and is fixed on its own. + +- [X] T124 A memoised pod listing freezes every liveness poll loop. `namespacePodListing` served two questions with opposite freshness requirements — *which pods exist* (discovery: which pod is a service's, whether a database is in the deployment at all; stable within a run, which is what the memo was added for) and *what those pods are doing* (`getPodStatusesWith` → `IsRunning`, which `confirmAppContainersQuiescedWithin` and `reportAppContainersRunningWithin` poll in a sleep loop **with no mutation between iterations**). The memo is dropped only by `dropPodCaches`, called only by `Start` and `scaleServices`, so those loops replayed their first poll's bytes until the window expired and could never observe the transition they exist to detect. Two confirmed consequences, both on the restore path: a restore aborts *after* `wipeTransientData` with "infrahub-server is still running 5m after it was stopped" naming a pod that terminated normally, with the data gone and the deployment at zero; and a restore that worked reports all six services as having failed to come back. **Fixed by splitting the question, not by invalidating from the caller**: `issueNamespacePodListing` issues the query (one place, one wording for its failure), `namespacePodListing` is the memoised discovery reading, and `freshNamespacePodListing` is the liveness reading, which touches the memo neither to read nor to write — so a poll loop added later cannot reintroduce the bug by forgetting to invalidate, and no caller decides another caller's freshness. Caller-driven `dropPodCaches` was rejected for both reasons plus a third: it would discard the discovery answers the run still needs. **The cost is T119's fifth clause**: `stopAppContainers`' six liveness listings are six again, because "a burst of six" and "a poll loop" are the same code path and only run-time state distinguishes them — correctness over one saved call. T092's discovery memo is intact and still collapses repeat location queries to one listing, pinned by `TestResolutionIssuesOneNamespaceListing` (which replaces `TestStopAppContainersIssuesOneNamespaceListing`, whose assertion *was* the bug). Two new tests fail against the unfixed code: `TestRunningCheckReadsAFreshNamespaceListing` at the backend, and `TestQuiesceConfirmationDoesNotReplayAnEarlierPollsAnswer`, which drives the real `KubernetesBackend` and the real wait against a stubbed `kubectl` — necessary because `quiescingBackend`, the double the existing suite uses, answers `IsRunning` from a map and never reaches a cache +- [X] T125 `CaptureComplete` is computed before the captures run, so FR-012's field can never be false. `applySourceProvenance` read `iops.incompleteCaptures` from inside `createBackupMetadata`, which `backup.go` calls at the point *before* `backupDatabase` and `backupTaskManagerDB` — and Neo4j's `recordIncompleteCapture` fires during its own capture. The map is therefore always empty when it is read, so `capture_complete` is `true` (or absent) in every artefact ever written, and `refuseIncompleteCapture` guarded a value nothing could produce. **Fixed by moving the read to where the metadata is serialized**, which is after the captures on both backends: a new `applyCaptureCompleteness` is called from `backup.go` just before `json.MarshalIndent` and from `plakar_backup.go`'s `ComponentMetadata` arm just after its gate. The field is now a report rather than a prediction. **What is deliberately unchanged is the structural protection**: both paths still gate on the verdict before serializing — the tarball path discards instead of archiving, the Plakar path refuses before the metadata component exists — so this tool still writes no artefact recording `false`, which is what FR-012 asks for. What the fix buys is that the label cannot disagree with the gate wherever a later change puts the gate, and that `refuseIncompleteCapture` now reads a value that means what it says for the artefacts a failed removal or an older tool leaves behind. FR-015 holds: an all-internal run writes the field no more than it wrote the other provenance fields, asserted through both steps. The old test could only pass by recording the incomplete capture *before* building the metadata — the reverse of production's order — and now runs in production's order +- [X] T126 `RestoreLatestBackup` creates transient workloads with no deferred release. Its S3 leg calls `prepareDatabaseRestore` itself, before pulling an object a refusal would make pointless, so it is an entry point that creates workloads — and unlike `CreateBackup`, `RestoreBackup`, `CreatePlakarBackup` and the Plakar restore it deferred no `releaseTransientWorkloads`. `prepareExternalRestoreWith` discards `release` on success, so a failed download, or a `task-manager-db` preparation that fails after Neo4j's workload was recorded, left a Pod and a Secret standing in the customer's namespace until their own deadline; neither path reaches `RestoreBackup`'s defer. **Fixed** with a deferred release at the entry point, which is safe alongside `RestoreBackup`'s own (the inner one finds nothing left). Both false doc comments are corrected: `external_restore.go`'s "the run's single deferred releaseTransientWorkloads is its only exit path" and `external_capture.go`'s "deferred once by each entry point" now say *every* entry point and why the `--latest --s3` path has two. `TestRestoreLatestBackupReleasesTransientWorkloads` fails against the unfixed code (workload left `ready`, still tracked, nothing deleted) +- [X] T127 `startAppContainers` abandons the remaining services on the first failure (FR-013). It returned on the first error and `cache` is first in `preferredOrder`, so one transient failure there left the other five at zero replicas, reported as a single `logrus.Errorf` naming one of six. The services do not depend on each other for *starting* — `preferredOrder` is about the order they come up in — so **every service is now attempted and the failures are joined and reported together**. The two stop-error arms in `backup.go` and `plakar_backup.go` now log at error level and say that the services named are still at zero and need starting by hand, which the aggregated error makes readable. `TestStartAppContainersAttemptsEveryService` fails against the unfixed code on both counts: five services never attempted, and only the first failure named +- [X] T128 `waitForNeo4jRestore` retries permanent failures for the whole bound. It retried *every* exec failure until `restore.Bound` — two hours by default, with Infrahub scaled to zero — and then reported "the database is not usable… Its reported state is the server could not be asked for its state: \", attributing an evicted transient pod, a revoked `pods/exec` or a departed API server to the customer's database and sending the operator to the wrong logs. **Fixed by bounding the two conditions by what they are**: the operator's bound stays with the database, because a large store legitimately takes hours to load, while a read that never reached the server gets `neo4jStatusUnreadableGrace` (one minute — the length of the condition it covers, a server refusing a session for a moment after replacing a database). Past that, and on the deadline arm when a bound shorter than the grace expires first, the new `neo4jRestoreUnreadable` says the failure is in the path to the database, names the three things on that path, and says the seed may still be loading. The wait is parameterised as `waitForNeo4jRestoreWithin` following `confirmAppContainersQuiescedWithin`'s idiom. Four cases added, including the one that must not regress — a moment's refused session is still retried — and the one that must keep the old attribution: a server that answered and is not online still gets the database's verdict and the operator's bound. Against the unfixed loop the new test does not merely fail, it hangs, which is the defect exactly +- [X] T129 The Plakar Community refusal fires after services are already stopped. `refuseExternalDumpRestore` was raised from inside `restoreNeo4jCommunityStream`, which both Plakar restore paths reach only after `stopAppContainers`, `confirmAppContainersQuiesced` and `restartDependencies` — so the run took the deployment down to discover something it already knew from the snapshot's own tags, and then told the operator "No Infrahub service was stopped", which by then was false. Every other refusal on this feature is raised at the gate (`external_backup_gate.go`). **Fixed**: a new `refuseExternalCommunityDumpRestore` is called from `restoreBackupGroup` as soon as the group's edition is resolved and from `restoreSingleSnapshot` before its community branch, both before anything is stopped. The in-load check is **kept as a backstop** rather than deleted — that is where the container is actually needed, and a Plakar path added later could reach it without passing a gate — but it now raises `refuseExternalDumpRestoreAfterQuiescing`, which does not repeat the gate's "nothing was stopped" claim, because by there something has been: asserting otherwise is precisely what went wrong when the refusal lived only in that position +- [X] T130 `undiscoverableEndpoint` misattributes an RBAC denial as "no address configured". A failed `discover` was only `logrus.Warnf`'d, and `applyNeo4jFacts`/`applyPostgresFacts` then returned a message asserting the settings "were read for it, and none of them records an address" — a claim about the customer's configuration made on evidence the run never obtained. A denial on `pods/exec`, a component pod that has gone, or an expired control bound produces exactly that message, and FR-004 and FR-019 are two conditions with two different remedies. **Fixed** by carrying the discovery error into `applyDiscoveredFacts`, where one new `noAddressFor` routes between `undiscoverableEndpoint` (asked and answered) and a new `unreadableDeploymentSettings` (never asked), which carries the cluster's own reason, names both remedies, and states outright that it is *not* a report that the setting is unset. `errDeploymentSettingsUnreadable` is a distinct sentinel from `errEndpointUndiscoverable`, so a caller cannot treat the two alike. The existing tolerance is preserved and asserted: an operator who supplied the address still gets their run, and the discovery failure stays a warning. Both new messages are added to the failure-message table in `contracts/cli-surface.md` and to `TestFailureMessageContract`, the second with `wantAbsent` rows for the claims each must not make +- [X] T131 FR-013's restore-path wiring had no test at all. `TestReturnAppContainersToScale` calls `returnAppContainersToScale(stopped)` **directly**, and every restore-level test asserted an *early* refusal, so no test ever reached `stopAppContainers` — mutation confirmed it: all three `defer` blocks calling the restart could be deleted with the suite green. The regression that would slip is a restore failing after quiesce and leaving Infrahub scaled to zero, which is the outage FR-013 exists to prevent. **This is T123's shape**: there a backend was handed in, here the `stopped` slice is, and a test handed that cannot observe whether the production path builds one. **Fixed** with `restoringBackend`, a deployment a whole restore can be driven through — edition probe, transient-data wipe, scale-down, status polls, copies into containers, scale-up — and four cases that quiesce and then fail at the next step, through `RestoreBackup`, `RestoreLatestBackup`, `restoreBackupGroup` and `restoreSingleSnapshot`. Mutation evidence: deleting the `defer` in `backup.go` gives `started = [cache message-queue task-manager task-manager-background-svc], want every service the run stopped` plus `infrahub-server is still stopped`; deleting both in `plakar_restore.go` gives the same for the two Plakar paths. Driving the Plakar entry points needed a real kloset repository in a unit test, which T133 made possible +- [X] T132 FR-026's destructive-step guard had no test at all, the same way. `TestConfirmAppContainersQuiesced` is a good test of the helper (10 assertions) and only calls the helper; removing `confirmAppContainersQuiesced` from `RestoreBackup`, and separately from all four `plakar_restore.go` sites, left the suite green. The regression is a restore overwriting databases while Infrahub workloads are still running. **Fixed** in the same file: the deployment answers every status query the scale-down makes and then stops answering, which is the uncertainty the guard exists for and the one arm that refuses at once — `appQuiesceTimeout` is five minutes and `RestoreBackup` takes no window, so the "pod will not terminate" arm stays with `confirmAppContainersQuiescedWithin`, which does. Six cases, and the destructive witness is `CopyTo`/`ExecWritePipe`, which nothing on a restore reaches before the deployment is meant to be quiesced. Mutation evidence, with the guard removed: `copies into containers = [database:/tmp/infrahubops]`, `copies into containers = [task-manager-db:/tmp/infrahubops_prefect.dump]`, `err = neo4j pid file is empty` where the guard's refusal was wanted, and `the restore = nil` on the group path — a run that reported success. The "nothing was written" assertion is deliberately made *before* the error assertion so a build without the guard reports what it did to the databases rather than only that it failed later +- [X] T133 **Found while fixing T131**, not raised by the review: `initPlakarContext` never set `KContext.CacheDir`. kloset reads a cache location from two places — the manager it is handed serves the scan and packing caches, while the repository's own state cache is derived from that field — and `path.Join` on an empty first element yields a *relative* path. So every plakar `create`, `restore` and snapshot listing wrote a `2.0.0/store/` tree into whatever directory the operator ran the tool from, and read the previous run's state back only if they ran it from the same one; neither `--plakar-cache-dir` nor the default under `~/.cache` reached that cache at all. It is what makes the Plakar e2e suite deposit a version-named directory in the working tree, and it appeared in `src/internal/app/` as soon as a unit test drove a real repository. **Fixed** in one line. `TestPlakarCacheLandsUnderTheConfiguredCacheDir` asserts both halves — present under the configured directory, absent from the working directory — because a cache written to both would satisfy the first alone +- [X] T134 `test_a_readable_but_corrupt_artifact_fails_and_never_reports_success` could not reach the scenario it names. It truncated the `.tar.gz` to half its length, which breaks gzip decompression: the run fails while *extracting*, upstream of the seed staging and of the load on the database server. Both assertions still held, so the case went green without exercising the validation that separates **FR-021** (the database did not come online after a seed) from **FR-007** (the server could not read the artifact at all). Replacing bytes inside the archive without repairing the metadata does not get much further, because the restore validates the checksums recorded at capture time. **Fixed** with `_archive_whose_neo4j_artifact_cannot_be_loaded`, which rebuilds the archive with the Neo4j members overwritten *and their checksums recomputed*, so the artifact is internally consistent and substantively wrong: every check this host can make passes, it stages and reads back, and the first thing that can notice is `neo4j-admin database restore` on the server. The members come from the metadata's own checksum map rather than from a path, so it survives a layout change. **Not executed** — the external topology is not provisioned here (T072) and the k8s leg needs `vcluster`; the case collects and skips by name, as before +- [X] T135 The transient pod's `activeDeadlineSeconds` formula was spelled three times. `externalDBProbeDeadline`, `externalDBCaptureDeadline` and `externalDBRestoreDeadline` each wrote `readyTimeout + N*bound + margin`, and the comment on one records the arithmetic having been gotten wrong **twice** — budgeting one bound where the pod hosts several was fixed in the probe's and left standing in the capture's, where the cluster deleted the pod part-way through the copy *after* a capture of a production database had succeeded. **Collapsed** into `externalDBWorkloadDeadline(calls, bound)`; each caller passes its own count and bound, the per-role rationale stays on the per-role constants, and all three deadlines are unchanged. The `members < 1` clamp stays in the probe, where it is documented, rather than becoming a second defence in the helper +- [X] T136 Matching a Cypher column to its header had three conventions, and one carried a latent correctness defect. `parseNeo4jDatabaseStatuses` and `parseNeo4jMemberRoles` both built `columns[strings.ToLower(strings.TrimSpace(name))]` verbatim and then read it differently — the first lowercasing its constant at the lookup, the second using it raw. Raw is correct only while every constant is itself lowercase, which today's `address`, `role` and `writer` are; the *next* one breaks it, and `neo4jStatusColumn` on the sibling path is already `currentStatus`. Such a member column would not be found, `hasAddress` would go false, `parseNeo4jMemberRoles` would return nil, and every member's role would vanish from `SourceRoles` — FR-005's provenance silently empty with no error, the opposite of what `64bfdb8` intended. `cypherResult`/`namesColumns`/`parseCypherRow` answered the same question a third way with `strings.EqualFold`. **Unified** on `cypherColumns`, where the indexing and the lookup both go through `cypherColumnKey`. `TestCypherColumnLookupIsIndifferentToCase` pins both directions and both are mutation-killed; the first mutation — the shipped raw lookup, generalised — also breaks `parseNeo4jDatabaseStatuses` and `parseCypherScalar("attributes.Value.value")`, which is independent evidence that mixed-case columns occur on this path. Note the review's diagnosis was one step off: the *header* side was always case-insensitive, so a mixed-case header is not the trap; the caller's spelling is +- [X] T137 Two smaller duplications, same commit. `copyFromTransientWorkload` and `copyToTransientWorkload` shared a ~15-line prologue and differed in the kubectl method and a preposition, while everything that has to be right about them — the Kubernetes-only refusal, the bound, the context that carries it, and the wrapping that turns a deadline into a message naming the pod — was stated twice; they are now one `copyAcrossTransientWorkload` with the direction as a value. `resolveScratchSize` and `resolveRestoreScratchSize` both TrimSpace → `parseStorageQuantity` → `formatGiB`, with the comment explaining *why* normalising is required — the parser accepts `"50 Gi"`, the API server rejects it — present on one copy only, so the other depended on an unstated rule and its `formatGiB` looked cosmetic. Both now go through `normaliseScratchOverride`. One behaviour difference, reported rather than hidden: the restore arm now emits the debug line the capture arm already emitted when it normalises a value +- [X] T138 `databaseTarget` had no construction witness. All four fields exported and no unexported one, so `databaseTarget{Service: serviceNeo4j, Location: EndpointLocationExternal, Operation: databaseRestore}` was a value any caller in the package could write — carrying a zero `Restore`, i.e. no authorisation — and `prepareExternalRestoreWith` takes a target **by value and never re-checks it**, as its own comment says ("carried by its argument rather than checked by its caller"). That literal would have performed an **unauthorised external restore with no refusal anywhere**. Not a live bug: every production construction site is inside `databaseTargetWith` and only `resolveDatabaseTargets` calls it. But it is the shape that has broken twice on this branch, and the doc comment overstated the guarantee. **Fixed** with an unexported `gated` field set only by `databaseTargetWith`, after `reachable` has answered, and `requireGated` at the head of both preparers. The two tests that built their own target now obtain one from the constructor, which is the point. `AGENTS.md`'s "the only way a `databaseTarget` is obtained" is corrected to say what actually makes one evidence +- [X] T139 `ExternalRestoreAuth{Allowed bool; Source}` was two fields answering one question: `reachable` read only `Allowed`, `describe` read only `Source`. `{Allowed: true}` was representable — it authorises a destructive write into unmanaged infrastructure and then logs "no channel this run can name", losing exactly the FR-010 fact about whether an operator was present. All three constructors in `cli.go` were correct; none had to be. **Fixed** by dropping the bool: `Source` alone, `Absent` means unauthorised, and `allowed()` is derived and unexported because `databaseTarget.reachable` is its only reader. `TestExternalRestoreAuthorisationIsItsChannel` pins that every value which authorises can name its channel. The `--force` separation is untouched and still structural: `Force` appears in none of `external_restore.go`, `plakar_restore.go` or `backup.go` +- [X] T140 `external_backup_gate.go` claimed "The targets are returned, and **both callers** read them". There are three non-test callers — `reportDatabaseLocationsWith` was added later — so it now says *every* caller. The adjacent claim about `databaseTargetWith` having one non-test caller was true and is left alone +- [X] T141 `TestVerifySeedReadableRefusesAnUnreadableURI` takes **4s–125s** depending on AWS SDK retry behaviour against `127.0.0.1:1`, which is why the `src/internal/app` package's wall time swings between ~47s and ~260s. Pre-existing and not introduced by this round, but it makes the suite's runtime unpredictable and blocks any per-test timeout tighter than the worst case. It should be bounded — a short explicit retry/timeout configuration on the client that test builds, not a shorter overall deadline. **Bounded in production, and the cost was not the retries.** The client is minio-go, not the AWS SDK, and minio's ten retries with a 1s backoff cap account for at most ~9s of it. The 125s is the *credential chain*: with nothing in the environment and no `~/.aws/credentials`, every attempt falls through to `credentials.IAM`, which reaches the link-local `169.254.169.254`. On a host with no instance-metadata service that address is blackholed rather than refused, so each of the eleven attempts waited out the transport's 30-second dial timeout and the read spent its whole `s3StatTimeout` on it — 125s of a 120s bound for a HEAD that failed in the first millisecond. `credentialProviders` now gives the IAM provider a client of this tool's own whose **dial** is bounded at `credentialProbeDialTimeout` = 1s, which is what AWS's own SDKs allow the metadata service by default (`AWS_METADATA_SERVICE_TIMEOUT`) and for the same reason. Everything else is `minio.DefaultTransport`'s own default — proxy from the environment, TLS floor, `SSL_CERT_FILE`, handshake and response-header bounds — so this is one delta, not a second transport policy. **A dial bound and deliberately not `http.Client.Timeout`**: minio-go hands the same client the STS `AssumeRoleWithWebIdentity` exchange IRSA on EKS depends on, a real round trip to `sts..amazonaws.com` that may legitimately exceed a second; a dial bound cannot refuse that exchange, only a connection nothing is answering. **Production benefit, not just the test**: every S3 operation on a non-cloud host — a laptop, a CI runner, an on-prem server, a node whose IMDS is blocked — paid that probe on every retry, so an operator facing an unreachable bucket or a missing credential now gets the refusal in seconds instead of waiting out `s3StatTimeout`/`s3ListTimeout`. **Measured**: the subtest 124.6–124.8s (three consecutive runs) → 4.8–9.8s; the whole `src/internal/app` package 262.6s → 26.3s. The assertion is untouched and the test is neither skipped nor shortened. `TestCredentialProbeIsBounded` pins the bound at the two places it can be lost — the dialer's timeout and the provider actually being handed a client that carries it — because neither is visible in a run's output and a regression in either reads as nothing but slowness; deleting the timeout and nilling the client each redden it + +--- + +## Phase 2W: Findings from running against real Neo4j Enterprise servers + +The first time this feature's mechanism was executed against a real server rather than a fake. +Topology: `neo4j:5-enterprise` (**5.26.30**) and `neo4j:2025-enterprise` (**2025.12.1**) as +host-side containers with a non-loopback backup listener, plus MinIO, on one Docker network. No +cluster required for any of it. Evidence: +`scratchpad/EVIDENCE-real-neo4j-enterprise.md`. + +- [X] T142 **`--expand-commands` made every external capture fail before the database was contacted.** Given that flag, `neo4j-admin` refuses to read any configuration file whose permissions are looser than 0640, and it reads both `neo4j.conf` and `neo4j-admin.conf` — so tightening one is not enough. The image this tool pins for the transient workload ships them world-writable: `neo4j:5-enterprise` has `/var/lib/neo4j/conf/neo4j.conf` at **0777**. The capture aborted with `Failed to read config … does not have the correct file permissions to evaluate commands`, naming a permission the operator never set in an image the operator did not build. Nothing was lost by omitting it: the flag exists so a *deployment's own* configuration may hold command-valued settings, and the transient workload has no such configuration. `neo4jCaptureCommand` now emits it only when `From` is empty, which is already the in-container/remote discriminator, so the internal argv stays byte-identical and FR-015's pin stays green. **No test could have caught this** — the argv is asserted as a string and nothing executed it +- [X] T143 **The seed provider needs a region, not only credentials, and it is not a CLI.** `docs/docs/backup/external-databases.mdx` said the provider "shells out to that tooling"; the failure names the built-in Java SDK directly (`software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain`). With credentials but no region the `CREATE DATABASE` statement **succeeds** and the database then sits `offline` with `statusMessage` = `Unable to load region from any of the providers in the chain …`. Confirmed on both 5.26.30 and 2025.12.1. An operator following the previous text exactly hit that. Documented as a prerequisite in its own right +- [X] T144 **S3-compatible stores work, so restore needs no AWS account.** The built-in `s3` provider honours `AWS_ENDPOINT_URL_S3`: the whole round trip below ran against MinIO. The guide previously said to configure a custom endpoint on the servers without saying how. Built-in providers on both servers, per `CALL dbms.listConfig('dbms.databases.seed_from_uri_providers')`: `S3SeedProvider,CloudSeedProvider` +- [X] T145 **The external e2e Helm overlay pointed at a key the chart does not define.** It wrote `global.infrahubEnv`; the chart's `global` block holds only `kubernetesClusterDomain`, `infrahubRepository`, `infrahubImageFlavor`, `imagePullPolicy`, `imagePullSecrets` and `commonLabels`. Helm accepts an unknown values key in silence, so every component would have stayed pointed at the Neo4j the overlay had just disabled, and the failure would have read as a broken topology rather than a wrong key. Environment is per component and nests under the component's own name **twice**: `infrahubServer.infrahubServer.env` and `infrahubTaskWorker.infrahubTaskWorker.env` — the singly-nested form is wrong the same silent way. Confirming these keys was named in the fixture's own docstring as part of T072. Verified by rendering the fixture's generated overlay against `oci://registry.opsmill.io/opsmill/chart/infrahub:4.33.2` (appVersion 1.11.2, pulls anonymously): `INFRAHUB_DB_ADDRESS` reaches both pod specs, and disabling the subchart leaves `postgresql`, `message-queue` and `cache-master` as the only StatefulSets with no workload or service named `database` anywhere +- [X] T146 **The round trip is proven, and the design it rests on is confirmed.** A capture taken from a *separate container* against the live backup listener — the transient-workload shape — produced `neo4j-.backup`, matching what `neo4jArtifactProduced` reads. Staged to the object store and seeded into a **different** server with `CREATE DATABASE … OPTIONS {existingData:'use', seedURI:'s3://…'}`, the data round-tripped: source `(:Probe {n:1})` → restored `(:Probe {n:1})`, verified by content and not by status. Two design decisions are now confirmed against reality rather than argued: the statement returns **success while the database is still loading**, and `SHOW DATABASE … YIELD currentStatus, statusMessage` reports `offline` plus an actionable cause on failure and `online` with an empty message on success — which is exactly what `waitForNeo4jRestore` polls and why T050 exists. The version gate is also right, not merely plausible: the atomic `CREATE OR REPLACE … seedURI` form is **rejected** at 5.26.30 (`OPTIONS is used to seed a database from URI, this is not allowed with Create and replace database`) and **accepted** at 2025.12.1, so `neo4jSeedReplaceFloor = "2025.01"` sits exactly on the boundary +- [X] T147 **Every probe statement and parser validated against real output.** All four statements return the exact column names the parsers expect, and values come back **quoted** (`"5.26.30"`, `"enterprise"`, `"online"`) — which `splitPlainRow` strips correctly, and which the fixtures already represented. `writer` comes back `TRUE` uppercase; `EqualFold` handles it. **One shape the fixtures do not represent**: on 2025.12.1 `dbms.components()` returns **two** rows — `"2025.12.1", "enterprise"` then `"5", ""`. The first row is read, which is correct, but a reordering would yield version `5` and an empty edition; `requireDeterminedEdition` would stop the run, which is the safe direction. Worth a fixture + +## Phase 2X: High-severity findings from the fourth review round, fixed in severity order + +The review that followed Phase 2W fanned out fourteen finders; their findings came back verified +but were never recorded as tasks, which is the same hand-off failure that lost two residuals +earlier in this feature. These four are the ones that matter: two mean the feature's main path +does not run or cannot be trusted, two mean it can act on the wrong object. Each is fixed by one +clean-context subagent, strictly in this order, with a mutation check (guard removed → test fails +→ guard restored) as part of its evidence. + +- [X] T148 **The external Neo4j restore path is unreachable.** `backup_metadata.go` resolves the edition of an external database through `iops.externalCaptureSourceFor(serviceNeo4j)`, which reads `iops.externalSources` — the map only a *capture* populates. A restore populates `iops.externalRestores`, so the accessor returns nil, the code falls through to `cypher-shell` in a container that does not exist, and `requireDeterminedEdition` stops the run. Reached from `backup.go`, `plakar_restore.go` (two sites). Fix: one accessor that answers "the external source for this service, whichever direction the run is going", and a test that drives a restore against an external Neo4j target and asserts the edition comes from the probe's facts with no exec issued. Mutation: reverting to the capture-only accessor must fail that test **Fixed: one accessor, either direction.** `externalCaptureSourceFor` is gone; `externalDatabaseFor` (`external_capture.go`) answers "what did this run establish about this database outside the deployment" from the capture's record *or* the restore's — `externalRestore` embeds the same `*externalSource` the probe filled — and falls to the memoised location only when neither holds it, so an external database nothing prepared is still refused rather than read as internal. All seven callers route through it: the four capture dispatches (`backupNeo4jEnterprise`, its stream, `backupTaskManagerDB`, its stream), which are unchanged in a capture run because that run records no restores, and the three facts readers — `detectNeo4jEdition`, `isNeo4jCluster` (reached from `restoreNeo4jEnterprise`, the same defect one step further down) and `redactDatabase`. `externalRestoreFor` stays because it answers a different question with a different type: the *workload* a restore holds for the run. Pinned by `TestDetectNeo4jEditionReadsTheGateOfEitherDirection`: a restore record with `Facts.Edition: enterprise` yields `enterprise` with no `cypher-shell` exec; a record with no edition is undetermined, not Community. **Mutation**: dropping the restore lookup fails both restore subtests with the production error (`nothing was prepared to reach it`), which is what `requireDeterminedEdition` was stopping the run on +- [X] T149 **A failed capture can commit an empty snapshot and exit 0.** `plakar_backup.go` checks only the errors returned by `builder.Backup(source)` and `builder.Commit()`. Nothing in non-test code reads kloset's per-source summary (`Summary`, `.Errors`, `Sources[…]` — grep confirms). A finder's probe against the vendored kloset reports both return nil when a record's data factory fails, the failure landing as a VFS error entry (`Summary.Below.Size=0, Errors=1`). If so, a `neo4j-admin` failure becomes a 0-byte snapshot logged as success, with `capture_complete: true` because `recordIncompleteCapture` is never reached. Fix: reproduce the probe as a test first; then make a snapshot whose source recorded an error, or whose database component carries no bytes, fail the run *before* it is reported and *before* the completeness field is written true. Secondary: `buildSnapshotTags` stamps the Neo4j edition on the postgres and metadata snapshots too, so a Postgres-only `--snapshot` restore is refused on an edition it never touches **Probe confirmed against kloset v1.1.0-beta.6, by a running test and not only by reading:** `TestKlosetCommitsASourceWhoseReaderFailed` drives a real repository with a record whose reader fails to open — `Backup()` = nil, `Commit()` = nil, and the *committed* snapshot's source summary reads bytes=0 errors=1; a reader that opens and produces nothing commits as bytes=0 errors=0. The source says why: `importSource` hands a `processRecord` failure to `sourceCtx.recordError` and carries on, and `Commit` sums `totalErrors` into an event and returns nil. So the run's only two checks could not see a failed `neo4j-admin`. **Fixed: the summary is read before the commit.** The per-component create/backup/commit is now `commitComponentSnapshot`, which reads `builder.Header.Sources` after `Backup` and refuses to commit a source that recorded an error or stored no bytes — no component this tool writes is legitimately empty, so the byte check covers all three. The refusal names the component, is recorded through `recordIncompleteCapture` against the service the capture was of (neo4j → `database`, postgres → `task-manager-db`; metadata is not a capture and only fails the run), and returns before `Commit`, so nothing exists to discard for that component — the components already committed go with the run through the existing deferred sweep. Reading before the commit rather than after means the empty snapshot never exists, rather than existing until a discard. Pinned by `TestCommitComponentSnapshotRefusesACaptureThatStoredNothing` (error names the component, `capturesComplete()` false, no snapshot listable, no "Component snapshot created" log) and `TestCommitComponentSnapshotCommitsACaptureThatStoredBytes` (a stream that moved bytes still commits — a check that refused everything would pass the first). **Mutation**: removing the summary check fails all three refusal subtests with `commitComponentSnapshot() = nil`, the silent success observed. **Secondary fixed on the reader, not the writer:** the edition tag is group metadata stamped on every component like the version and the component list, and the listing reads it from whichever snapshot it meets first, so removing it from postgres/metadata would change the layout and still leave every existing repository refused. `restoreSingleSnapshot` now resolves the edition only when the component is neo4j. Pinned by `TestASingleSnapshotRestoreReadsTheEditionForTheComponentItRestores`: a postgres snapshot tagged enterprise restores through to the quiesce guard on a community server; a neo4j snapshot tagged enterprise is still refused before anything is stopped. **Mutation**: reading the tag for every component fails the postgres subtest with the production refusal `cannot restore Enterprise backup on Community edition Neo4j` +- [X] T150 **Ownership claims across a service boundary in both directions.** `nameIsDeploymentOwned` in `kubernetes_ownership.go` finds the service inside the name and a release prefix at the front, but tests no *right-hand* boundary: `infrahub-task-manager-db-exporter` is claimed as `task-manager-db`, so an external RDS reads Internal and `pg_restore --clean --create` runs against a metrics sidecar. The mirror: `pg-task-manager-db-1` (a StatefulSet pod under a prefix that is not a release name) is *not* claimed, reads External, and an in-cluster database is seeded from S3. The strict policy (`unanchoredNamePolicyFor`) applies only when `len(prefixes)==0`, which is not the Helm case. Fix: a name is owned only when what follows `` is empty or a workload-generated suffix (StatefulSet ordinal, ReplicaSet pod hash pair), and the unanchored decision must not depend on whether a release prefix happens to exist. Both cases pinned; mutation on each **Fixed on the right-hand side, once for every tier.** What follows the service in a name now has one reading, `controllerGeneratedSuffix` in `kubernetes_ownership.go`: nothing, one digit-carrying segment (a StatefulSet ordinal, or a hash on its own — the live `infrahub-task-manager-5f8c9` fixture), or a digit-carrying hash and a five-character suffix (a ReplicaSet pod). Anything else — `-exporter`, `-metrics`, `-backup-28123456-x9v2t`, `-exporter-6b8f7c9d4-lm2xq`, and `-db-0`, which is what `infrahub-task-manager-db-0` carries after `task-manager` — is a different workload. `serviceWithGeneratedSuffixIndex` is the one place that reading becomes a claim and all three name tiers go through it: the own-name rule (`isWord`, the old heuristic, is deleted with it), the prefixed rule, and the unanchored tier — which before read no right-hand side at all, so `infrahub-cache-exporter` was the cache in a namespace that declared nothing. `nameMatchesService` keeps no right boundary on purpose: it is the near-miss test `logUnclaimedResource` fires on. Pinned by `TestControllerGeneratedSuffix`, nine new `TestNameIsDeploymentOwned` cases, `TestTheSuffixRuleDoesNotTurnOnWhatElseDeclaredItself` (three prefix sets × both databases and `cache`) and `TestLocationReadsAnExternalDatabaseBesideAComponentNamedAfterIt`, which drives the location decision, both pod resolvers and the workload resolver against a declared release beside `infrahub-task-manager-db-exporter-6b8f7c9d4-lm2xq`: external, `errNoPodsMatched`, no workload; add `infrahub-task-manager-db-0` and it is internal, that pod and that StatefulSet; a transient-marked pod in the claimable shape stays excluded (FR-028). The `TestRunningCheckAnswersTheSelectorsFromTheListing` fixture that asserted the exporter *was* claimed now uses an unlabelled `infrahub-cache-1`, which is what a claimed-but-unlabelled pod is once exporters are not. **Direction B is by design, and is pinned as such rather than fixed.** `pg-task-manager-db-1` is the `postgres-database-0` shape b83099a (T103/T105) chose to refuse for a database: a StatefulSet under a release this deployment never declared is another product's, and admitting it is exactly what routed a co-hosted product's pod into the discovery read. The spec (FR-001/FR-002) does not name the rule; the code, its tests in both labelled and unlabelled namespaces, and that commit do. The invariant T150 asks for already holds for databases: the refusal does not turn on whether `infrahub` is declared beside it — with a release declared the prefix is not this one, with nothing declared `unanchoredNamePolicyFor` refuses — and `TestADatabaseUnderAPrefixNothingDeclaredIsExternalByDesign` pins that the location decision, the running check and the pod resolver read it the same way in both namespaces, so it is external on a decision that is logged, not by omission. For the app services the left-hand rule does flip on a declared release (`othertool-cache-0` claims with nothing declared and not beside `infrahub`): that is the shipped-behaviour policy `unanchoredNameClaims` preserves, pinned by `TestGetPodStatusesWith_ForeignWorkloads`, and reversing it is a design decision, not this fix. **Mutation**: dropping the suffix check on the prefixed path fails five `TestNameIsDeploymentOwned` cases and every reader in the end-to-end test with the production symptom (`location = "internal"`, the exporter resolved as the pod, `deployment/infrahub-task-manager-db-exporter` as the workload); reverting the unanchored tier to `nameMatchesService` fails the invariant test on `infrahub-cache-exporter` with nothing declared; claiming any anchored shape regardless of prefix, or handing databases the permissive policy, fails the direction-B pins (`location = "internal"`, `statuses = [Running]`, `pg-task-manager-db-1` resolved) +- [X] T151 **Merged stderr reaches fields that must hold exactly one value.** `boundedExternalRunner` in `external_timeout.go` calls `runCommandContext`, which is `cmd.CombinedOutput()`. A kubectl warning on stderr therefore lands in `workload.PodUID = strings.TrimSpace(uid)` as `"\n"`, the credential Secret's ownerReference never matches a live pod, GC never collects it, and a Secret holding `PGPASSWORD`/`NEO4J_PASSWORD` outlives the run. Recurrence of a round-2 finding. `runCommandSeparateContext` (`command_executor.go`) already exists and is unused here. Fix: the runner returns stdout alone and carries stderr into the error and the diagnostics — every consumer that parses this output (UID, JSONPath readers, readiness) is downstream of it. Test: a runner whose command writes a warning to stderr and the UID to stdout; the UID field must equal the UID exactly. Mutation: restoring `CombinedOutput` must fail it. **Fixed by giving the package one stdout-only reader and routing both bounded runner constructors through it.** `separatedRunner` (`environment_kubernetes.go`) is now the only podRunner that reads the executor: `runCommandSeparateContext`, stdout returned as the value, stderr folded into the error by the existing `withStderr` (the same answer `execBounded` already gives) and dropped on success. `boundedRunner` — the collect primitives' runner behind `getPodForServiceContext`/`getAllPodsContext`, which had the identical merged shape feeding parsed pod names — and `boundedExternalRunner` both delegate to it, so the external fix did not leave a sibling standing. `TestBoundedExternalRunnerReadsStdoutAlone` drives real child processes through the real `CommandExecutor`: with a warning on stderr, `workload.PodUID` and the Secret manifest's `ownerReferences[].uid` equal the UID exactly; a failure keeps kubectl's stderr in the error and `isForbiddenError` recognises it; a stall is still a timeout after the fold; the collect runner answers the same way. Mutation (putting `runCommandContext` back in `separatedRunner`) fails it with `PodUID = "Warning: kubectl plugin …\n8f14e45f-…"`. **Side effect worth knowing:** `isForbiddenError`/`isAlreadyExists` read `err.Error()`, and on the query and remove paths kubectl's reason used to sit in the *discarded output* while `err` was a bare `exit status 1` — so FR-019's `workloadPermissionError` on the UID read (`transient_workload.go:1968`) and on `remove` could never fire; it can now. **Residuals — merged output still feeding a parsed field, outside this function family:** the *unbounded* `getPodForService`/`GetAllPods` (`environment_kubernetes.go:608`, `:990`) pass `executor.runCommand` (`CombinedOutput`, and no separate unbounded variant exists — adding one is the third runner T151 was told not to add); `getReplicaCount` (`:360`, `Atoi` on merged output — a warning fails the internal scale-down rather than mis-parsing it); `kubernetes_utils.go:42` namespace discovery; `Exec` (`:234`), `ExecContext` (`:1101`), `ExecReplica` (`:1173`) and the collect readers at `:1230`, `:1306`, `:1320`, `:1344`; Docker side `environment_docker.go:109/132/141` exec and `:469/477/496` compose listings +- [ ] T152 [P] **Residual from T148, low.** `externalDatabaseFor` and `externalRestoreFor` each own a copy of the locate→error/nil tail with direction-specific wording. Two copies of one decision is the class this feature keeps paying for; fold into one helper parameterised on the direction's noun. Also noted by T148: a fresh `golangci-lint` cache can report phantom SA5011 in `environment_kubernetes_test.go` (`golangci-lint cache clean` clears it), and a clean `git worktree` of HEAD cannot typecheck because `embedded/neo4jwatchdog/neo4j_watchdog_*` are untracked build artefacts — both are environment notes, not code +- [ ] T153 [P] **Residuals from T149, medium then low.** (i) `restoreBackupGroup`'s edition gate was not examined for the class T149's secondary fixed in `restoreSingleSnapshot` — the Neo4j edition tag is on every snapshot of a group, so any reader that resolves it for a non-Neo4j component refuses on an edition it never touches; check and pin. (ii) `CreatePlakarBackup` still has no end-to-end unit test: `commitComponentSnapshot` is proven by its own tests and reached by grep only (`plakar_backup.go:255`). (iii) The refusal reports counts (`bytes=0 errors=1`), not the reader's own error text, because `StreamingImporter` (`importer.go`) does not remember it — surfacing the `neo4j-admin`/`pg_dump` failure verbatim needs the importer to keep the first error. (iv) The edition probe still *runs* for postgres/metadata single-snapshot restores; only its resolution is scoped. Decision recorded by T149: the zero-byte refusal covers the metadata snapshot too, since metadata is never legitimately empty. Note: the kloset pin asserts `errors==1` exactly and will fail on a kloset upgrade by design; its message says what to do +- [ ] T154 [P] **Residuals from T150 — one spec question, one behaviour change to confirm, the rest low.** (i) *Spec question*: T150 confirmed Direction B is by design — `pg-task-manager-db-1` (an in-cluster database under a prefix no release declares, the shape an operator such as CloudNativePG produces) is refused by the database policy T103/T105 chose, so it reads **External** and is probed and captured through the transient pod rather than scaled and exec'd. Defensible (the tool must not scale an operator's cluster to zero) but it is a position the spec should state, and the diagnostics should say "treated as external because no release claims it" rather than leave the operator to infer it. (ii) *Behaviour change*: hand-rolled `-cache-redis-0` / `-message-queue-rabbitmq-0` shapes that the name tier previously claimed are now refused, so for an **unlabelled** deployment the stop is skipped with only a debug log. Label evidence still claims them first, so Helm deployments are unaffected; confirm the chart's actual pod names for `cache` and `message-queue` and pin them. (iii) The prefix tier is `HasPrefix`, not "prefix immediately before the service": `infrahub-redis-cache-0` is claimed as `cache`. (iv) A letters-only 5-char segment after the service is declined as a hash (~0.03% of ReplicaSet suffixes), and a CronJob named exactly `-database` is still claimed. (v) The left-hand rule flips for *app* services only (`othertool-cache-0` claims when nothing is declared, not when something is) — the shipped `unanchoredNameClaims` design, pinned; reversing it is a product decision. Env note: `rtk` rewrites `grep 'a\|b'` to `rg`, which does not understand `\|` +- [ ] T155 [P] **Residuals from T151 — the same merged-output shape outside the `podRunner` family.** T151 fixed the two runners (`boundedRunner`, `boundedExternalRunner`) by routing both through `separatedRunner`, and in doing so FR-019 Forbidden detection now fires on the query and remove paths (`transient_workload.go`) where kubectl's reason used to sit in discarded output behind a bare `exit status 1`. Still reading `CombinedOutput` into a *parsed* field, each needing its own judgement: the unbounded `getPodForService` / `GetAllPods` (`environment_kubernetes.go` ~608, ~990) go through `executor.runCommand`, for which no separated variant exists; `getReplicaCount` (~360) does `Atoi` on merged output; `kubernetes_utils.go:42` namespace discovery; `Exec` (~234), `ExecContext` (~1101), `ExecReplica` (~1173), whose callers parse `cypher-shell` / `psql` output; the collect readers (~1230, ~1306, ~1320, ~1344); and on Docker, `environment_docker.go` exec (~109, ~132, ~141) and compose listings (~469, ~477, ~496). The exec family is the one that matters most: a driver warning on stderr lands inside a parsed probe answer + +## Phase 2Y: Findings from the first real end-to-end run on Kubernetes (minikube, 2026-09-03) + +The k8s path executed end to end for the first time: fresh minikube profile `infrahub-ext`, chart +`4.33.2` with `neo4j.enabled=false` and no workaround, `neo4j:2025.10.1-enterprise` + MinIO outside +the cluster, mixed topology (Neo4j External, Postgres Internal). Quickstart Scenarios 1, 4 and 5 and +the Neo4j half of 3 all **passed**: capture (1.35 MB Neo4j component, `capture_complete: true`, no +leftovers), refusal without authorisation stopping nothing, seed restore (`online after 1 status +read(s)`, marker gone by Cypher and GraphQL, replicas restored), version gate naming both versions and +writing no archive, `infrahub-collect` creating no transient pod. Evidence: +`scratchpad/EVIDENCE-e2e-minikube.md` (+ `logs/`). Real Helm pod shapes for T150/T154: +`infrahub-cache-master-0`, `infrahub-message-queue-0`, `infrahub-postgresql-0` (task-manager-db), +`prefect-server-` (no release prefix), `infrahub-infrahub-server-`, +`infrahub-infrahub-task-worker-`; no `task-manager-background-svc`. The server uses Neo4j's +default `neo4j` database, not one named `infrahub`. + +- [X] T156 **HIGH, safety: an explicit `--k8s-namespace` falls back to Docker when kubectl fails.** `app.go` ~409–449 builds the backend order `[kubernetes, docker, kubernetes]` when a namespace is given, and a soft failure (`ErrCLIUnavailable`, `ErrEnvironmentNotFound`) `continue`s into the next backend. Demonstrated with `KUBECONFIG=/dev/null`: the tool detected the host's Docker Compose project instead. On a host that also runs an Infrahub compose project — this one does — a `restore` whose cluster is momentarily unreachable would restore into the wrong deployment. An explicit target must pin the backend: when `--k8s-namespace` (or `--project`) is given, only that backend is tried and its failure is the run's failure. Pin with a test that makes kubectl unavailable and asserts no Docker detection was attempted **Fixed 2026-10-13** after it happened in CI: with the namespace check refused, `test_a_revoked_pod_listing_fails_rather_than_switching_to_external_mode` detected another job's Docker Compose project on the runner and started backing it up. `backendOrder` now returns only the named backend when `--k8s-namespace` or `--project` is set, and its detection failure is the run's failure; `TestAnExplicitTargetPinsItsBackend` asserts Docker is never called +- [ ] T157 **HIGH, safety: a target that does not exist is accepted.** `--project ` is accepted (`environment_docker.go` ~50–58) and a nonexistent `--k8s-namespace` is detected as valid with exit 0 (e2e finding 4). Both are the same class as T156: the explicit target is a preference the code checks loosely. Refuse when the named project/namespace is not found, naming what was looked for and where. Pin both +- [ ] T158 **HIGH, functional: the store-size probe reads a metric Neo4j 2025.x does not expose, so every capture on a current server fails first.** `neo4jStoreSizeStatement` (`external_endpoint.go` ~1665) queries JMX `neo4j.database..store.size.total`; on `2025.10.1-enterprise` it answers nothing and the run fails with "server reported no store size", demanding `--external-db-scratch-size`. The comment calls this FR-022's deliberate fallback, but a fallback that fires on the default configuration of every supported current server is the primary path. Find the 2025.x source of the store size (metric rename, `SHOW DATABASES YIELD`, `dbms.database.size`-style call, or the `neo4j.metrics` prefix changing with `server.metrics.prefix`) and validate against the running `neo4j-ext` container; keep the flag as the override, not the requirement +- [ ] T159 **MED, diagnostics: two errors do not say what to do.** (i) A TLS mismatch against a server with TLS disabled (`external_endpoint.go` ~1415–1419) never names `--external-db-insecure-tls`; the operator has to find it in `--help`. (ii) The `CREATE OR REPLACE DATABASE … seedURI` statement is not echoed at info level, so quickstart's "watch for the seed statement" cannot be followed; only the FR-021 poll lines are visible. Name the flag in (i); log the statement (credentials redacted) in (ii) +- [ ] T160 **LOW: minor misreports observed in the run.** `infrahub-collect` reports the external Enterprise database as "service not deployed" / edition community (it should read the external probe or say "external, not collected"); `source_roles` says `unknown` for a standalone server (should say `standalone` or `single`); the absent `task-manager-background-svc` produces skip noise on every run against chart 4.33.2, which does not deploy it; the credential Secret outlives process exit by ~1 s — that is garbage-collector latency after the pod's deletion, expected, but worth one sentence in the guide +- [ ] T161 **Docs: `docs/docs/backup/external-databases.mdx` misled the tester five times.** (D1, D2) the restore examples at ~107–112 and ~175 name no archive and pass no `--latest`, so they are refused on arity (verbatim error in the evidence); (D3) ~252 uses `--backup-id`, which is Plakar-only and silently ignored on the tarball backend — the newest archive is restored instead; (D4) nothing says who talks to the object store from where — `--s3-endpoint` is used by the *tool process* (host vantage point) while the *database* pulls the seed through its own `AWS_ENDPOINT_URL_S3` — nor which host-side credentials the tool needs; (D5) `--external-db-scratch-size` is only under troubleshooting although T158 makes it mandatory today. Also the pile-2 item: `--backup-id` should be refused, not ignored, on the tarball backend + +--- + +## Phase 3: User Story 1 — Round-trip a deployment whose databases live outside the cluster (P1) + +**Goal**: an operator with an external Neo4j Enterprise database, and optionally an external +PostgreSQL, takes a complete backup and restores it with no manual steps on the database hosts. + +**Independent test**: stand up Infrahub on Kubernetes with the databases outside the namespace, back +up, destroy the data, restore, and confirm Infrahub serves the pre-backup data. + +### Backup path + +> **Carried over from T085.** The reverted Phase 3 read a Cypher scalar by taking `lines[0]`, while +> its own comment said it meant to skip the header — so a value shifted by one blank or notice line +> parsed as the header. Whatever T037/T038 reintroduce for parsing a scalar, a version token, member +> roles or a store size must skip the **first non-empty** line, and must read `execBounded`'s stdout, +> which no longer carries stderr. The merged-output half of T085 is fixed; this half had nothing left +> in the tree to fix. + +- [X] T034 [US1] Pass the resolved host list to the Neo4j capture as an ordered `--from` list in `src/internal/app/backup_neo4j.go`, keeping the existing in-pod path untouched for internal deployments. Both paths now build their argv through `neo4jCaptureCommand`, and the internal argv is asserted character for character against the pre-feature command (FR-015). `captureTargets` gained the service arm the reverted version lacked: Neo4j members are re-pointed at the backup listener, PostgreSQL members keep the client port the dump actually travels over +- [X] T035 [US1] Refuse an external Neo4j Community capture with a message stating the mechanism requires direct access to the database's own storage (FR-008), in `src/internal/app/backup_neo4j.go`. Raised at the gate on the edition the probe read over Bolt, which is the last point at which the answer is free — one probe pod spent, no service stopped, no abort window announced. An edition the server did not report is not read as Community; `requireDeterminedEdition` stops that run one step later +- [X] T036 [US1] Derive capture completeness from what the operation reported rather than its exit status, in `src/internal/app/backup_neo4j.go` — exit code 1 means "failed **or** succeeded with some servers uncontactable" and no code distinguishes them (research R6). The signal is a reported success **and** the artefact being present; the output's uncontactable markers only tell the operator which outcome they hit, and never promote a capture to complete. `neo4jArtifactProduced` now reads `-.backup` in full, so neither `2-` nor `-staging-` can satisfy it +- [X] T037 [US1] Query the server store size before capture to size scratch space (FR-022), in `src/internal/app/external_endpoint.go`. The probe runs at the gate and the capture workload is built later from what it read, since a pod's storage is fixed at creation. T090's wiring landed here: `ReapTransientStrays` runs once at the start of the external path, `ExecutionTarget` is how every workload's pod name is obtained, and `ReleaseTransientWorkloads` is one deferred call in `CreateBackup` and `CreatePlakarBackup`. The run ID is minted once per run, not once per database +- [X] T038 [US1] Query cluster member roles at capture time for the metadata, without claiming which member served the capture (FR-005). Observed per supplied endpoint with `unknown` as a value, matched on host because a supplied Neo4j endpoint carries the backup port while the server reports its client address. Consumed now by the line the operator reads before a capture, which states outright that the backup command does not report which member served it; T042 adds the metadata field +- [X] T039 [P] [US1] Point the PostgreSQL capture at the resolved host instead of `localhost` in `src/internal/app/backup_taskmanager.go` — at the member that answered the probe, not `Hosts[0]`, so the dump is taken from the one endpoint the run has evidence is reachable +- [X] T040 [US1] Pass credentials through the mounted secret rather than on the exec command line (FR-014), in `src/internal/app/backup_taskmanager.go` and `src/internal/app/backup_neo4j.go`. `ExecOptions.Env` becomes an `env KEY=VALUE` argv prefix, so it carries only `PGSSLMODE`; `TestNoCredentialReachesAnExternalArgv` builds every command this path constructs and asserts no username or password appears in any argv or exec environment +- [X] T041 [US1] Require certificate verification for external connections unless explicitly opted out, and do not let a discovered insecure-TLS setting silently disable it (FR-029). `--external-db-insecure-tls` is reintroduced with a real consumer — the Bolt URI scheme and `PGSSLMODE`. `--external-db-tls-policy` stays removed: the channel takes no policy name, only a two-valued verify/opt-out, so T095's note that T041 would restore it is superseded and the contract records why + +### Artifact and metadata + +- [X] T042 [P] [US1] Add `source_endpoints`, `source_roles`, `capture_complete` and `external_location` to `BackupMetadata` in `src/internal/app/backup_metadata.go`, leaving `metadataVersion` unchanged (FR-024) +- [X] T043 [US1] Remove the partial artifact from every location it reached and fail the run when a capture is incomplete (FR-012), in `src/internal/app/backup.go` — this is what keeps `retention.go` out of scope, since retention decides from names and ranks and never reads metadata +- [X] T044 [P] [US1] Add tests in `src/internal/app/backup_metadata_test.go` pinning compatibility in both directions: a new tool reads a pre-feature artifact, and a previously released version neither refuses a new artifact nor acts incorrectly on fields it cannot see (FR-024) +- [X] T045 [P] [US1] Add a test asserting an incomplete capture leaves no artifact at any location and leaves retention's decisions identical to a run that failed before writing anything (FR-012) + +### Restore path + +- [X] T046 [US1] Gate external restore behind the dedicated authorisation, distinct from `--force` and from enabling scheduled restore (FR-009), in `src/internal/app/backup.go`. Most of it was already there and reachable — `databaseTarget.reachable` refuses an unauthorised external restore from inside the one constructor every restore path holds, so all three entry points are covered without any of them remembering to ask. What this chunk changed is what remained wrong now that the capability exists: `errExternalRestoreUnavailable` and `externalRestoreRefusal` are gone (the build can perform the restore), the message no longer tells the operator that supplying the flag will not help, and it now names `--force` and scheduled restore as explicitly not conferring it. `ExternalRestoreAuth.Source` gained its first non-test reader — the warning that records which channel authorised writing to unmanaged infrastructure, which is the fact FR-010 turns on +- [X] T047 [US1] Confirm Infrahub workloads have actually stopped before the destructive step, rather than only that a scale-down was requested (FR-026), in `src/internal/app/backup.go`. `confirmAppContainersQuiesced` polls the services the run actually stopped until each reports itself down, and fails otherwise — on all four restore paths. It reads the `stopped` list, which is why the five discarded returns had to be fixed first: confirming the whole service set would fail deployments that legitimately run without a task manager. An undetermined status fails the run, the same answer `stopAppContainers` gives the same uncertainty (64bfdb8) and for the same reason — a destructive step must not proceed on a guess about whether the writers are gone +- [X] T048 [US1] Pre-flight that the database server can itself read the restore artifact, aborting with the database untouched if it cannot (FR-007), in `src/internal/app/backup.go`. `preflightExternalNeo4jRestore` runs after extraction and before `wipeTransientData`, so a refusal leaves Infrahub up and both databases as they were. It stages the artifact in the configured object store under a `seed/` sub-prefix (invisible to retention, which matches keys exactly) and then reads it back through a client built from the URI it will publish rather than the one that wrote it. **What it cannot prove is stated rather than papered over**: the route from the database host to the object store, and the server-side provider tooling and credentials, are invisible from here and no documented interface answers either — so the refusal names them as prerequisites (FR-020) and T050 is what turns one that does not hold into a failed run. No new flag: the artefact store is the existing `--s3-*` surface, and a run with no bucket configured is refused naming those flags +- [X] T049 [US1] Implement the Neo4j remote restore by directing the server to seed the database from the object-store URI, using the current cloud seed provider and the Cypher form matching the server version (research R2), in `src/internal/app/external_restore.go`. `CREATE OR REPLACE DATABASE … OPTIONS {existingData: 'use', seedURI: …}` at or above 2025.01, and `DROP … IF EXISTS` then `CREATE` below it, with the version ordered by `parseDatabaseVersion` so the `5.x`/calendar boundary is not crossed as text. The older form's window in which the database does not exist is warned about rather than left implicit. The URI is validated before it reaches a string literal, and a scheme that is not `s3`/`gs`/`azb` is refused — those are the providers a server carries as shipped, and the rest need `dbms.databases.seed_from_uri_providers` this run cannot see. Every statement goes through `execBoundedAgainst` (T032). Credentials stay in the workload's secret: no argv carries one +- [X] T050 [US1] Poll the server until the restored database reaches a usable state, and fail otherwise (FR-021) — acceptance of the seed request is not evidence the data loaded, because validation happens only when the database starts. `waitForNeo4jRestore` reads `SHOW DATABASE … YIELD currentStatus, statusMessage` until every member reports online, fails immediately on the `Unable to start database` the server reports for a seed it could not fetch or validate, and fails at the operator's bound otherwise. Every row is read, not row zero: a seed that loaded on one member and failed on another is a half-restored database. The staged artifact is deleted only after the database is online — until then the server may still be fetching it — and left in place on failure with its URI in the error. Parsed per the T085 rule: columns by name, first non-empty line skipped, stdout only +- [X] T051 [P] [US1] Point the PostgreSQL restore at the resolved host, reusing the existing TCP branch in `src/internal/app/backup_taskmanager.go`. Both branches now build their argv through `postgresRestoreCommand`, and the two internal ones are asserted character for character against the commands that path produced before this feature existed (FR-015). The external request names the member that answered the probe — not `Hosts[0]`, the same evidence `externalPostgresDumpRequest` keeps — carries no `-U` and no password, and the dump is copied into the workload's scratch volume through a new bounded `CopyToPodContext`. A dump larger than that volume is refused naming `--external-db-scratch-size` before the copy rather than filling it mid-transfer +- [X] T052 [US1] Return Infrahub workloads to their prior scale on every exit path including failure (FR-013), and confirm data is served without further intervention (FR-026). **The known defect is fixed**: all five restore-path callers of `stopAppContainers` now consume the returned list, and each has a deferred `returnAppContainersToScale` that the success path clears — so a restore that fails after quiescing no longer leaves the deployment scaled to zero with the data intact and nothing serving it. Prior scale means prior: a service that was already down is not started. The second clause is landed only as far as this layer can reach — `reportAppContainersRunning` waits for every service the run stopped to report itself running again and warns naming the ones that did not, which is what the tool can observe; it reports rather than fails, because turning a slow image pull into a non-zero exit would report a completed restore as a failed one. **That Infrahub then serves the restored data is not assertable from here** — the tool speaks to the deployment and the databases, never to Infrahub's API — and stays with T056/T058 +- [X] T053 [P] [US1] Add tests for the authorisation gate, the unreadable-URI pre-flight, and the not-online-after-restore failure, in `src/internal/app/external_restore_test.go`, `external_backup_gate_test.go` and `backup_containers_test.go`. 57 tests added against the 1306 baseline (1363 passing, 0 failing). Beyond the three named: both seed statement forms and the version boundary between them, URI refusals, the seed artifact being named rather than guessed, `SHOW DATABASE` parsed per member and behind a notice line, the quiesce confirmation's three outcomes, prior-scale restoration, both `pg_restore` argvs unchanged, and the restore workload's role, deadline and credential handling. The gate file's assertions were rewritten rather than deleted: an unauthorised external restore still refuses before anything is touched, `--force` still does not confer it, and an authorised one now gets past the gate + +### End-to-end + +- [X] T054 [US1] Add external-topology fixtures to `tests/e2e/conftest.py` for a database outside the namespace with Infrahub pointed at it by `INFRAHUB_DB_ADDRESS`. **Written, and the gating half is verified; the deployment half has never executed.** `tests/e2e/conftest.py` now carries `external_db_services`, `requires_external_db`, `external_topology`, `external_seed_store`, `infrahub_k8s_external`, `external_db_env`, `transient_objects` and the label/prefix constants mirrored from `transient_workload.go`, all documented against one env-var contract (`INFRAHUB_TESTING_EXTERNAL_DB` plus per-service address, credential, edition and seed-store variables). Three skip layers were each provoked and observed to fire for their own reason — service not declared, declared but no address, address but wrong edition, and the seed store missing — so a case cannot pass by silently finding no external database. **Not done**: `infrahub_k8s_external` has never been run. It needs `vcluster`, which is absent from the development machine (CI installs it via `loft-sh/setup-vcluster`), and the chart keys in `_external_values_overlay` that point Infrahub at a database it does not deploy (`neo4j.enabled`, `prefect-server.postgresql.enabled`, `global.infrahubEnv`) are **not pinned by anything in this repository** — confirming them against the published chart is part of T072, and until then a real run is expected to need the `INFRAHUB_TESTING_EXTERNAL_HELM_VALUES` override the fixture provides for exactly that. The `_infrahub_values_path` / `_helm_install_infrahub` extraction the external fixture shares with `infrahub_k8s` **was** verified behaviour-preserving (community returns the base file unchanged; enterprise wraps identically; an overlay merges into a branch instead of replacing it) **Ran 2026-10-13:** its cases passed in CI ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425), commit `2bfed31`) and locally against the T072 topology +- [X] T055 [US1] Add `tests/e2e/test_k8s_external_neo4j.py` covering quickstart Scenarios 1 and 4 (mixed topology; version gate in both directions). **Written and collectable; never executed.** Three cases: Scenario 1 asserts the resolved endpoint is logged *before* the transient workload is created (an ordering assertion, not a presence one), that exactly one artifact covers both databases, and that `external_location` marks `database` true and `task-manager-db` not — the per-service verdict FR-001 requires, which a single deployment-wide flag would also satisfy. Scenario 4 runs both directions: an older `--external-db-image-neo4j` must abort with both versions named and **no archive written at all**, a newer one must warn and continue. The two tooling images are env-overridable because the quickstart's compatibility envelope is explicitly unmeasured; the defaults assume a calendar-versioned (`2025.x`) server. **Blocked on the topology, not on the code**: `INFRAHUB_TESTING_EXTERNAL_DB=database` plus `INFRAHUB_TESTING_EXTERNAL_NEO4J_ADDRESS` and `_EDITION=enterprise` (T072) **Ran 2026-10-13:** its cases passed in CI ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425), commit `2bfed31`) and locally against the T072 topology +- [X] T056 [US1] Add `tests/e2e/test_k8s_external_both.py` covering quickstart Scenarios 2, 3 and 5 (both external; full round trip; restore refused without authorisation). **Written and collectable; never executed.** Ordered so the two non-destructive cases observe a deployment nothing has restored into. Scenario 2 asserts per-database `source_endpoints` and `source_roles` and that no transient object survives. Scenario 5 refuses *twice* — once with `--force` and once as `restore --force --latest`, the shape a scheduled restore takes — asserting each refusal names the flag, the configuration key and its own "no Infrahub service was stopped" claim, and then **verifies that claim independently**: replica counts unchanged and the deleted tag still absent. Scenario 3 is the round trip, and asserts the workloads came back to the scale the restore *found* rather than to what the chart declares. **Blocked on**: both services declared external, `_EDITION=enterprise`, and — for Scenario 3 — the `INFRAHUB_TESTING_EXTERNAL_SEED_*` object store the **database host** can read, which is the piece no existing fixture can stand in for (T072) **Ran 2026-10-13:** its cases passed in CI ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425), commit `2bfed31`) and locally against the T072 topology +- [X] T057 [US1] Add the robustness cases from quickstart to the external suites: killed mid-capture, killed between credential creation and workload deletion, one endpoint of several unreachable, database larger than default scratch. **Written and collectable; never executed.** All four are in `test_k8s_external_neo4j.py` rather than split across both suites: each needs only one external database, and a case gated on the weaker topology runs sooner. The kills are pinned to the transient workload reporting ready — via `start_cli`/`wait_for_cli_output` in `tests/helpers/utils.py`, which stream the tool's output to a file and watch it — so "mid-capture" cannot degrade into "before it started" on a slow host, which would assert nothing. The credential case asserts the Secret carries a **Pod owner reference** and then deletes the pod *by hand*, with no tool involvement, which is what FR-023 actually claims. The unreachable-endpoint case appends an RFC 2606 `.invalid` host to the real address list and asserts nothing is retained, because a partial capture and an outright failure share an exit status. **Blocked on** the same topology as T055 (T072) **Ran 2026-10-13:** its cases passed in CI ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425), commit `2bfed31`) and locally against the T072 topology +- [ ] T058 [US1] Run the existing `tests/e2e/test_k8s_*.py` and Docker suites unchanged and confirm identical artifact layout (FR-015, quickstart Scenario 6), including restoring an artifact from the previous released version and restoring an external-origin artifact into an internal deployment. **Run, and it found an FR-015 regression this feature caused — see T123, which this task now blocks on.** Both extra checks the task names are **done and green**, as `tests/e2e/test_docker_external_compat.py` (2 passed): an artifact taken with the previous release's own binary (v2.3.0, asserted not to carry `external_location`, so it genuinely predates this feature) restores with the current binary, and an artifact whose metadata carries `external_location`, `source_endpoints` and `source_roles` restores into an internal Docker deployment with neither `--allow-external-restore` nor any `--s3-*` flag — which is FR-016's real claim, that a restore dispatches on where the database *is* rather than on what the artifact says about where it came from. Of the existing suites: `test_docker_tarball.py` **5 passed** and `test_docker_s3.py` **2 passed**, both unmodified. `test_docker_collect.py` and `test_docker_retention.py` **23 passed, 2 deselected** (the two `enterprise`-marked cases). `test_docker_plakar.py` **3 failed, 1 xpassed** — all three failing at the same point, `run_restore` exiting 1, which is the regression. So the community Docker leg stands at 35 passed / 3 failed / 1 xpassed, with every failure one cause. The whole `test_k8s_*.py` leg is **not runnable on a developer machine**: the `vcluster` session fixture shells out to `vcluster create --driver=docker` and the CLI is absent (`FileNotFoundError: [Errno 2] No such file or directory: 'vcluster'`); CI installs it with `loft-sh/setup-vcluster`. The local minikube context is not what those suites use, so it does not substitute. CI command: `uv run pytest -v -m k8s tests/e2e`. Three local-only deviations were needed and none touched a test: `-p no:pytest-infrahub-performance-test` (infrahub-testcontainers' plugin calls `psutil.cpu_freq()` at session start, which raises on macOS/arm64, so *no* run of `tests/e2e` can start without it), `INFRAHUB_TESTING_IMAGE_VER=1.11.0` (the default 1.8.2 server image fails to boot with `KeyError: 'display-label-process-jinja2'` in Infrahub's own trigger setup, so the stack never comes up), and `DOCKER_HOST` pointed at the OrbStack socket (the `minio_docker` fixture goes through testcontainers' docker-py client, which looks for `/var/run/docker.sock`). **Left unticked**: the k8s leg is unrun and T123 is open, so FR-015 is not demonstrated +- [X] T123 `restore`'s argument contract is validated before the configuration it reads is resolved, so `--backend plakar … restore` with no archive is refused. **A live FR-015 regression on this branch, found by T058; `test_docker_plakar.py` fails three ways on it and `test_k8s_plakar.py` / `test_k8s_plakar_s3.py` will too, so CI is red.** `resolveRestoreInvocation` special-cases the plakar backend, whose no-argument restore has always resolved the latest complete group — but `restoreCmd.Args` calls it with `iops.Config().Backend`, and cobra runs `ValidateArgs` **before** `PersistentPreRunE`. `b867f8e` (T094) moved configuration resolution out of `cobra.OnInitialize` — which cobra runs in `preRun()`, *before* argument validation — into the root command's `PersistentPreRunE`, for the good reason that `OnInitialize` appends to unregisterable package-global state and made every CLI test run every earlier case's resolution. The consequence was not noticed: the `Args` validator now sees the default `tarball` backend whatever `--backend` says, and rejects the plakar invocation with `requires exactly 1 arg(s), only received 0`. Confirmed by comparison, not by inference: the same command line against a binary built from `origin/main` passes the gate and reaches environment detection (evidence in the chunk-6 report). `restore --latest` is **not** affected — `RunE` resolves a second time, by which point the backend is right — so the blast radius is exactly bare `restore` on the plakar backend. Fixing it belongs in its own task rather than in the e2e chunk: the check sits in `Args` deliberately, because the contract spans flags and arguments, and the candidate fix is to move it to `PreRunE`, which still runs before `RunE` but after the configuration is resolved. **Do not** adjust the plakar tests to match the new behaviour; they assert the documented contract **Fixed, and the three plakar e2e cases are green.** Option (A) of the three offered, not the `PreRunE` move this entry proposed: the contract genuinely spans flags and arguments, so it stays in `Args`, and what moved is *when the configuration is resolved*. `ConfigureRootCommand` now holds its resolution as a value behind a `sync.Once`, reachable as `app.ResolveConfiguration()`, and `restoreCmd.Args` asks for it before reading `iops.Config().Backend`; the `PersistentPreRunE` that `b867f8e` introduced calls the same thing and is then the no-op, so that commit's test-pollution fix is untouched and `TestConfigureRootCommandLeavesNoResolutionBehind` still passes. The once-guard is load-bearing rather than decorative: the body is otherwise repeat-safe (viper and env reads, no append to any global), but it warns about a malformed `--external-db-timeout` and a malformed `INFRAHUB_ALLOW_EXTERNAL_RESTORE`, and an operator must not be told twice about one mistake. `iops.Config().Backend` remains the only answer to "which backend?" — option (B) would have added a second one, which is the pattern four earlier chunks spent their time deleting. Moving the check to `PreRunE` (option C) was rejected for changing when the user sees an arity error relative to other validation, on the one command that overwrites data. Why the Go suite missed it is now fixed too: `TestRestoreCommandValidatesBeforeRunning` hands `resolveRestoreInvocation` a backend directly, so it could not see the wiring at all. `restore`'s construction is extracted as `newRestoreCommand(iops)` and the new `TestShippedRestoreCommandInvocationContract` (11 cases) drives *that* command under a real `ConfigureRootCommand` root, both backends, `--backend plakar` and `INFRAHUB_BACKEND=plakar` — it fails 3/11 against the unfixed code with the exact production symptom. `test_docker_plakar.py`: **3 passed, 1 xfailed** (the xfail is the pre-existing dedup case, untouched); no test file was edited. + +**Checkpoint**: User Story 1 is independently deliverable. SC-001, SC-002, SC-003, SC-005, SC-006 +and SC-007 are demonstrable. + +--- + +## Phase 4: User Story 2 — Diagnose an unsupported or misconfigured external deployment (P2) + +**Goal**: an operator whose deployment cannot be backed up learns which condition applies and what +to change, instead of a service-resolution failure. + +**Independent test**: provoke each condition and assert the message names the condition, the +resource, and the operator action. + +- [X] T059 [P] [US2] Report external database locations in `environment detect` output, per database, with where each was resolved to — the first command an operator reaches for when a backup fails (US2 scenario 5) +- [X] T060 [P] [US2] Implement the failure-message contract from `contracts/cli-surface.md` in the paths that produce each condition, ensuring every message names condition, resource and operator action (SC-008) +- [X] T061 [US2] Make the missing-workload-permission failure name the required verbs and resource (FR-019) +- [X] T062 [US2] Make the undiscoverable-endpoint failure state that discovery was attempted, where it looked, and the override flag (FR-004) +- [X] T063 [P] [US2] Add tests asserting each diagnostic message contains its three required elements, in `src/internal/app/external_endpoint_test.go` +- [ ] T064 [US2] Add the diagnostic scenario table from quickstart to the external e2e suites + - **Written, not run.** All six rows are in place — four in `tests/e2e/test_k8s_external_neo4j.py` + (Community refusal, undiscoverable endpoint, revoked pod listing, absent pod-creation + permission) and two in `tests/e2e/test_k8s_external_both.py` (unreadable artifact URI, + readable-but-corrupt artifact). 16 cases collect; all 16 skip by name for the missing + topology (T072). Left unticked because a skipped test is not a passing test: no CI topology + hosts a database outside the cluster, and the k8s e2e leg additionally needs `vcluster`. + Runs as: `INFRAHUB_TESTING_EXTERNAL_DB=database,task-manager-db` plus the address, edition + and seed-store variables documented in `tests/e2e/conftest.py`, then + `uv run pytest -m 'e2e and k8s' tests/e2e/test_k8s_external_neo4j.py tests/e2e/test_k8s_external_both.py`. + The Community row needs `INFRAHUB_TESTING_EXTERNAL_NEO4J_EDITION=community`, so it and the + enterprise cases cannot be satisfied by one topology. + +**Checkpoint**: SC-008 demonstrable at the unit level — every row of the failure-message +contract is asserted for its three elements in `TestFailureMessageContract` +(`src/internal/app/external_endpoint_test.go`), and each message is produced by a path with +non-test callers. End-to-end demonstration waits on T064's topology (T072). +Ships independently of, and after, User Story 1. + +--- + +## Phase 5: Polish & Cross-Cutting Concerns + +- [X] T065 [P] Write `dev/adr/0008-transient-workload-for-external-databases.md` recording the posture change: ADR-0007 establishes offline-by-default for an ops tool in restricted networks, and unlike the benchmark's opt-in pull this image pull is load-bearing and cannot degrade to "skipped" — written, recording that the narrowing is exactly one condition wide (all-internal deployments pull nothing, FR-015), that the image flags stay flag-only while the five placement flags are environment-bound, and that ADR-0003's read-only guarantee survives because `--include-backup` refuses the external path +- [X] T066 [P] Document the customer-side prerequisites in `docs/docs/backup/`: non-loopback backup listener, the seed provider's server-side tooling and credentials, and the client TLS policy — following Diataxis, with Vale and rumdl passing — written as `docs/docs/backup/external-databases.mdx`, a how-to guide covering the server-side `server.backup.listen_address` change on port 6362 (not Bolt 7687), the database host's own egress plus provider tooling and credentials for the seed fetch, the two-valued TLS decision and `--external-db-insecure-tls`, the restore authorisation, the mirrored-image and placement flags, and that seed validation happens only at database start. Vale 0 errors, rumdl clean; registered in `docs/sidebars.ts` +- [X] T067 [P] Document the new flags and the external-restore authorisation in `docs/docs/reference/configuration.mdx` — added an external-database environment-variable table (12 bound keys), a settings-read-from-the-deployment table, an external-database flag table (all 14 flags, with the two image flags marked as having no environment equivalent by design), and an external-restore authorisation subsection stating that `--force` and scheduled restore confer nothing and that `--external-db-tls-policy` does not exist. Every flag verified against `infrahub-backup create --help` / `restore --help` +- [X] T068 [P] Document direct-to-cloud capture as a supported manual escape hatch for customers who cannot meet the seed prerequisites (critique P3) — written as `docs/docs/backup/external-database-direct-to-cloud.mdx`, explicit that the procedure is manual and unautomated, that its artefacts are not restorable with `infrahub-backup restore`, and that completeness must be judged from command output because exit code 1 conflates failure with partial success +- [X] T069 Update `AGENTS.md` architecture section to name `external_endpoint.go` and `transient_workload.go` — **already satisfied by T120**, verified rather than rewritten: §10 names all seven `external_*.go` files and `transient_workload.go`, §11 names the `kubernetes_*.go` files, and the nine non-test files this branch adds (`git diff --diff-filter=A 8186fcf..HEAD`) are each named. No file has shipped since that the list omits — `kubernetes_workloads.go` and `kubernetes_utils.go` both predate the base, and chunk 7's `ReportDatabaseLocations` lives in the already-named `external_endpoint.go`. Two things were **wrong** rather than missing and are fixed: §11's stated acceptance test ("`grep -rn nameMatchesService src/ | grep -v _test` returns nothing outside `kubernetes_ownership.go`") does not hold — the predicate is *defined* in `kubernetes_pod_match.go`, so the claim is now about its call sites and `parseLabelledPods` is attributed to the file it is actually in; and the `transient_workload.go` entry now records that the FR-028 exclusion reads the marker label before the name (T074), since a reader who reintroduced a name-only filter would recreate exactly that hazard +- [X] T070 Run `make fmt`, `make vet`, `make lint` and `make test` and fix everything they report (Principle IV) — `make fmt` clean; `make build` builds all three binaries. `make vet`, `make lint` and `make test` each fail **only** on `tools/neo4jwatchdog`, which uses `unix.Inotify*` and cannot build on darwin — the exclusion AGENTS.md documents, unrelated to `src/`. Nothing in `src/` was reported by any of them, so nothing there needed fixing. One new finding worth recording: `make lint` invokes `golangci-lint` bare, and `make dev-setup` installs it into `$(go env GOPATH)/bin`, which is not on `PATH` — so unscoped `make lint` dies with `No such file or directory` before linting anything. AGENTS.md documents the opposite failure. Scoped runs, which is what the branch has been gated on: `golangci-lint run --build-tags untested_go_version ./src/...` → **0 issues**; `go vet -tags untested_go_version ./src/...` → clean; `go test -tags untested_go_version -count=1 ./src/...` → **417 test functions, 1401 including subtests, 0 failing, 0 skipped** +- [X] T071 Confirm no call site naming `database` or `task-manager-db` was edited; if any was, record why in the plan's Complexity Tracking — audited against the base `8186fcf`, and **one edit exists**, recorded in the plan's Complexity Tracking with the criterion, the counts under both a narrow and a wide reading, and the justification. Criterion: the literal in the *service-argument position* of a service-taking call, not the `neo4j-admin database backup` subcommand word, not a metadata key, not a comment. Base holds 93 lines naming either literal, 48 of them service-argument lines; 3 lines were changed, of which exactly **1** is a service-argument line — `iops.Exec("database", []string{…})` became `iops.Exec("database", neo4jCaptureCommand(…))` in `backupNeo4jEnterpriseStream`/`backupNeo4jEnterprise`, so the external capture runs the identical invocation FR-015 requires. **No service name changed on any of the 48, none was deleted and none was switched to the new `serviceNeo4j`/`serviceTaskManagerDB` constants.** Stated the other way too, because it is the honest number: 15 of the 33 enclosing functions had their bodies edited, all by *inserting* a gate, a restart or a refusal ahead of the existing calls +- [X] T074 Label-guard the two remaining unfiltered name fallbacks — `getPodStatuses`/`IsRunning` and `getPodForServiceWith` in `src/internal/app/environment_kubernetes.go` — so FR-028 does not rest on the transient pod's naming convention in those paths. Raised by chunk 2e (T027 named only `GetAllPods`, `getAllPodsWith` and `ServiceReplicas`). They cannot match the current pod name, so this is latent rather than live: change `transientObjectPrefix` to something containing a service name and it becomes real with no test noticing — **chunk 2Tb's claim about T103 does not hold, and neither half was narrowed away.** T103 label-guarded the *ownership policy* those fallbacks apply (`unanchoredNamePolicyFor`), which is a different question from FR-028's exclusion; the exclusion itself was two name-or-selector tests. The server-side half was already a choke point (`podListingArgsFor` always emits `!infrahub-backup/transient`), but the client-side half — `withoutTransientPods`, which both fallbacks reach — tested `isTransientObjectName`, i.e. `transientObjectPrefix`. So the guarantee did rest on the spelling, in both paths, exactly as the task says. Fixed by making the pod's own declaration the evidence: `podOwnershipJSONPath` now carries `transientLabelMarker` at a fixed field, `labelledPod` gains `Transient`, and `withoutTransientPods` drops on the marker **before** the name (the name test stays for `podNamePhaseJSONPath`, which carries no labels). Latency proved rather than assumed, then closed anyway: `TestTransientExclusionSurvivesARenamedPrefix` asserts today's prefix yields a name no service in `appServicesStoppedForBackup` claims, then hands both fallbacks a marked pod named `infrahub-cache-xdb-neo4j-capture-…` — what a prefix containing a service name would produce. Reverting the one-line guard makes both subtests fail with `IsRunning` reporting `[Running]` for `cache` and the resolver resolving to the transient pod, which is the failure "no test noticing" referred to. **T122 is untouched**: the four selector queries and T097's undetermined-status reporting are exactly as they were, and the fixed field layout is now documented for whoever reproduces those matches from the listing + +### Cannot be completed on a developer machine + +- [X] T072 [INFRA] Provision the CI topology the external suites need: a database outside the cluster under test but reachable from it, with an Enterprise licence, plus an object store the database itself can reach. The plan called this "a fixture change"; it is new CI topology (critique E7). **Substantially de-risked: the topology was stood up on a developer machine on 2026-09-03, so this is provisioning work rather than an open question.** What was proven there, and what a CI job needs: `neo4j:5-enterprise` (or `:2025-enterprise`) as a host-side container with `NEO4J_ACCEPT_LICENSE_AGREEMENT=yes`, `NEO4J_server_backup_enabled=true` and `NEO4J_server_backup_listen__address=0.0.0.0:6362`, publishing 7687 and 6362; MinIO beside it on the **same** container network, publishing 9000. From a pod inside minikube all three of `host.minikube.internal:6362`, `:7687` and `:9000` are reachable, and because the database and the store share a network the store is reachable *by the database* — the distinction the fixture's own comment calls the hard part, and the one that makes restore testable at all. The chart and the Infrahub images both pull **anonymously** (`oci://registry.opsmill.io/opsmill/chart/infrahub:4.33.2`, appVersion 1.11.2), so no registry credential is needed, and `vcluster` installs from Homebrew. **No cloud account is required** — T144 established that the seed provider honours `AWS_ENDPOINT_URL_S3`, so MinIO qualifies as the seed store. Remaining: set `AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_ENDPOINT_URL_S3` on the database container (T143), wire the `INFRAHUB_TESTING_EXTERNAL_*` variables the fixtures read, and get the Infrahub deployment itself healthy — the one step not yet exercised here, and the reason this stays open. Licence note: Enterprise is available under a development licence for exactly this use **Update 2026-09-03 (Phase 2Y): the one unexercised step is now exercised — chart 4.33.2 came up healthy against an external Neo4j with `neo4j.enabled=false` and the doubly-nested env maps, with no workaround; the full round trip passed on minikube (S1–S7 in `EVIDENCE-e2e-minikube.md`). What remains is literally the CI job: stand the two containers up in the workflow, export the `INFRAHUB_TESTING_EXTERNAL_*` variables, pass `--external-db-scratch-size` until T158 lands, and run the suites** **Update 2026-10-13: run locally and written as a CI job.** Local topology on Docker Desktop with a docker-driver vcluster: `neo4j:2025.10.1-enterprise`, `postgres:18-alpine` and MinIO on one Docker network, reached from pods at `host.docker.internal`. Two prerequisites the earlier notes did not list: the servers need TLS, because the tool verifies certificates by default and always encrypts PostgreSQL (FR-029), so both serve a self-signed certificate (Bolt at `tls_level=OPTIONAL`, so Infrahub itself still connects in clear) and the suites pass `--external-db-insecure-tls` through `INFRAHUB_TESTING_EXTERNAL_INSECURE_TLS`; and the scratch size, passed through `INFRAHUB_TESTING_EXTERNAL_SCRATCH_SIZE` until T158 lands. The both-external overlay also needed `prefect-server.secret`, without which the chart does not render. The CI job is in `.github/workflows/ci.yml`, enterprise leg of `e2e-tests-k8s`; it reaches the containers through the Docker bridge gateway. **Passed in CI on 2026-10-13** ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425)): external Neo4j session 11 passed, 1 skipped (the Community case); both-external session 5 passed +- [ ] T073 [INFRA] **The mechanism is now validated; what remains is the customer estate.** T146 proved the whole round trip against real Neo4j Enterprise servers — remote capture from a live backup listener, staged to an object store, seeded into a different server, data verified by content — and T147 validated every probe statement and parser against real output. So the questions this task was written to answer about *the mechanism* are answered: the statement form is accepted (and the version gate sits exactly on the 5.26/2025.01 boundary), `s3` is a built-in provider, and acceptance genuinely does not imply the data loaded, which is why the poll exists. Two prerequisites were discovered by doing it and are now documented rather than assumed: the server needs a **region** as well as credentials (T143), and an S3-compatible endpoint works (T144). **What is still genuinely open is the population, not the plumbing**: whether a given customer's Neo4j hosts can be given egress to an object store at all, whether their operators will accept a server configuration change to enable a non-loopback backup listener, and the decision criterion for falling back to the recreate procedure when they cannot. That is a conversation, not a test. Original text: validate the seed prerequisite against a real target customer's database estate, and record a decision criterion for falling back to the recreate procedure. Restore depends on a single mechanism whose prerequisites are least likely to hold for exactly this population, and both fallbacks were rejected (critique X1). Same conversation should confirm the workload-permission assumption (critique P1) + +--- + +## Phase 6: Database tools in the infrahub-backup image (ADR 0017, FR-030 to FR-033) + +Added 2026-10-12 after review of PR #172. Run in order: 6A, 6B, 6C, 6D. + +### Phase 6A: The image carries the tools + +- [X] T162 Add the database tools to the `Dockerfile` runtime stage per FR-030: a Java runtime, the `neo4j-admin` and `cypher-shell` distributions copied from `neo4j:2025.10.1-enterprise` in a build stage, and the PostgreSQL 18 client tools. Pin every version. Keep the non-root `infrahub` user (uid 1000). Change the base image only if the Neo4j tools do not run on Alpine, and record why in a `Dockerfile` comment. Done: Neo4j tools from `neo4j:2025.10.1-enterprise` under `/opt/neo4j`, Alpine OpenJDK 21 and `postgresql18-client`, pinned by ARG; base moved from alpine:3.21 to alpine:3.23 because 3.21 has no PostgreSQL 18 client, the Neo4j tools run on Alpine +- [X] T163 Add `scripts/check-image-tools.sh` and a `make docker-check-tools` target that build the image and run `neo4j-admin --version`, `cypher-shell --version`, `pg_dump --version`, `pg_restore --version` and `psql --version` inside it as the image's user, failing when any is missing. Run it locally and record the output and the image size before and after T162 in the commit message. Done: the script also refuses an image that runs as root; passed locally on arm64, amd64 build also checked + +### Phase 6B: The transient workload runs the infrahub-backup image + +- [X] T164 Replace `--external-db-image-neo4j` and `--external-db-image-postgres` with one `--external-db-image` flag in `src/internal/app/cli.go`, not bound to the environment (FR-027 comment kept), and update `contracts/cli-surface.md`. Done: `ExternalDBImageFlag`, registered unbound with the FR-027 comment; the version-gate, spec-validation and tooling-version messages name it through `externalDBImageOption`; contract updated with a change-of-record paragraph, an exit-behaviour row and a failure-message row +- [X] T165 Default the workload image to `registry.opsmill.io/opsmill/infrahub-backup:` from the build version (`src/internal/app/app.go`, `utils.go`); remove `defaultExternalDBImageNeo4j` and `defaultExternalDBImagePostgres`. A development build with no `--external-db-image` is refused at the external gate before anything is created or stopped, with a message naming the flag (FR-031). Done: `defaultExternalDBImage()` reads `publishedImageTag(version)` — a canonical release semver with or without the leading "v", tag without the "v" as the release workflow pushes it; empty, hash, `git describe`, `+meta` and `-dirty` are development builds. The refusal (`refuseUnpublishedExternalDBImage`) is in `externalCaptureOps`, after the collect refusal and before the reaper, so both directions stop before any kubectl call +- [X] T166 Set the workload's `runAsUser`, `runAsGroup` and `fsGroup` to the image's declared uid in `transient_workload.go` (`databaseImageUser`), rewrite its comment for the new image, and adjust anything that depended on the vendor images' paths or users (FR-033). Done: `databaseImageUser` is the constant 1000 for every role and database; the comment records that the image declares its user by name, which the kubelet cannot check `runAsNonRoot` against. The `--expand-commands` comment in `backup_neo4j.go` now names the copied 0777 conf in `/opt/neo4j/conf`; nothing else depended on vendor paths (the Job already overrides the entrypoint with `sh -c "exec sleep"`, and busybox `sleep`, `tar`, `keytool` and the system CA bundle are in the image) +- [X] T167 Unit tests for T164 to T166: the single flag and its absence from the environment bindings, the default image tag, the development-build refusal happening before any kubectl write, and the security context in both roles. Done: `external_image_test.go` (tag table, default image, development-build refusal through both gates with a recording kubectl stub and a positive control, empty flag on a release), `TestExternalDBImageIsOneUnboundFlag`, uid 1000 asserted for probe, capture and restore roles on both databases, a failure-message contract row +- [X] T168 [P] Update `docs/docs/backup/external-databases.mdx`, `docs/docs/reference/configuration.mdx` and `dev/knowledge/external-database-backup.md` for the single image and flag: an air-gapped operator mirrors `infrahub-backup` only. Vale and rumdl pass. Done: both docs pages and `dev/knowledge/external-database-backup.md` describe one image and one flag; `container-image-user-semantics.md` gained the named-user case. Vale 0/0/0, rumdl clean + +### Phase 6C: Direct mode when infrahub-backup runs in the namespace + +- [X] T169 Add `src/internal/app/external_local.go`: detection per FR-032 (service-account namespace file equals the target namespace, and every tool the role needs is found on `PATH`), returning the reason when direct mode is not taken, logged at info Done: `directModeHost.detectDirectMode` reads the namespace file and `lookPath` through injectable fields; per-database tool lists (`localToolsFor`) include the shell utilities the sequences call; also refuses a TMPDIR the unquoted shell commands cannot carry. `decideDirectMode` decides once per database and logs one info line naming the mode and the reason +- [X] T170 Implement the local runner behind the operations the capture and restore sequences already call on a transient workload (run a command, read a file, copy out, check free scratch space), so `external_capture.go` and `external_restore.go` do not fork. Credentials reach the child only through its environment (FR-014); scratch is a run-specific directory removed on every exit path; the scratch-size check reads local free space Done: `localWorkload` (a run-specific `os.MkdirTemp` directory) serves the four seams the sequences already go through — `execBounded`, `ExecStreamPipe`, `copyAcrossTransientWorkload`, and a new `refuseUnfittableRestoreDump` — keyed by an execution target `local:--`. `translate` maps `/scratch` onto the directory in argv, env values and copy paths, so the CA material lands there too. Credentials go in `cmd.Env` only. Capture scratch compared with `syscall.Statfs` free space (Linux and macOS, refused elsewhere); released by the sequence and by `releaseTransientWorkloads` +- [X] T171 Wire direct mode into the gate so that, when taken, no Job and no Secret is created and no Job or Secret permission is exercised Done: `externalCaptureOps(services...)` takes direct mode after the collect refusal and before the image refusal, run identifier and reaper, so no kubectl call reaches jobs or secrets. A development build is not refused in direct mode, because no image is run +- [X] T172 Unit tests for T169 to T171: detection true and false cases (no namespace file, other namespace, a missing tool), credentials absent from every argv, scratch removed on success and on failure, and no kubectl create call in direct mode Done: `external_local_test.go` — detection table, `translate` table, no kubectl call in direct mode with a positive control, the Job path kept outside the namespace, the collect refusal, the real PostgreSQL probe run as child processes with credentials in the environment and absent from every argv, scratch removed on success, failure, refusal and by the backstop, the restore fit check and copy, CA trust in scratch, and a real statfs read +- [X] T173 [P] Document direct mode in `docs/docs/backup/external-databases.mdx` (running infrahub-backup as a pod in the namespace, and the permissions it then needs) and add `external_local.go` to `AGENTS.md` Done: new section in `docs/docs/backup/external-databases.mdx` with the exact permission list, network and disk needs, and the flags that have no effect; prerequisite line points to it; `AGENTS.md` lists `external_local.go`; `changelog/172.added.md` extended. Vale 0/0/0, rumdl clean + +### Phase 6D: End-to-end + +- [X] T174 Make the Kubernetes end-to-end job in `.github/workflows/ci.yml` build the image and load it into the test cluster, and pass it with `--external-db-image` Done: the job builds `infrahub-backup:e2e-` and exports `INFRAHUB_TESTING_EXTERNAL_DB_IMAGE`. The vcluster is created by pytest, so loading is in the `external_db_image` fixture: `docker save` piped to `ctr -n k8s.io images import` in each `vcluster.cp.`/`vcluster.node.` container of the docker-driver cluster. That mechanism was observed on a throwaway docker-driver vcluster on 2026-10-12: a loaded tag started a pod with `imagePullPolicy: IfNotPresent` (which the transient Job sets). A `latest` or untagged reference is refused, because it would be pulled. The external suites still skip in CI until T072 declares the topology, so in CI this step only builds the image **Update 2026-10-13:** the enterprise leg now declares the topology (T072), so the external suites run there +- [X] T175 Move `tests/e2e/test_k8s_external_neo4j.py` and any other suite to `--external-db-image`; rework the version-gate cases, which selected an older vendor image, so they still test both directions; fix the owner-kind assertion that still expects `Pod` (ADR 0018) **Written and collectable; never executed** (the external topology, T072, is not provisioned). Every invocation in `test_k8s_external_neo4j.py` and `test_k8s_external_both.py` passes `--external-db-image` from `infrahub_k8s_external["external_db_image"]`; the flag is not bound to the environment (FR-027), so it has to be on the command line. The version-gate cases build an image from the one under test whose `neo4j-admin --version` reports `4.0.0` or `2099.12.0` and which runs the real tool otherwise (`tooling_image_reporting`), so both directions run against the live server whatever its version; both derived images were observed reporting the chosen version and running the real `neo4j-admin` as uid 1000 in a vcluster pod, and a throwaway Go check confirmed both values abort and warn against `5.26.1`, `2025.10.1` and `2026.09.0`. The FR-023 case now asserts the Secret is owned by the run's Job (by UID) and deletes the Job, not the pod. `transient_objects` lists Jobs as well; `TRANSIENT_WRITE_RULES` mirrors `transientWorkloadRBAC`; the permission row also asserts `jobs` is named **Run 2026-10-13 against the T072 topology:** `test_k8s_external_neo4j.py` 10 passed, 1 skipped (the Community case, by design), 1 failed; `test_k8s_external_both.py` 5 passed. The failure is `test_one_unreachable_endpoint_of_several_is_not_a_retained_artifact`, which is outside this task and recorded as T177. The run found and fixed: restores staged every seed in one `seed/` folder, and Neo4j reads the artifacts beside a seed as one backup chain, so a failed restore's staged copy made every later restore fail with "not part of a valid backup chain" (now one folder per artifact); the namespace check dropped kubectl's message; the upload failure did not say nothing was stopped. Test corrections: permission sets that predate workload listing, a race when scaling `infrahub-server` to zero, `source_roles` expected for PostgreSQL, the `task-manager-db` component name, and the post-run check now waits for the cluster to remove the Job's pod **Passed in CI on 2026-10-13** at `2bfed31` ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425)): `test_k8s_external_neo4j.py` with `test_k8s_external_in_namespace.py` 11 passed, 1 skipped (the Community case); `test_k8s_external_both.py` 5 passed. The case that failed in the local run is the one T177 changed +- [X] T176 Add an end-to-end case that runs infrahub-backup as a Job in the namespace with the built image and asserts a capture completes with no transient workload created. If it cannot run locally, say so and give the CI command **Written and collectable; never executed** (needs T072). `tests/e2e/test_k8s_external_in_namespace.py` runs the image as a Job under a service account holding `DIRECT_MODE_RULES` only (no jobs or secrets verbs), and asserts the Job completes, the direct-mode line is logged, no transient workload line is logged, one archive exists and no transient Job, pod or Secret remains. Writing the Role found that the documented permissions were short: `kubectl scale` needs `patch` on `deployments/scale` and `statefulsets/scale` (observed as a Forbidden on a throwaway vcluster), and workload resolution lists deployments and statefulsets. Fixed in `external-databases.mdx` and `kubernetes-backup.mdx`, fragment `changelog/+kubernetes-scale-permissions.fixed.md`. Command, once the topology is declared: `INFRAHUB_TESTING_EXTERNAL_DB=database INFRAHUB_TESTING_EXTERNAL_NEO4J_ADDRESS= INFRAHUB_TESTING_EXTERNAL_NEO4J_EDITION=enterprise INFRAHUB_TESTING_EXTERNAL_DB_IMAGE=infrahub-backup:e2e-local uv run pytest -v -m k8s tests/e2e/test_k8s_external_in_namespace.py` **Passed 2026-10-13** against the T072 topology: `test_a_run_inside_the_namespace_captures_with_no_transient_workload PASSED`; passed in CI on the same day ([run](https://github.com/opsmill/infrahub-backup/actions/runs/38412962425)) +- [X] T177 **Decide FR-012 for one unreachable endpoint of several.** Observed 2026-10-13: with `--neo4j-address` naming a reachable server and an unresolvable one, `neo4j-admin` 2025.10.1 exits 0 and writes the backup, and the tool keeps it with a warning (`classifyNeo4jCapture`: "a capture the vendor called successful"). `test_one_unreachable_endpoint_of_several_is_not_a_retained_artifact` expects a failure, because the Neo4j documentation says this case exits 1. FR-012 says "every requested database is captured", which this capture satisfies. **Decided 2026-10-13 by the PR author: the test changes.** The capture is kept with the warning; the case is now `test_one_unreachable_endpoint_of_several_still_captures_the_database` and asserts success, the warning naming the unreachable endpoint, one archive recorded as complete, and no transient objects left +- [X] T178 **A seed that fails waits for the whole `--external-db-timeout`.** Observed 2026-10-13: Neo4j reported the database `offline` with a `statusMessage` naming the failure ("not part of a valid backup chain") within seconds, and the restore waited the default 2 hours before failing. **Decided 2026-10-13 by the PR author: it does.** `neo4jDatabaseStatus.failed` now also reads `offline` with a non-empty `statusMessage` as a failed seed; `offline` with no message still waits, because a database being replaced can pass through it +- [ ] T179 **The capture fails when the Neo4j backup port requires TLS**, with the vendor image as with the `infrahub-backup` image: the tool gives `neo4j-admin` no client backup TLS policy. Tracked in [#178](https://github.com/opsmill/infrahub-backup/issues/178) + +--- + +## Dependencies + +```text +Phase 1 (Setup) + └── Phase 2 (Foundational) ← blocks everything + ├── Phase 3 (US1, P1) ← MVP + │ └── Phase 4 (US2, P2) depends on US1's error paths existing + └── Phase 5 (Polish) +``` + +- **T013 blocks T014**: the internal/external decision cannot be safe until `getPodForService` + distinguishes an empty match from a failure. +- **T027 blocks T026's safety**: registering the pod for execution before excluding it from + enumeration would put a backup client into troubleshooting bundles. +- **T032 blocks T034 and T049**: an unbounded remote operation on the restore path can hang with + Infrahub scaled down. +- **T043 depends on T036**: completeness must be derivable before it can be acted on. +- **T050 blocks T052**: do not scale Infrahub back up until the restore is known to have landed. +- **T072 and T073 gate feature completion**, not feature start. + +## Parallel Execution Examples + +**Phase 2, after T005–T006 land**: T007, T010, T015, T016, T018, T030, T031, T033 are all tests in +distinct files with no shared state. + +**Phase 3, after T034 lands**: T039 (PostgreSQL host), T042 (metadata fields) and T044/T045 +(compatibility and completeness tests) touch different files. + +**Phase 5**: T065–T068 are four independent documents. + +## Implementation Strategy + +**MVP is Phase 1 + Phase 2 + Phase 3 (User Story 1).** That is the whole value of the feature: the +round trip closes. User Story 2 raises the floor for the population User Story 1 cannot serve and is +genuinely separable. + +Sequence Phase 2 before anything user-visible. Four of its tasks (T013, T027, T032, plus T024's +sizing) are corrections to existing behaviour that the critique showed would otherwise make new +requirements unsatisfiable — they are cheapest to land before the paths above them exist. + +Do not defer T072 to the end. Without the CI topology, every end-to-end task in Phase 3 is written +blind, and Principle IV makes container-touching flows non-optional. diff --git a/src/cmd/infrahub-backup/main.go b/src/cmd/infrahub-backup/main.go index 331c7cba..b490d4a1 100644 --- a/src/cmd/infrahub-backup/main.go +++ b/src/cmd/infrahub-backup/main.go @@ -175,6 +175,93 @@ func resolveRetentionFlags(cmd *cobra.Command, iops *app.InfrahubOps) error { // version is set via ldflags at build time var version string +// newRestoreCommand builds the `restore` command. +// +// It is a function rather than inline in main so a test can drive the very command the +// binary ships — its flag names, its Args, and cobra's ordering between the two — +// instead of a copy of it. A copy is what let T123 through: the invocation contract was +// pinned against a hand-built command that was handed a backend directly, so it stayed +// green while the shipped command read a backend nothing had resolved yet. +func newRestoreCommand(iops *app.InfrahubOps) *cobra.Command { + var restoreExcludeTaskManagerDB bool + var restoreMigrateFormat bool + var restoreLatest bool + var restoreLatestFromS3 bool + var restoreSleepDuration time.Duration + var restoreNeo4jClusterOnlineTimeout time.Duration + + restoreCmd := &cobra.Command{ + Use: "restore [backup-file]", + Short: "Restore Infrahub from a backup archive", + SilenceUsage: true, + // Cobra parses flags before validating arguments, so the invocation contract — + // which spans both — is enforced here, before RunE can act on it. + // + // What cobra does *not* do before validating arguments is run any persistent + // pre-run, which is where --backend is resolved into Configuration. So the + // resolution is asked for by name first (it is idempotent, and the pre-run's + // own call is then the no-op). Reading iops.Config().Backend without that call + // read the default here however the operator invoked the tool, and judged a + // plakar restore — which legitimately names no archive — by the tarball arity + // rule: T123. + Args: func(cmd *cobra.Command, args []string) error { + if err := iops.ResolveConfiguration(); err != nil { + return err + } + _, err := resolveRestoreInvocation(iops.Config().Backend, args, restoreLatest, restoreLatestFromS3) + return err + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateBackendFlags(iops); err != nil { + return err + } + request, err := resolveRestoreInvocation(iops.Config().Backend, args, restoreLatest, restoreLatestFromS3) + if err != nil { + return err + } + if restoreNeo4jClusterOnlineTimeout <= 0 { + return fmt.Errorf("--neo4j-cluster-online-timeout must be greater than zero, got %s", restoreNeo4jClusterOnlineTimeout) + } + iops.Config().Neo4jClusterOnlineTimeout = restoreNeo4jClusterOnlineTimeout + forceRestore, _ := cmd.Flags().GetBool("force") + options := resolveRestoreOptions() + + if request.Latest { + return iops.RestoreLatestBackup(request.S3, restoreExcludeTaskManagerDB, restoreMigrateFormat, restoreSleepDuration, options.DecryptKey, forceRestore, options.ResetDeploymentID) + } + return iops.RestoreBackup(request.Archive, restoreExcludeTaskManagerDB, restoreMigrateFormat, restoreSleepDuration, options.DecryptKey, forceRestore, options.ResetDeploymentID) + }, + } + restoreCmd.Flags().BoolVar(&restoreExcludeTaskManagerDB, "exclude-taskmanager", false, "Skip restoring the task manager database even if present in the archive") + restoreCmd.Flags().BoolVar(&restoreMigrateFormat, "migrate-format", false, "Run neo4j-admin database migrate --to-format=block after the restore completes") + restoreCmd.Flags().DurationVar(&restoreSleepDuration, "sleep", 0, "Sleep duration before restore begins (e.g., 5m, 300s) for manual file transfer") + restoreCmd.Flags().DurationVar(&restoreNeo4jClusterOnlineTimeout, "neo4j-cluster-online-timeout", app.DefaultNeo4jClusterOnlineTimeout, "With a Neo4j cluster: how long to wait for the restored database to come online on every server before the restore fails") + // Registered without a flag variable on purpose: resolveRestoreOptions reads these two + // through viper, and a variable next to the binding is what let the two disagree. + restoreCmd.Flags().String("decrypt-key", "", "Path to private key PEM file for decrypting an encrypted backup") + restoreCmd.Flags().Bool("force", false, "Force restore of incomplete backup group") + restoreCmd.Flags().Bool("reset-deployment-id", false, "Generate a new Root node UUID after restore to detach this instance from the source deployment ID") + restoreCmd.Flags().BoolVar(&restoreLatest, "latest", false, "Restore the most recent backup instead of naming an archive (mutually exclusive with [backup-file])") + restoreCmd.Flags().BoolVar(&restoreLatestFromS3, "s3", false, "With --latest: choose from the configured S3 bucket/prefix instead of the local backup directory") + + // Bind restore flags to Viper for environment variable support (INFRAHUB_). + viper.BindPFlag("decrypt-key", restoreCmd.Flags().Lookup("decrypt-key")) + viper.BindPFlag("reset-deployment-id", restoreCmd.Flags().Lookup("reset-deployment-id")) + // The rest of restore's flags are deliberately not bound, and are read from their own + // flag variables: + // --latest and --s3 are per-invocation switches: a persistent --latest would turn a + // mistyped `restore` into a data-overwriting default, and a persistent --s3 would + // silently move the pool for every restore. + // --exclude-taskmanager, --migrate-format, and --sleep share a name with a bound + // `create` flag, and viper resolves a key through whichever command bound it last, so + // binding them here would make each command answer with the other's value. + // INFRAHUB_SLEEP therefore configures `create` only. + // --neo4j-cluster-online-timeout is not bound because viper turns a malformed duration + // in the environment into 0 instead of reporting it. + + return restoreCmd +} + func main() { app.SetVersion(version) iops := app.NewInfrahubOps() @@ -197,15 +284,9 @@ func main() { var excludeTaskManagerDB bool var encrypt bool var encryptKey string - var restoreExcludeTaskManagerDB bool - var restoreMigrateFormat bool - var restoreLatest bool - var restoreLatestFromS3 bool var s3Upload bool var s3KeepLocal bool var sleepDuration time.Duration - var restoreSleepDuration time.Duration - var restoreNeo4jClusterOnlineTimeout time.Duration // Variables for from-files subcommand var neo4jPath string @@ -285,67 +366,8 @@ func main() { fromFilesCmd.MarkFlagRequired("neo4j-path") createCmd.AddCommand(fromFilesCmd) - - restoreCmd := &cobra.Command{ - Use: "restore [backup-file]", - Short: "Restore Infrahub from a backup archive", - SilenceUsage: true, - // Cobra parses flags before validating arguments, so the invocation contract — - // which spans both — is enforced here, before RunE can act on it. - Args: func(cmd *cobra.Command, args []string) error { - _, err := resolveRestoreInvocation(iops.Config().Backend, args, restoreLatest, restoreLatestFromS3) - return err - }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := validateBackendFlags(iops); err != nil { - return err - } - request, err := resolveRestoreInvocation(iops.Config().Backend, args, restoreLatest, restoreLatestFromS3) - if err != nil { - return err - } - if restoreNeo4jClusterOnlineTimeout <= 0 { - return fmt.Errorf("--neo4j-cluster-online-timeout must be greater than zero, got %s", restoreNeo4jClusterOnlineTimeout) - } - iops.Config().Neo4jClusterOnlineTimeout = restoreNeo4jClusterOnlineTimeout - forceRestore, _ := cmd.Flags().GetBool("force") - options := resolveRestoreOptions() - - if request.Latest { - return iops.RestoreLatestBackup(request.S3, restoreExcludeTaskManagerDB, restoreMigrateFormat, restoreSleepDuration, options.DecryptKey, forceRestore, options.ResetDeploymentID) - } - return iops.RestoreBackup(request.Archive, restoreExcludeTaskManagerDB, restoreMigrateFormat, restoreSleepDuration, options.DecryptKey, forceRestore, options.ResetDeploymentID) - }, - } - restoreCmd.Flags().BoolVar(&restoreExcludeTaskManagerDB, "exclude-taskmanager", false, "Skip restoring the task manager database even if present in the archive") - restoreCmd.Flags().BoolVar(&restoreMigrateFormat, "migrate-format", false, "Run neo4j-admin database migrate --to-format=block after the restore completes") - restoreCmd.Flags().DurationVar(&restoreSleepDuration, "sleep", 0, "Sleep duration before restore begins (e.g., 5m, 300s) for manual file transfer") - restoreCmd.Flags().DurationVar(&restoreNeo4jClusterOnlineTimeout, "neo4j-cluster-online-timeout", app.DefaultNeo4jClusterOnlineTimeout, "With a Neo4j cluster: how long to wait for the restored database to come online on every server before the restore fails") - // Registered without a flag variable on purpose: resolveRestoreOptions reads these two - // through viper, and a variable next to the binding is what let the two disagree. - restoreCmd.Flags().String("decrypt-key", "", "Path to private key PEM file for decrypting an encrypted backup") - restoreCmd.Flags().Bool("force", false, "Force restore of incomplete backup group") - restoreCmd.Flags().Bool("reset-deployment-id", false, "Generate a new Root node UUID after restore to detach this instance from the source deployment ID") - restoreCmd.Flags().BoolVar(&restoreLatest, "latest", false, "Restore the most recent backup instead of naming an archive (mutually exclusive with [backup-file])") - restoreCmd.Flags().BoolVar(&restoreLatestFromS3, "s3", false, "With --latest: choose from the configured S3 bucket/prefix instead of the local backup directory") - - // Bind restore flags to Viper for environment variable support (INFRAHUB_). - viper.BindPFlag("decrypt-key", restoreCmd.Flags().Lookup("decrypt-key")) - viper.BindPFlag("reset-deployment-id", restoreCmd.Flags().Lookup("reset-deployment-id")) - // The rest of restore's flags are deliberately not bound, and are read from their own - // flag variables: - // --latest and --s3 are per-invocation switches: a persistent --latest would turn a - // mistyped `restore` into a data-overwriting default, and a persistent --s3 would - // silently move the pool for every restore. - // --exclude-taskmanager, --migrate-format, and --sleep share a name with a bound - // `create` flag, and viper resolves a key through whichever command bound it last, so - // binding them here would make each command answer with the other's value. - // INFRAHUB_SLEEP therefore configures `create` only. - // --neo4j-cluster-online-timeout is not bound because viper turns a malformed duration - // in the environment into 0 instead of reporting it. - rootCmd.AddCommand(createCmd) - rootCmd.AddCommand(restoreCmd) + rootCmd.AddCommand(newRestoreCommand(iops)) var pruneDryRun bool var pruneForce bool diff --git a/src/cmd/infrahub-backup/main_test.go b/src/cmd/infrahub-backup/main_test.go index 4566b20a..17b79407 100644 --- a/src/cmd/infrahub-backup/main_test.go +++ b/src/cmd/infrahub-backup/main_test.go @@ -598,10 +598,12 @@ func TestResolveRestoreInvocation(t *testing.T) { // invocation is rejected before RunE runs — the only ordering in which "nothing was // restored" is guaranteed (FR-002, FR-003). // -// It builds its own command with its own flag registration rather than reaching into the -// one main() assembles, so it does NOT pin the published flag names: renaming --latest or -// --s3 in main.go would not fail this test. Covering that would mean exporting the command -// construction, which is a larger change than the risk warrants. +// It builds its own command with its own flag registration rather than driving the one +// the binary ships, so it pins neither the published flag names nor where the backend +// comes from: it is handed a backend directly. That is precisely how T123 stayed hidden +// from it, so what the binary actually enforces is pinned by +// TestShippedRestoreCommandInvocationContract instead. This case remains as the +// Args-before-RunE ordering on its own, decoupled from the wiring. func TestRestoreCommandValidatesBeforeRunning(t *testing.T) { tests := []struct { name string @@ -714,3 +716,201 @@ func TestRestoreCommandValidatesBeforeRunning(t *testing.T) { }) } } + +// restoreInvocation is what the shipped `restore` command was left holding once cobra +// had parsed its flags, validated its arguments, and reached its action. +type restoreInvocation struct { + ran bool + args []string + backend app.BackendType + latest bool + s3 bool +} + +// shippedRestoreCommand wires the `restore` command the binary ships under a root +// configured exactly as main configures it, and swaps its action for a recorder so the +// invocation contract can be driven without a deployment to overwrite. +// +// Everything the contract spans is the real thing: the flag names, the Args hook, and +// cobra's ordering between flag parsing, argument validation, and the persistent pre-run +// that resolves --backend. viper is global, so each call resets it and restores it +// afterwards. +func shippedRestoreCommand(t *testing.T, env map[string]string) (*cobra.Command, *restoreInvocation) { + t.Helper() + + viper.Reset() + t.Cleanup(viper.Reset) + + for _, name := range []string{"INFRAHUB_BACKEND", "INFRAHUB_REPO", "INFRAHUB_LATEST", "INFRAHUB_S3"} { + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatalf("unsetting %s = %v, want nil", name, err) + } + } + for name, value := range env { + t.Setenv(name, value) + } + + iops := app.NewInfrahubOps() + root := &cobra.Command{Use: "infrahub-backup", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { return nil }} + app.ConfigureRootCommand(root, iops) + + restoreCmd := newRestoreCommand(iops) + record := &restoreInvocation{} + restoreCmd.RunE = func(cmd *cobra.Command, args []string) error { + record.ran = true + record.args = args + record.backend = iops.Config().Backend + record.latest, _ = cmd.Flags().GetBool("latest") + record.s3, _ = cmd.Flags().GetBool("s3") + return nil + } + + root.AddCommand(restoreCmd) + root.SetOut(io.Discard) + root.SetErr(io.Discard) + + return root, record +} + +// TestShippedRestoreCommandInvocationContract is the invocation contract as the binary +// enforces it, driven through the command main ships rather than a copy of it. +// +// A copy is what let T123 through. TestRestoreCommandValidatesBeforeRunning hands +// resolveRestoreInvocation a backend directly, so it stayed green when the shipped +// command started reading a backend nothing had resolved yet: cobra validates arguments +// before it runs the persistent pre-run, so `--backend plakar restore` was judged by the +// tarball arity rule and refused for naming no archive — a Plakar restore names none. +// +// Every row therefore states which backend the invocation actually ran under, because +// "accepted" is only the right answer for the backend the command line asked for. +func TestShippedRestoreCommandInvocationContract(t *testing.T) { + tests := []struct { + name string + env map[string]string + args []string + wantBackend app.BackendType + wantArgs []string + wantLatest bool + wantS3 bool + // errContains are fragments the refusal must carry: what was wrong, and the flag + // or form that fixes it. + errContains []string + }{ + { + // T123: the regression itself. A Plakar restore takes no positional archive. + name: "plakar: bare restore is accepted", + args: []string{"--backend", "plakar", "--repo", "/repo", "restore"}, + wantBackend: app.BackendPlakar, + }, + { + // The same invocation configured the way an unattended run configures it. + name: "plakar from the environment: bare restore is accepted", + env: map[string]string{"INFRAHUB_BACKEND": "plakar"}, + args: []string{"restore"}, + wantBackend: app.BackendPlakar, + }, + { + // FR-004: --latest is an alias for the no-argument form on this backend, so + // it is accepted and takes the same route. + name: "plakar: --latest is accepted", + args: []string{"--backend", "plakar", "--repo", "/repo", "restore", "--latest"}, + wantBackend: app.BackendPlakar, + wantLatest: true, + }, + { + // resolveRestoreInvocation accepts this: the group to restore comes from + // --backup-id, and the argument is not consulted. It is pinned here so a + // change to that reading is a change to this test. + name: "plakar: a named archive is accepted and not consulted", + args: []string{"--backend", "plakar", "--repo", "/repo", "restore", "infrahub_backup_20260804_120000.tar.gz"}, + wantBackend: app.BackendPlakar, + wantArgs: []string{"infrahub_backup_20260804_120000.tar.gz"}, + }, + { + name: "plakar: --latest --s3 is refused", + args: []string{"--backend", "plakar", "--repo", "/repo", "restore", "--latest", "--s3"}, + errContains: []string{"--s3", "plakar", "--repo"}, + }, + { + // FR-003: the tarball arity rule is unchanged, message included — a typo must + // not become a data-overwriting default. + name: "tarball: bare restore is still refused", + args: []string{"restore"}, + errContains: []string{"requires exactly 1 arg(s), only received 0", "--latest"}, + }, + { + name: "tarball: a named archive is accepted", + args: []string{"restore", "infrahub_backup_20260804_120000.tar.gz"}, + wantBackend: app.BackendTarball, + wantArgs: []string{"infrahub_backup_20260804_120000.tar.gz"}, + }, + { + name: "tarball: --latest is accepted without an archive", + args: []string{"restore", "--latest"}, + wantBackend: app.BackendTarball, + wantLatest: true, + }, + { + name: "tarball: --latest --s3 is accepted", + args: []string{"restore", "--latest", "--s3"}, + wantBackend: app.BackendTarball, + wantLatest: true, + wantS3: true, + }, + { + // FR-002: nothing runs when the command line asks for both. + name: "tarball: --latest with an archive is refused", + args: []string{"restore", "--latest", "infrahub_backup_20260804_120000.tar.gz"}, + errContains: []string{"mutually exclusive"}, + }, + { + name: "tarball: --s3 without --latest is refused", + args: []string{"restore", "--s3"}, + errContains: []string{"--s3", "requires it", "s3://bucket/key"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root, record := shippedRestoreCommand(t, tc.env) + root.SetArgs(tc.args) + + err := root.Execute() + + if len(tc.errContains) > 0 { + if err == nil { + t.Fatalf("Execute(%v) = nil, want a refusal", tc.args) + } + for _, want := range tc.errContains { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to contain %q", err, want) + } + } + if record.ran { + t.Error("the restore ran, want the invocation refused before any restore work") + } + return + } + + if err != nil { + t.Fatalf("Execute(%v) = %v, want nil", tc.args, err) + } + if !record.ran { + t.Fatalf("Execute(%v) accepted nothing, want the restore to proceed", tc.args) + } + if record.backend != tc.wantBackend { + t.Errorf("backend = %q, want %q: the invocation was judged against the wrong backend", record.backend, tc.wantBackend) + } + if strings.Join(record.args, ",") != strings.Join(tc.wantArgs, ",") { + t.Errorf("args = %v, want %v", record.args, tc.wantArgs) + } + if record.latest != tc.wantLatest { + t.Errorf("--latest = %v, want %v", record.latest, tc.wantLatest) + } + if record.s3 != tc.wantS3 { + t.Errorf("--s3 = %v, want %v", record.s3, tc.wantS3) + } + }) + } +} diff --git a/src/internal/app/app.go b/src/internal/app/app.go index 330ee77c..4b96d180 100644 --- a/src/internal/app/app.go +++ b/src/internal/app/app.go @@ -44,6 +44,206 @@ type PlakarConfig struct { BackupID string // Specific backup-id tag for restore (empty = latest complete group) } +// Defaults for reaching a database that lives outside the deployment. +const ( + // defaultNeo4jBackupPort is the port the Neo4j backup service listens on. It + // is the one endpoint detail discovery cannot supply: Infrahub never connects + // to the backup listener, so no setting in the deployment records it + // (research R4). Overridable with --neo4j-backup-port. + defaultNeo4jBackupPort = 6362 + + // defaultExternalDBTimeout bounds a single operation against an external + // database (FR-025). Generous enough for a capture of a large store, finite + // so that an endpoint which accepts a connection and then stops responding + // cannot hang a restore while Infrahub is scaled down. + defaultExternalDBTimeout = 2 * time.Hour + + // externalDBImageRepository is where the release workflow publishes this + // tool's own image. The transient workload runs that image (FR-031, ADR + // 0017): it carries neo4j-admin, cypher-shell and the PostgreSQL client + // at versions the Dockerfile pins, so one tag names one tool set and an + // air-gapped operator mirrors one image rather than two vendor ones. See + // defaultExternalDBImage for how the tag is chosen. + externalDBImageRepository = "registry.opsmill.io/opsmill/infrahub-backup" + + // ExternalDBImageFlag selects the transient workload's image. It is the one + // image flag, settable by flag only (see ConfigureRootCommand), because the + // tool holds workload-creation rights (FR-027). + ExternalDBImageFlag = "external-db-image" +) + +// ExternalRestoreAuthSource names the channel that supplied an external-restore +// authorisation. An unattended restore may only use the configuration channel +// (FR-010), so which channel supplied it is part of the answer, not just whether +// one did. +type ExternalRestoreAuthSource string + +const ( + ExternalRestoreAuthAbsent ExternalRestoreAuthSource = "" + ExternalRestoreAuthFlag ExternalRestoreAuthSource = "flag" + ExternalRestoreAuthConfig ExternalRestoreAuthSource = "config" +) + +// ExternalRestoreAuth is the resolved authorisation to restore into a database +// the deployment does not manage (FR-009). Its zero value is "not authorised", +// so a restore path that never consults it cannot proceed by default. +// +// The channel is the whole of it. It used to carry an `Allowed bool` beside the +// source, and the two answered one question between them: reachable read only +// the bool, describe read only the source. `{Allowed: true}` was therefore a +// representable value that authorised a destructive write into infrastructure +// the deployment does not manage and then logged "no channel this run can +// name" — losing precisely the FR-010 fact the record exists for, which is +// whether an operator was present. Every constructor in cli.go set both +// consistently, so it never happened; it was true by convention, and one +// derivable field is how it stops being a convention. +type ExternalRestoreAuth struct { + Source ExternalRestoreAuthSource +} + +// allowed reports whether a restore into an unmanaged database is authorised, +// which is exactly "some channel said so". It is unexported because +// databaseTarget.reachable is the only reader — FR-009's refusal lives there +// and nowhere else (see databaseTarget.reachable). +func (a ExternalRestoreAuth) allowed() bool { + return a.Source != ExternalRestoreAuthAbsent +} + +// describe names the channel that authorised the restore, for the line that +// records a destructive write to infrastructure the deployment does not manage. +// +// The channel is the part worth naming. An unattended restore can only be +// authorised through configuration, because nothing on this path prompts +// (FR-010), so "--allow-external-restore" and "INFRAHUB_ALLOW_EXTERNAL_RESTORE" +// answer different questions after the fact: whether an operator was present. +func (a ExternalRestoreAuth) describe() string { + switch a.Source { + case ExternalRestoreAuthFlag: + return "--" + AllowExternalRestoreFlag + case ExternalRestoreAuthConfig: + return AllowExternalRestoreEnvVar + default: + return "no channel this run can name" + } +} + +// ExternalDBTLS holds the TLS settings of a client connection to a database, as +// the deployment's own INFRAHUB_DB_TLS_* settings describe it. +type ExternalDBTLS struct { + Enabled bool // INFRAHUB_DB_TLS_ENABLED + CAFile string // INFRAHUB_DB_TLS_CA_FILE + // Insecure records INFRAHUB_DB_TLS_INSECURE as the deployment set it. It + // describes what Infrahub does; it is not on its own an instruction to skip + // certificate verification for the connections this tool opens (FR-029). + Insecure bool +} + +// ExternalDBConfig holds the settings for reaching a database outside the +// deployment. Its zero value engages nothing, and a deployment whose databases +// are all internal never consults it (FR-015). Each field is either an operator +// override of a value discovery supplies, a setting discovery reads from the +// deployment's running components, or -- for the backup port and the tooling +// image -- a value discovery cannot supply at all. +type ExternalDBConfig struct { + // Neo4jAddress overrides the discovered Neo4j host list. It takes the same + // host[:port][,host[:port]...] shape as Infrahub's own INFRAHUB_DB_ADDRESS, + // and a member without an explicit port inherits the client port. + Neo4jAddress string + // Neo4jBackupPort is the port the backup service listens on, not the Bolt + // port Infrahub itself uses. + Neo4jBackupPort int + // Neo4jProtocol is bolt or neo4j (INFRAHUB_DB_PROTOCOL). The routed form is + // required to reach a multi-member deployment. + Neo4jProtocol string + // Neo4jTLS applies to the client connection used to probe the server version + // and to restore, and it is what the deployment configured for Infrahub's + // own connections. It is read, never obeyed: whether *this* tool verifies a + // certificate is InsecureTLS's decision alone (FR-029). + Neo4jTLS ExternalDBTLS + // PostgresAddress overrides the discovered PostgreSQL endpoint as host[:port]. + PostgresAddress string + // PostgresTLS is what the deployment's own task-manager connection URL says + // about TLS — today only its `sslrootcert`, which is the certificate + // authority the deployment's own client verifies that server against and + // therefore the one this tool's client needs. It is read, never obeyed, for + // the same FR-029 reason Neo4jTLS is. + PostgresTLS ExternalDBTLS + // Image is the image the transient workload runs, for both databases: this + // tool's own image at the running binary's version unless --external-db-image + // says otherwise. Empty on a development build that was given no flag, which + // the external path refuses (see refuseUnpublishedExternalDBImage). + Image string + // ScratchSize is the scratch space granted to the transient workload, e.g. + // 50Gi. Empty means derive it from the store size queried before the capture + // (FR-022). + ScratchSize string + // InsecureTLS is this tool's own explicit opt-out from protecting the + // connections it opens to an external server: it skips certificate + // verification, and it is also the only thing that lets the channel be + // unencrypted where the deployment records TLS as disabled. Without it the + // channel is encrypted and verified whatever the deployment configures for + // Infrahub's own connections. + // + // It is deliberately separate from Neo4jTLS.Insecure, which records what the + // deployment configured for Infrahub's own connections. FR-029 turns on that + // separation: a discovered insecure-TLS setting must not silently disable + // verification for the credentials this tool sends, so only this field can. + InsecureTLS bool + // Scheduling carries what the operator's cluster requires of a pod it did + // not author: where the tooling image may be pulled from, and where the pod + // is allowed to run. Every field is empty by default and every empty field + // is absent from the manifest, so a cluster that requires none of it gets + // exactly the pod it got before. + Scheduling ExternalDBScheduling + + // Timeout bounds any single operation against an external database (FR-025). + Timeout time.Duration + // Restore carries the authorisation to restore into a database the + // deployment does not manage, and the channel that supplied it. + Restore ExternalRestoreAuth +} + +// ExternalDBScheduling is the placement and image-access surface for the +// transient workload, held as the operator wrote it and parsed once per run by +// resolveTransientScheduling. +// +// It exists because the image flags FR-027 provides are not usable on their +// own. An operator who must mirror the tooling image into their own registry — +// which is the whole reason those flags exist — has a registry that needs +// credentials, and a pod that can name no pull secret sits in ImagePullBackOff +// until the readiness wait gives up, with nothing the operator can pass to fix +// it. The same is true of a namespace whose only capable nodes are tainted, or +// which admits a pod only under a named service account. +// +// It is held as text rather than as parsed values so that one place — and one +// error message per input, naming the flag — turns the operator's words into +// manifest fields. +type ExternalDBScheduling struct { + // ImagePullSecrets is a comma-separated list of secret names in the + // deployment's namespace, each holding registry credentials for the + // tooling image. + ImagePullSecrets string + + // ServiceAccount is the service account the pod runs under. The pod mounts + // no token whatever this says (automountServiceAccountToken is false), so + // what it carries is the account's own image-pull secrets and whatever the + // namespace's admission requires of one. + ServiceAccount string + + // NodeSelector is a comma-separated list of key=value labels a node must + // carry to run the workload. + NodeSelector string + + // Tolerations is a comma-separated list of taints the workload tolerates, + // each written as kubectl writes one: key=value:Effect, key:Effect, or a + // bare key to tolerate every effect for it. + Tolerations string + + // PriorityClass is the PriorityClass the pod is admitted under, for a + // namespace whose quota or preemption rules require one. + PriorityClass string +} + // Configuration holds the application configuration type Configuration struct { BackupDir string @@ -61,6 +261,9 @@ type Configuration struct { Retention RetentionConfig Backend BackendType Plakar *PlakarConfig + // ExternalDB holds the settings for databases that live outside the + // deployment. It is inert for a deployment whose databases are all internal. + ExternalDB ExternalDBConfig // Neo4jClusterOnlineTimeout is how long a restore into a Neo4j cluster waits for the // restored database to come online on every server before it fails. Neo4jClusterOnlineTimeout time.Duration @@ -74,6 +277,113 @@ type InfrahubOps struct { dockerBackend *DockerBackend kubernetesBackend *KubernetesBackend infrahubInternalAddress string // cached INFRAHUB_INTERNAL_ADDRESS from task-worker + // discoveredNeo4j and discoveredPostgres hold where the deployment's own + // components say each database lives, read alongside the credentials. They + // are kept off Configuration because that carries the operator's overrides, + // which discovery must not overwrite; endpoint resolution reads both. + discoveredNeo4j discoveredEndpoint + discoveredPostgres discoveredEndpoint + // externalSources holds what the run's probe established about each database + // that lives outside the deployment, keyed by the deployment service name. + // It is written once at the gate and read, through externalDatabaseFor, by + // the capture paths and by every decision that asks what the probe read, so + // a database with no container is described by one answer rather than by + // whichever caller asked last. A service absent from both this and + // externalRestores is internal, which is what every capture path branches + // on. + externalSources map[string]*externalSource + // externalRestores holds the workload and the facts this run prepared for + // each database that lives outside the deployment and is about to be + // written to, keyed by the deployment service name. + // + // It is the restore path's counterpart of externalSources, and it is a + // second map rather than a flag on the first because the two hold different + // things: a capture's source is probed and then given a workload sized from + // what the probe read, while a restore's workload is created at the gate and + // held for the whole run — the Bolt seed and the online confirmation both + // run in it, long after the deployment's own components have been scaled to + // zero. A service absent from it is internal, which is what every restore + // path branches on. + externalRestores map[string]*externalRestore + // externalCATrust remembers the trust environment installed in each + // transient workload, keyed by pod, so a workload's keystore is built once + // however many clients ask for it (see installExternalCATrust). + externalCATrust map[string]map[string]string + // deploymentCAFiles remembers the certificate authorities read out of the + // deployment's own components, keyed by ":". + // + // It is what makes the read survive the scale-down. The trust material is + // read out of a running component — infrahub-server for Neo4j, task-manager + // for PostgreSQL — but a transient workload is created per role, and the + // PostgreSQL capture workload is created *after* an offline Neo4j capture + // has scaled task-manager to zero. Read again at that point the `cat` + // fails, the install degrades to the system trust store, and pg_dump then + // fails verify-full against a privately signed server while blaming the + // certificate rather than the container that is no longer there. + // + // The gate probes every external database while the deployment is still up, + // so the entry every later install needs is already here. Only a successful + // read is remembered: a failure says the component could not answer, not + // that there is no authority. + deploymentCAFiles map[string]string + // deploymentCredentials records, per deployment database service name, + // that the username and password this run holds were both read out of the + // deployment's own environment, and from which variables. It is the + // provenance withDeploymentCredentialRefs needs before it may point a + // transient workload at the deployment's Secret instead of creating one: a + // service absent from it has credentials the operator supplied, or + // defaults, and the Secret may hold something else. + deploymentCredentials map[string]deploymentCredentialSource + // deploymentCredentialRefs remembers, per database service name, what + // resolveDeploymentCredentialRefs decided: the Secret keys a transient + // workload references, or nil for "carry an owned Secret". Remembered + // because the component it is read from can be scaled to zero before the + // run's last workload is created. + deploymentCredentialRefs map[string]map[string]transientSecretKeyRef + // incompleteCaptures records, per deployment service name, why a capture of + // that database was not a complete one. It is the run's memory of FR-012: + // the capture path fails as soon as it reaches this verdict, and this is + // what lets the artefact's own removal be decided from the verdict rather + // than from an error value that any layer between could have replaced. + // Empty is the normal state, including for a run that never went external. + incompleteCaptures map[string]string + // externalRun is the identifier every transient object this run creates is + // labelled with, minted once per run rather than once per database — see + // externalRunID. + externalRun string + // transientStraysReaped records that this run has already swept the objects + // earlier runs left behind, so a second external database does not sweep + // again. + transientStraysReaped bool + // externalCaptureForbidden records that this run must not capture a + // database that lives outside the deployment, because doing so means + // creating and then deleting a Pod and a Secret in the operator's + // namespace. It is set by CollectBundle, and read at the one place a + // transient workload becomes possible (externalCaptureOps). Its zero value + // is the backup tool's: capture permitted. + externalCaptureForbidden bool + // directModeHost is what direct mode reads from the host (the + // service-account namespace file, PATH, the temporary directory, free + // space). Nil means the real host; tests supply a fake one. + directModeHost *directModeHost + // directModeDecisions remembers, per database service name, whether this + // run executes that database's tools in its own process (FR-032), so the + // probe, the capture and the restore of one database run in one place and + // the decision is logged once. + directModeDecisions map[string]bool + // localWorkloads are the scratch directories direct mode stands in for + // transient Jobs with, keyed by the execution target handed to the + // sequences. releaseTransientWorkloads removes whatever is left in it. + localWorkloads map[string]*localWorkload + // localWorkloadSeq numbers the execution targets direct mode hands out. + localWorkloadSeq int + // resolveConfiguration runs, at most once, the resolution of Configuration + // from the root command ConfigureRootCommand was given. It is held here so + // ResolveConfiguration can be asked for it by name from a command whose + // argument validation runs before cobra reaches the persistent pre-run. + // Nil until a root command is configured, which is every binary's second + // statement and no test's obligation. + resolveConfiguration func() error // neo4jCredentialLookupErr is why the Neo4j credentials could not be read from the // deployment when the default username or password was used in their place, so a // rejected Neo4j login can be reported against the lookup that caused it. @@ -92,6 +402,11 @@ func NewInfrahubOps() *InfrahubOps { Backend: BackendTarball, Plakar: &PlakarConfig{}, Neo4jClusterOnlineTimeout: DefaultNeo4jClusterOnlineTimeout, + ExternalDB: ExternalDBConfig{ + Neo4jBackupPort: defaultNeo4jBackupPort, + Image: defaultExternalDBImage(), + Timeout: defaultExternalDBTimeout, + }, } return &InfrahubOps{ config: config, @@ -117,6 +432,15 @@ func (iops *InfrahubOps) getKubernetesBackend() *KubernetesBackend { return iops.kubernetesBackend } +// backendOrder is the backends ensureBackend tries, in order. +// +// An explicit target pins its backend: with --k8s-namespace only Kubernetes is +// tried, with --project only Docker, and with both those two in that order. +// Falling through to the other backend would act on a deployment the operator +// did not name. That was observed: with the namespace check refused, a run given +// --k8s-namespace detected an unrelated Docker Compose project on the same host +// and went on to back it up, and a restore would have written to it. Only a run +// that names no target detects. func (iops *InfrahubOps) backendOrder() []EnvironmentBackend { order := []EnvironmentBackend{} add := func(backend EnvironmentBackend) { @@ -137,6 +461,9 @@ func (iops *InfrahubOps) backendOrder() []EnvironmentBackend { if iops.config.DockerComposeProject != "" { add(iops.getDockerBackend()) } + if iops.explicitTarget() { + return order + } add(iops.getDockerBackend()) add(iops.getKubernetesBackend()) @@ -144,20 +471,37 @@ func (iops *InfrahubOps) backendOrder() []EnvironmentBackend { return order } +// explicitTarget reports whether the operator named the deployment to act on. +func (iops *InfrahubOps) explicitTarget() bool { + return iops.config.K8sNamespace != "" || iops.config.DockerComposeProject != "" +} + func (iops *InfrahubOps) ensureBackend() (EnvironmentBackend, error) { if iops.backend != nil { return iops.backend, nil } detectionErrors := []string{} + // missingRuntimes are the backends skipped because the CLI they shell out to + // is not there. They are collected rather than only logged, because when + // nothing is detected they are the answer: FR-018 requires the failure to + // name what is missing, and "no Infrahub environment detected" names a + // deployment when the actual problem is an absent tool on the host. + missingRuntimes := []string{} for _, backend := range iops.backendOrder() { if backend == nil { continue } if err := backend.Detect(); err != nil { - // Soft failures: CLI not available OR environment not found during auto-detect - if errors.Is(err, ErrEnvironmentNotFound) || errors.Is(err, ErrCLIUnavailable) { + // Soft failures: CLI not available OR environment not found during auto-detect. + // With an explicit target there is no auto-detect, so the named backend's + // failure is reported as it is rather than as a soft skip. + if !iops.explicitTarget() && (errors.Is(err, ErrEnvironmentNotFound) || errors.Is(err, ErrCLIUnavailable)) { logrus.Debugf("Skipping %s backend: %v", backend.Name(), err) + if errors.Is(err, ErrCLIUnavailable) { + missingRuntimes = append(missingRuntimes, runtimeCommandOf(backend)) + } + continue } // Hard failures: something went wrong that should be reported @@ -173,6 +517,14 @@ func (iops *InfrahubOps) ensureBackend() (EnvironmentBackend, error) { return nil, fmt.Errorf("environment detection errors: %s", strings.Join(detectionErrors, "; ")) } + // Every backend was skipped for want of its own CLI, so nothing was ever + // asked about a deployment. Saying none was detected would send the operator + // to look at their cluster for a tool that is missing from this host + // (FR-018, constitution Principle III). + if len(missingRuntimes) == len(iops.backendOrder()) && len(missingRuntimes) > 0 { + return nil, fmt.Errorf("cannot detect an Infrahub environment: none of the required command-line tools (%s) was found on PATH, so neither deployment type could be queried", strings.Join(missingRuntimes, ", ")) + } + return nil, fmt.Errorf("no Infrahub environment detected") } @@ -237,6 +589,11 @@ func (iops *InfrahubOps) buildTaskWorkerExecOpts(existingOpts *ExecOptions) *Exe } func (iops *InfrahubOps) ExecStreamPipe(service string, command []string, opts *ExecOptions) (io.ReadCloser, func() error, error) { + // Direct mode (FR-032): a target this run's own process stands in for. + if local := iops.localWorkloadForOptions(opts); local != nil { + return local.stream(command, opts) + } + backend, err := iops.ensureBackend() if err != nil { return nil, nil, err @@ -338,6 +695,30 @@ func (iops *InfrahubOps) DetectEnvironment() error { return nil } +// ReportEnvironment is what `environment detect` runs: the detection above, and +// then where each database lives (US2 scenario 5). +// +// It is a separate method rather than an addition to DetectEnvironment because +// that one is also the first step of `create`, `restore` and the task-manager +// flushes, and none of those wants the report — the backup paths establish the +// same locations a moment later through their own gate, and the flushes touch +// no database location at all. Adding it there would have put two pod listings +// on every flush to serve a report nobody asked for. +// +// Shared by all three binaries through AttachEnvironmentCommands, and it must +// stay correct for the read-only one: it resolves locations and reads the +// deployment's own configuration, and creates nothing (ADR-0003). +func (iops *InfrahubOps) ReportEnvironment() error { + if err := iops.DetectEnvironment(); err != nil { + return err + } + + // After the credentials, because an external endpoint is resolved out of + // the same component environment the credential fetch has just read: in + // this order the deployment is asked once rather than twice. + return iops.ReportDatabaseLocations() +} + func (iops *InfrahubOps) getInfrahubVersion() string { output, err := iops.Exec("infrahub-server", []string{"python", "-c", "import infrahub; print(infrahub.__version__)"}, nil) if err != nil { diff --git a/src/internal/app/app_config.go b/src/internal/app/app_config.go index ba2c63d8..22a54ad1 100644 --- a/src/internal/app/app_config.go +++ b/src/internal/app/app_config.go @@ -2,9 +2,12 @@ package app import ( "fmt" + "net/url" "os" "regexp" + "strconv" "strings" + "unicode" "github.com/jackc/pgx/v5" "github.com/sirupsen/logrus" @@ -34,6 +37,28 @@ var prefectConnectionEnvVars = []string{ "PREFECT_API_DATABASE_CONNECTION_URL", } +// The Infrahub settings that record where the Neo4j database is, read out of +// the server's own environment by applyNeo4jEnvironment. +// +// Named rather than written inline at the one place they are read, because +// FR-004's failure has to state where discovery looked: an operator told only +// "supply --neo4j-address" cannot tell a setting they never set from a +// component this run could not read, and those have different remedies. A +// constant is what keeps the message and the reader from drifting apart. +const ( + neo4jAddressEnvVar = "INFRAHUB_DB_ADDRESS" + neo4jPortEnvVar = "INFRAHUB_DB_PORT" +) + +// The deployment components discovery reads those settings out of. Neo4j's are +// Infrahub's own, so the server carries them; the task manager's endpoint +// exists only inside the Prefect connection URL, which the task manager +// carries. +const ( + componentInfrahubServer = "infrahub-server" + componentTaskManager = "task-manager" +) + // fetchDatabaseCredentials retrieves database credentials from environment or containers func (iops *InfrahubOps) fetchDatabaseCredentials() error { if _, err := iops.ensureBackend(); err != nil { @@ -81,6 +106,9 @@ func (iops *InfrahubOps) loadCredentialsFromEnvironment() { for _, name := range prefectConnectionEnvVars { if value := os.Getenv(name); value != "" { + // The operator handed us the same connection URL the deployment + // keeps, so the external path has nothing left to ask for — if it + // could be read, which is applyPrefectConnection's to decide. iops.applyPrefectConnection(value) break } @@ -103,7 +131,7 @@ func (iops *InfrahubOps) hasPostgresCredentials() bool { // fetchNeo4jCredentials fetches Neo4j credentials from the infrahub-server container func (iops *InfrahubOps) fetchNeo4jCredentials() error { - envOut, err := iops.Exec("infrahub-server", []string{"env"}, nil) + envOut, err := iops.Exec(componentInfrahubServer, []string{"env"}, nil) if err != nil { if line := commandErrorLine(envOut); line != "" { err = fmt.Errorf("%w: %s", err, line) @@ -111,47 +139,257 @@ func (iops *InfrahubOps) fetchNeo4jCredentials() error { return fmt.Errorf("could not read the Neo4j credentials from the infrahub-server container: %w", err) } - for _, line := range strings.Split(envOut, "\n") { - if after, ok := strings.CutPrefix(line, "INFRAHUB_DB_DATABASE="); ok && iops.config.Neo4jDatabase == "" { - iops.config.Neo4jDatabase = after - } - if after, ok := strings.CutPrefix(line, "INFRAHUB_DB_USERNAME="); ok && iops.config.Neo4jUsername == "" { - iops.config.Neo4jUsername = after + iops.applyNeo4jEnvironment(envOut) + // The server's environment has now been read, whatever it turned out to + // contain, so the external path does not ask a second time. + iops.discoveredNeo4j.read = true + + return nil +} + +// ensureNeo4jDiscovery makes sure the deployment's own view of where its Neo4j +// database lives has been read, and reads it if not. +// +// fetchNeo4jCredentials — the only caller of applyNeo4jEnvironment, and so the +// only place the address settings are read — runs only when the credentials are +// not already fully configured. That gate is right for credentials and wrong for +// the address: a deployment whose credentials come from the operator's own +// environment would never reach the code that reads the address, and the address +// is needed precisely when there is no container to fall back on. +// +// This widens the gate in the one direction FR-015 permits. The query is issued +// only after a database has been positively established as external, so a +// deployment whose databases are all internal never reaches here and issues no +// exec it does not issue today. It asks once per run: a deployment that exports +// no address has still been asked, and asking again would not change the answer. +// +// The exec is time-bounded, unlike the credential fetch it widens (FR-025). It +// runs only on the external path, so bounding it leaves fetchNeo4jCredentials — +// the shared path an internal deployment takes — exactly as it was (FR-015). +func (iops *InfrahubOps) ensureNeo4jDiscovery() error { + if iops.discoveredNeo4j.read { + return nil + } + + bound := externalDBControlBound(iops.config) + envOut, err := iops.execBoundedAgainst(bound, "reading the database settings", "the "+componentInfrahubServer+" container", componentInfrahubServer, []string{"env"}, nil) + if err != nil { + return fmt.Errorf("failed to read the database settings from infrahub-server: %w", err) + } + + iops.applyNeo4jEnvironment(envOut) + iops.discoveredNeo4j.read = true + + return nil +} + +// ensurePostgresDiscovery is ensureNeo4jDiscovery for the task-manager database, +// whose endpoint the deployment records only inside the Prefect connection URL. +// +// It applies the endpoint alone and leaves the credentials untouched. The +// credentials already have their own resolution, in which the operator's +// environment wins over the deployment's, and re-reading the connection URL for +// an address must not quietly overrule it. +// +// It is time-bounded for the same reason, and with the same FR-015 argument, as +// ensureNeo4jDiscovery: this is the external path's own read, and +// fetchPostgresCredentials keeps the unbounded exec an internal deployment +// already makes. +func (iops *InfrahubOps) ensurePostgresDiscovery() error { + if iops.discoveredPostgres.read { + return nil + } + + bound := externalDBControlBound(iops.config) + envOut, err := iops.execBoundedAgainst(bound, "reading the task-manager database connection", "the "+componentTaskManager+" container", componentTaskManager, []string{"env"}, nil) + if err != nil { + return fmt.Errorf("failed to read the task-manager database connection from task-manager: %w", err) + } + + iops.applyPrefectEndpointFrom(prefectConnectionFromEnvironment(envOut)) + // Recorded as read even when the deployment exports no connection URL: the + // question has been put, and the answer will not change within the run. + iops.discoveredPostgres.read = true + + return nil +} + +// applyNeo4jEnvironment reads what the deployment's own server says about its +// database out of an `env` dump: the credentials this tool has always read, and +// — additively — where the database is and how a client reaches it. The address +// keys are real Infrahub settings that this tool simply never looked at +// (research R4). +// +// Reading them changes nothing for a deployment that does not set them: an +// absent variable leaves its field untouched, and every field they fill is +// consulted only once a database has been positively established as external +// (FR-015). +func (iops *InfrahubOps) applyNeo4jEnvironment(envOut string) { + env := envValues(envOut) + + if value, ok := env["INFRAHUB_DB_DATABASE"]; ok && iops.config.Neo4jDatabase == "" { + iops.config.Neo4jDatabase = value + } + readUser, readPassword := false, false + if value, ok := env["INFRAHUB_DB_USERNAME"]; ok && iops.config.Neo4jUsername == "" { + iops.config.Neo4jUsername = value + readUser = value != "" + } + if value, ok := env["INFRAHUB_DB_PASSWORD"]; ok && iops.config.Neo4jPassword == "" { + iops.config.Neo4jPassword = value + readPassword = value != "" + } + // Recorded only when this read supplied both: a pair the deployment never + // held together — one value from the operator, one from here — is not + // what any Secret of the deployment's holds (see + // deploymentCredentialSource). + if readUser && readPassword { + iops.recordDeploymentCredentials(serviceNeo4j, deploymentCredentialSource{ + component: componentInfrahubServer, + username: []string{"INFRAHUB_DB_USERNAME"}, + password: []string{"INFRAHUB_DB_PASSWORD"}, + }) + } + + // The host list, in the shape --from already wants: comma-separated + // host[:port] members. Recorded as discovered rather than as the + // configured address, so --neo4j-address still wins over it. + if value := env[neo4jAddressEnvVar]; value != "" { + iops.discoveredNeo4j.Address = value + } + // The Bolt port the application itself uses — what a member of the + // address list inherits when it carries no port. It is not the backup + // listener's port; see resolveNeo4jBackupPort. + if value := env[neo4jPortEnvVar]; value != "" { + if port, err := parsePort(value); err == nil { + iops.discoveredNeo4j.ClientPort = port + } else { + logrus.Warnf("Ignoring %s reported by the deployment: %v", neo4jPortEnvVar, err) } - if after, ok := strings.CutPrefix(line, "INFRAHUB_DB_PASSWORD="); ok && iops.config.Neo4jPassword == "" { - iops.config.Neo4jPassword = after + } + if value := env["INFRAHUB_DB_PROTOCOL"]; value != "" && iops.config.ExternalDB.Neo4jProtocol == "" { + iops.config.ExternalDB.Neo4jProtocol = value + } + if value, ok := env["INFRAHUB_DB_TLS_ENABLED"]; ok { + iops.config.ExternalDB.Neo4jTLS.Enabled = parseDeploymentBool("INFRAHUB_DB_TLS_ENABLED", value, iops.config.ExternalDB.Neo4jTLS.Enabled) + } + if value, ok := env["INFRAHUB_DB_TLS_INSECURE"]; ok { + iops.config.ExternalDB.Neo4jTLS.Insecure = parseDeploymentBool("INFRAHUB_DB_TLS_INSECURE", value, iops.config.ExternalDB.Neo4jTLS.Insecure) + } + if value := env["INFRAHUB_DB_TLS_CA_FILE"]; value != "" { + iops.config.ExternalDB.Neo4jTLS.CAFile = value + } +} + +// envValues reads an `env` dump into a map, one entry per KEY=VALUE line, the +// value being everything after the first `=` so a password carrying one +// survives. A key the dump repeats keeps its last value. The two readers of +// such dumps used to walk the lines themselves, probing each against every +// name they knew; adding a setting now adds a lookup rather than a probe. +func envValues(envOut string) map[string]string { + values := map[string]string{} + for _, line := range strings.Split(envOut, "\n") { + if key, value, ok := strings.Cut(line, "="); ok { + values[key] = value } } - return nil + return values +} + +// parseDeploymentBool reads a boolean setting as the deployment wrote it, +// keeping the current value and warning when the text is not a boolean: a +// setting nobody can read is not evidence for either answer, and quietly +// choosing false for a TLS switch would downgrade a connection the deployment +// meant to protect. +func parseDeploymentBool(name, text string, current bool) bool { + if text == "" { + return current + } + + value, err := strconv.ParseBool(text) + if err != nil { + logrus.Warnf("Ignoring %s reported by the deployment: %q is not a boolean", name, text) + + return current + } + + return value } -// fetchPostgresCredentials fetches PostgreSQL credentials from the task-manager container +// fetchPostgresCredentials fetches PostgreSQL credentials from the task-manager +// container. +// +// The read is recorded only when the connection URL could actually be read, +// which is the same reading loadCredentialsFromEnvironment takes of the same +// return value and for the same reason: a URL that did not parse — or a +// container that exports none at all — applied nothing, and recording the read +// anyway suppresses ensurePostgresDiscovery, the one read that would have +// supplied the endpoint a restore writes into. Recording it unconditionally +// here made this the second instance of the defect the bool was added to fix. +// +// Unlike fetchNeo4jCredentials, "the question has been put" is not the right +// reading: that one reads several independent settings out of one `env` dump, +// so the dump is the answer whatever it contained, while this one reads a +// single URL and has nothing at all if that URL is unusable. func (iops *InfrahubOps) fetchPostgresCredentials() error { - envOut, err := iops.Exec("task-manager", []string{"env"}, nil) + envOut, err := iops.Exec(componentTaskManager, []string{"env"}, nil) if err != nil { return err } - connections := map[string]string{} - for _, line := range strings.Split(envOut, "\n") { - for _, name := range prefectConnectionEnvVars { - if after, ok := strings.CutPrefix(line, name+"="); ok { - connections[name] = after - } - } + iops.applyPostgresEnvironment(envOut) + + return nil +} + +// applyPostgresEnvironment applies the task-manager's `env` dump: the +// connection URL's credentials and endpoint, and — when this read is what +// supplied both the user and the password the run now holds — which variables +// beside the URL held them. +// +// The test is on the values this read applied, not on what was in place +// before it. applyPrefectConnection replaces each value the URL carries, so an +// operator who supplied only part of a connection — a user without a +// password — ends up holding the deployment's pair, and that pair is the one +// the transient workload should read. A URL that carries no password leaves +// the operator's in place, and a pair the deployment never held together is +// not recorded (see deploymentCredentialSource). +func (iops *InfrahubOps) applyPostgresEnvironment(envOut string) { + connConfig := iops.applyPrefectConnection(prefectConnectionFromEnvironment(envOut)) + if connConfig == nil || connConfig.User == "" || connConfig.Password == "" { + return } - // Apply the first match in preference order so newer Prefect naming wins - // regardless of the order the container reports its environment. + // The user and password arrive inside the connection URL, which is one + // value and, in the default chart, one Secret key — nothing a client can + // be handed a username and a password from. What can be referenced is a + // variable beside the URL that held the same value, which is how + // prefect-server renders a database whose user and password come from + // Secret keys. Which variables those are is settled here, while the + // values they held are still at hand, so the decision later needs only + // the pod spec and never the values again. + env := envValues(envOut) + iops.recordDeploymentCredentials(serviceTaskManagerDB, deploymentCredentialSource{ + component: componentTaskManager, + username: envNamesHolding(env, iops.config.PostgresUsername, prefectUserEnvVars), + password: envNamesHolding(env, iops.config.PostgresPassword, prefectPasswordEnvVars), + }) +} + +// prefectConnectionFromEnvironment picks the task-manager connection URL out of +// an `env` dump, preferring the newest Prefect naming regardless of the order +// the container reports its environment. It returns "" when the container +// carries none, which every caller treats as nothing to apply. +func prefectConnectionFromEnvironment(envOut string) string { + connections := envValues(envOut) for _, name := range prefectConnectionEnvVars { if conn := connections[name]; conn != "" { - iops.applyPrefectConnection(conn) - break + return conn } } - return nil + return "" } // applyNeo4jDefaults applies default Neo4j credentials @@ -180,20 +418,18 @@ func (iops *InfrahubOps) applyPostgresDefaults() { } } -// applyPrefectConnection parses and applies Prefect database connection string -func (iops *InfrahubOps) applyPrefectConnection(connStr string) { - if connStr == "" { - return - } - - // Normalize connection string - re := regexp.MustCompile("postgres(.*)://(.*)") - normalized := re.ReplaceAllString(connStr, "postgres://$2") - - connConfig, err := pgx.ParseConfig(normalized) - if err != nil { - logrus.Warnf("Could not parse PREFECT_API_DATABASE_CONNECTION_URL: %v", err) - return +// applyPrefectConnection parses and applies a Prefect database connection +// string: the credentials it carries, where it says the database lives, and +// that the connection has been read. That last is recorded only when the URL +// parsed, because an unreadable URL applied nothing — recording the read anyway +// suppressed the deployment-side read that would have supplied the endpoint: +// one typo in an operator's environment, and it was never discovered at all. +// It returns the parsed connection, or nil when nothing was applied, so a +// caller can tell which values this call supplied. +func (iops *InfrahubOps) applyPrefectConnection(connStr string) *pgx.ConnConfig { + connConfig := parsePrefectConnection(connStr) + if connConfig == nil { + return nil } if connConfig.Database != "" { @@ -205,4 +441,220 @@ func (iops *InfrahubOps) applyPrefectConnection(connStr string) { if connConfig.Password != "" { iops.config.PostgresPassword = connConfig.Password } + + iops.applyPrefectEndpoint(connStr) + iops.discoveredPostgres.read = true + + return connConfig +} + +// applyPrefectEndpointFrom applies only where the task-manager database lives, +// from a connection URL, leaving the credentials as they already resolved. It is +// the external path's discovery read — see ensurePostgresDiscovery for why the +// credentials are deliberately left alone. +func (iops *InfrahubOps) applyPrefectEndpointFrom(connStr string) { + // Parsed first and then read for its endpoint: a string pgx cannot read is + // not a connection URL, and scraping a host out of one anyway would adopt + // text that means nothing. + if parsePrefectConnection(connStr) != nil { + iops.applyPrefectEndpoint(connStr) + } +} + +// applyPrefectEndpoint records where the task-manager database lives. The host +// and port were parsed and thrown away until this feature. They are the only +// record the deployment keeps of where that database is, so an externally hosted +// one is discoverable from exactly here (research R4). Kept as discovered rather +// than as the configured address, so --postgres-address still wins over them. +// +// It reads the connection string rather than pgx's parse of it, and that is the +// whole of the difference. pgx.ParseConfig fills a Host and Port it was not +// given, from libpq's own defaults: PGHOST and PGPORT out of the *operator's* +// environment, and failing those a local socket directory. So a URL that states +// no host — `postgres:///prefect`, which the operator's own PGHOST completes for +// them every day — made the discovered "external database" the machine the tool +// happens to be running on. The restore path writes into what discovery +// resolved. The `!= ""` and `!= 0` guards below could never fire on the pgx +// values, because pgx always supplies both. +func (iops *InfrahubOps) applyPrefectEndpoint(connStr string) { + host, port, caFile := statedPrefectConnection(connStr) + if host != "" { + iops.discoveredPostgres.Address = host + } + if port != 0 { + iops.discoveredPostgres.ClientPort = port + } + if caFile != "" { + iops.config.ExternalDB.PostgresTLS.CAFile = caFile + } } + +// statedPrefectConnection returns what a connection string itself states — the +// host and port the database is at, and the certificate authority it names — +// and nothing else: an empty host means the string named none, not that the +// database is somewhere convenient. +// +// Both shapes libpq accepts are read, because both reach here — the URL form +// the deployment writes, including the query-parameter form a socket directory +// takes (`postgres:///prefect?host=/var/run/postgresql`), and the keyword form +// an operator may have in their environment. One parse yields all three +// values; the endpoint and the CA used to be read by two functions that parsed +// the same string twice. +func statedPrefectConnection(connStr string) (host string, port int, caFile string) { + trimmed := normalizePrefectConnection(connStr) + if trimmed == "" { + return "", 0, "" + } + if !strings.Contains(trimmed, "://") { + fields, err := statedKeywordFields(trimmed) + if err != nil { + // A string libpq itself would refuse states nothing this run can + // trust: reading part of it would discover a truncated host or + // certificate path rather than the one the deployment uses. + logrus.Warnf("Could not read the task-manager database connection string: %v", err) + + return "", 0, "" + } + + return fields["host"], statedPort(fields["port"]), fields["sslrootcert"] + } + + parsed, err := url.Parse(trimmed) + if err != nil { + return "", 0, "" + } + + query := parsed.Query() + host, port = parsed.Hostname(), statedPort(parsed.Port()) + if host == "" { + host = strings.TrimSpace(query.Get("host")) + } + if port == 0 { + port = statedPort(query.Get("port")) + } + + return host, port, strings.TrimSpace(query.Get("sslrootcert")) +} + +// statedKeywordFields parses the keyword/value connection form — +// `host=db.example.com port=5432 sslrootcert='/certs/my ca.crt'` — the way +// libpq does: keywords are separated by whitespace, whitespace is allowed +// around `=`, a value may be single-quoted to hold spaces, and a backslash +// takes the next character literally, so `\'` and `\\` are a quote and a +// backslash. Keywords are lower-cased. +// +// A string libpq would refuse is an error rather than a partial result: a +// keyword with no `=` after it, or a quoted value that is never closed. +func statedKeywordFields(connStr string) (map[string]string, error) { + fields := map[string]string{} + input := []rune(connStr) + i := 0 + + skipSpace := func() { + for i < len(input) && unicode.IsSpace(input[i]) { + i++ + } + } + + for { + skipSpace() + if i >= len(input) { + return fields, nil + } + + keyStart := i + for i < len(input) && input[i] != '=' && !unicode.IsSpace(input[i]) { + i++ + } + key := string(input[keyStart:i]) + if key == "" { + return nil, fmt.Errorf("a value with no keyword before its \"=\" in the keyword/value connection string") + } + + skipSpace() + if i >= len(input) || input[i] != '=' { + return nil, fmt.Errorf("missing \"=\" after %q in the keyword/value connection string", key) + } + i++ + skipSpace() + + var value strings.Builder + if i < len(input) && input[i] == '\'' { + i++ + closed := false + for i < len(input) { + switch input[i] { + case '\\': + i++ + if i < len(input) { + value.WriteRune(input[i]) + i++ + } + case '\'': + i++ + closed = true + default: + value.WriteRune(input[i]) + i++ + } + if closed { + break + } + } + if !closed { + return nil, fmt.Errorf("unterminated quoted value for %q in the keyword/value connection string", key) + } + } else { + for i < len(input) && !unicode.IsSpace(input[i]) { + if input[i] == '\\' { + i++ + if i >= len(input) { + break + } + } + value.WriteRune(input[i]) + i++ + } + } + + fields[strings.ToLower(key)] = value.String() + } +} + +// statedPort is parsePort for a value a connection string may simply not carry: +// the range check is the same one every other port in this package gets, and a +// port that is absent or unusable is reported as "not stated" rather than as an +// error, because the caller's next step is to leave the field alone. +func statedPort(text string) int { + port, err := parsePort(strings.TrimSpace(text)) + if err != nil { + return 0 + } + + return port +} + +// parsePrefectConnection reads a Prefect connection URL, returning nil for an +// absent or unreadable one so every caller has a single thing to check. +func parsePrefectConnection(connStr string) *pgx.ConnConfig { + if connStr == "" { + return nil + } + + connConfig, err := pgx.ParseConfig(normalizePrefectConnection(connStr)) + if err != nil { + logrus.Warnf("Could not parse the task-manager database connection URL: %v", err) + return nil + } + + return connConfig +} + +// normalizePrefectConnection rewrites the driver-qualified schemes Prefect +// writes — postgresql+asyncpg://, postgresql+psycopg:// — to the plain one, so +// the URL is read the same way wherever it is read. +func normalizePrefectConnection(connStr string) string { + return prefectConnectionScheme.ReplaceAllString(strings.TrimSpace(connStr), "postgres://$2") +} + +var prefectConnectionScheme = regexp.MustCompile("postgres(.*)://(.*)") diff --git a/src/internal/app/app_config_test.go b/src/internal/app/app_config_test.go index 7cf9f1d6..bc5720a8 100644 --- a/src/internal/app/app_config_test.go +++ b/src/internal/app/app_config_test.go @@ -1,6 +1,10 @@ package app -import "testing" +import ( + "maps" + "strings" + "testing" +) func newTestOps() *InfrahubOps { return &InfrahubOps{config: &Configuration{}} @@ -95,3 +99,512 @@ func TestLoadCredentialsFromEnvironmentPrefersServerName(t *testing.T) { t.Errorf("password = %q, want %q (PREFECT_SERVER_* should win)", iops.config.PostgresPassword, "server-pw") } } + +// TestApplyPrefectConnectionKeepsHostAndPort is T009's point: the host and port +// the connection URL carries are the deployment's only record of where the +// task-manager database lives, so they are kept instead of discarded. They land +// in the discovered endpoint rather than in the configured address, so that +// --postgres-address still overrides them. +func TestApplyPrefectConnectionKeepsHostAndPort(t *testing.T) { + tests := []struct { + name string + connStr string + host string + port int + }{ + { + name: "external host and explicit port", + connStr: "postgresql+asyncpg://prefect:prefect-rocks@pg.example.com:6432/prefect", + host: "pg.example.com", + port: 6432, + }, + { + name: "in-deployment service name", + connStr: "postgres://prefect:s3cr3t@task-manager-db:5432/prefect", + host: "task-manager-db", + port: 5432, + }, + { + name: "empty connection string leaves the endpoint untouched", + connStr: "", + }, + { + name: "unparseable connection string leaves the endpoint untouched", + connStr: "postgres://prefect:pw@[::1:5432/prefect", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectConnection(tt.connStr) + + if iops.discoveredPostgres.Address != tt.host { + t.Errorf("discovered address = %q, want %q", iops.discoveredPostgres.Address, tt.host) + } + if iops.discoveredPostgres.ClientPort != tt.port { + t.Errorf("discovered client port = %d, want %d", iops.discoveredPostgres.ClientPort, tt.port) + } + if iops.config.ExternalDB.PostgresAddress != "" { + t.Errorf("configured address = %q, want it left to the operator's override", iops.config.ExternalDB.PostgresAddress) + } + }) + } +} + +// TestApplyNeo4jEnvironment is T008's point: the address, port, protocol and TLS +// settings the deployment already exports are read rather than ignored. +func TestApplyNeo4jEnvironment(t *testing.T) { + tests := []struct { + name string + envOut string + wantAddress string + wantPort int + wantProtocol string + wantTLS ExternalDBTLS + }{ + { + name: "single external host", + envOut: strings.Join([]string{ + "INFRAHUB_DB_ADDRESS=neo4j.example.com", + "INFRAHUB_DB_PORT=7688", + "INFRAHUB_DB_PROTOCOL=bolt", + }, "\n"), + wantAddress: "neo4j.example.com", + wantPort: 7688, + wantProtocol: "bolt", + }, + { + name: "cluster member list and routed protocol", + envOut: strings.Join([]string{ + "PATH=/usr/local/bin", + "INFRAHUB_DB_ADDRESS=core-1:7688,core-2,core-3", + "INFRAHUB_DB_PROTOCOL=neo4j", + }, "\n"), + wantAddress: "core-1:7688,core-2,core-3", + wantProtocol: "neo4j", + }, + { + name: "TLS settings are read as the deployment wrote them", + envOut: strings.Join([]string{ + "INFRAHUB_DB_ADDRESS=neo4j.example.com", + "INFRAHUB_DB_TLS_ENABLED=true", + "INFRAHUB_DB_TLS_INSECURE=1", + "INFRAHUB_DB_TLS_CA_FILE=/certs/ca.pem", + }, "\n"), + wantAddress: "neo4j.example.com", + wantTLS: ExternalDBTLS{Enabled: true, Insecure: true, CAFile: "/certs/ca.pem"}, + }, + { + name: "TLS explicitly disabled stays disabled", + envOut: strings.Join([]string{ + "INFRAHUB_DB_TLS_ENABLED=false", + }, "\n"), + }, + { + name: "unreadable port and TLS switch are ignored rather than guessed", + envOut: strings.Join([]string{ + "INFRAHUB_DB_ADDRESS=neo4j.example.com", + "INFRAHUB_DB_PORT=bolt", + "INFRAHUB_DB_TLS_ENABLED=maybe", + }, "\n"), + wantAddress: "neo4j.example.com", + }, + { + name: "empty values are treated as unset", + envOut: strings.Join([]string{ + "INFRAHUB_DB_ADDRESS=", + "INFRAHUB_DB_PORT=", + "INFRAHUB_DB_PROTOCOL=", + "INFRAHUB_DB_TLS_CA_FILE=", + }, "\n"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iops := newTestOps() + iops.applyNeo4jEnvironment(tt.envOut) + + if iops.discoveredNeo4j.Address != tt.wantAddress { + t.Errorf("discovered address = %q, want %q", iops.discoveredNeo4j.Address, tt.wantAddress) + } + if iops.discoveredNeo4j.ClientPort != tt.wantPort { + t.Errorf("discovered client port = %d, want %d", iops.discoveredNeo4j.ClientPort, tt.wantPort) + } + if iops.config.ExternalDB.Neo4jProtocol != tt.wantProtocol { + t.Errorf("protocol = %q, want %q", iops.config.ExternalDB.Neo4jProtocol, tt.wantProtocol) + } + if iops.config.ExternalDB.Neo4jTLS != tt.wantTLS { + t.Errorf("TLS = %+v, want %+v", iops.config.ExternalDB.Neo4jTLS, tt.wantTLS) + } + if iops.config.ExternalDB.Neo4jAddress != "" { + t.Errorf("configured address = %q, want it left to the operator's override", iops.config.ExternalDB.Neo4jAddress) + } + }) + } +} + +// TestApplyNeo4jEnvironmentCredentialsUnchanged is FR-015: a deployment that sets +// none of the new settings behaves exactly as before. The credentials are read as +// they always were, an already-configured value is still not overwritten, and +// nothing external is engaged. +func TestApplyNeo4jEnvironmentCredentialsUnchanged(t *testing.T) { + envOut := strings.Join([]string{ + "PATH=/usr/local/bin", + "INFRAHUB_DB_DATABASE=infrahub", + "INFRAHUB_DB_USERNAME=neo4j", + "INFRAHUB_DB_PASSWORD=admin", + }, "\n") + + iops := newTestOps() + iops.applyNeo4jEnvironment(envOut) + + if iops.config.Neo4jDatabase != "infrahub" { + t.Errorf("database = %q, want %q", iops.config.Neo4jDatabase, "infrahub") + } + if iops.config.Neo4jUsername != "neo4j" { + t.Errorf("username = %q, want %q", iops.config.Neo4jUsername, "neo4j") + } + if iops.config.Neo4jPassword != "admin" { + t.Errorf("password = %q, want %q", iops.config.Neo4jPassword, "admin") + } + if (iops.discoveredNeo4j != discoveredEndpoint{}) { + t.Errorf("discovered endpoint = %+v, want nothing discovered", iops.discoveredNeo4j) + } + if (iops.config.ExternalDB != ExternalDBConfig{}) { + t.Errorf("external configuration = %+v, want it untouched", iops.config.ExternalDB) + } + + // An existing value still wins over the container's, as it did before. + configured := newTestOps() + configured.config.Neo4jPassword = "from-the-operator" + configured.applyNeo4jEnvironment(envOut) + + if configured.config.Neo4jPassword != "from-the-operator" { + t.Errorf("password = %q, want the already-configured value to survive", configured.config.Neo4jPassword) + } +} + +// TestApplyPrefectEndpointAdoptsOnlyAStatedHost is finding 18 at the point +// where it decides what the tool will connect to — and, on the restore path, +// write into. +// +// pgx.ParseConfig fills a Host and Port the connection string did not give it, +// from libpq's defaults: PGHOST and PGPORT out of the *operator's* own +// environment, and failing those a local socket directory. Adopting its parse +// therefore turned a URL with no host into a "discovered external database" +// pointing at whatever machine the tool was running on. +func TestApplyPrefectEndpointAdoptsOnlyAStatedHost(t *testing.T) { + t.Setenv("PGHOST", "the-operators-laptop.local") + t.Setenv("PGPORT", "15432") + + t.Run("a URL that states no host discovers nothing", func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectConnection("postgres:///prefect") + if !iops.discoveredPostgres.read { + t.Fatal("applyPrefectConnection() did not record the read, want it recorded: the URL is well-formed, it just names no host") + } + + if iops.discoveredPostgres.Address != "" { + t.Errorf("discovered address = %q, want none: the URL named no host, and the operator's PGHOST is not the deployment's database", + iops.discoveredPostgres.Address) + } + if iops.discoveredPostgres.ClientPort != 0 { + t.Errorf("discovered client port = %d, want none: the URL named no port", iops.discoveredPostgres.ClientPort) + } + // The credentials it did state are still applied. + if iops.config.PostgresDatabase != "prefect" { + t.Errorf("database = %q, want prefect: only the endpoint is withheld", iops.config.PostgresDatabase) + } + }) + + t.Run("a URL that states a host discovers it", func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectConnection("postgresql+asyncpg://prefect:pw@pg.example.com:6432/prefect") + + if iops.discoveredPostgres.Address != "pg.example.com" { + t.Errorf("discovered address = %q, want pg.example.com", iops.discoveredPostgres.Address) + } + if iops.discoveredPostgres.ClientPort != 6432 { + t.Errorf("discovered client port = %d, want 6432", iops.discoveredPostgres.ClientPort) + } + }) + + t.Run("a host stated as a query parameter is read", func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectConnection("postgres://prefect:pw@/prefect?host=/var/run/postgresql&port=5433") + + if iops.discoveredPostgres.Address != "/var/run/postgresql" { + t.Errorf("discovered address = %q, want the socket directory the URL named", iops.discoveredPostgres.Address) + } + if iops.discoveredPostgres.ClientPort != 5433 { + t.Errorf("discovered client port = %d, want 5433", iops.discoveredPostgres.ClientPort) + } + }) + + t.Run("the keyword form is read too", func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectConnection("host=pg.example.com port=6432 user=prefect password=pw dbname=prefect") + + if iops.discoveredPostgres.Address != "pg.example.com" { + t.Errorf("discovered address = %q, want pg.example.com", iops.discoveredPostgres.Address) + } + if iops.discoveredPostgres.ClientPort != 6432 { + t.Errorf("discovered client port = %d, want 6432", iops.discoveredPostgres.ClientPort) + } + }) + + t.Run("a port outside the usable range is not stated", func(t *testing.T) { + iops := newTestOps() + iops.discoveredPostgres.ClientPort = 5432 + iops.applyPrefectEndpoint("postgres://prefect:pw@pg.example.com/prefect?port=0") + + if iops.discoveredPostgres.ClientPort != 5432 { + t.Errorf("discovered client port = %d, want the previous value left alone", iops.discoveredPostgres.ClientPort) + } + }) +} + +// TestLoadCredentialsFromEnvironmentRecordsOnlyAReadItMade is the other half of +// finding 18. discoveredPostgres.read suppresses the deployment-side discovery +// read, and it was set from the operator's environment whether or not the URL +// there could be parsed — so one typo in a variable permanently suppressed the +// read that would have supplied the endpoint, and the run went looking for a +// database it had no address for. +func TestLoadCredentialsFromEnvironmentRecordsOnlyAReadItMade(t *testing.T) { + t.Run("an unparseable URL leaves the deployment read outstanding", func(t *testing.T) { + t.Setenv("PREFECT_SERVER_DATABASE_CONNECTION_URL", "postgres://prefect:pw@[::1:5432/prefect") + iops := newTestOps() + + iops.loadCredentialsFromEnvironment() + + if iops.discoveredPostgres.read { + t.Error("discovery was recorded as done from a URL that could not be read; the deployment's own connection string is never consulted after this") + } + }) + + t.Run("a URL that was read records the read", func(t *testing.T) { + t.Setenv("PREFECT_SERVER_DATABASE_CONNECTION_URL", "postgres://prefect:pw@pg.example.com:6432/prefect") + iops := newTestOps() + + iops.loadCredentialsFromEnvironment() + + if !iops.discoveredPostgres.read { + t.Error("discovery was not recorded as done from a URL that supplied the endpoint") + } + if iops.discoveredPostgres.Address != "pg.example.com" { + t.Errorf("discovered address = %q, want pg.example.com", iops.discoveredPostgres.Address) + } + }) +} + +// prefectEnvBackend answers `env` in the task-manager container with a scripted +// dump, which is what fetchPostgresCredentials reads. +type prefectEnvBackend struct { + bareBackend + env string +} + +func (b *prefectEnvBackend) Exec(service string, command []string, opts *ExecOptions) (string, error) { + if service == "task-manager" && len(command) > 0 && command[0] == "env" { + return b.env, nil + } + + return "", nil +} + +// TestFetchPostgresCredentialsRecordsOnlyAReadItMade is T106: the same defect +// TestLoadCredentialsFromEnvironmentRecordsOnlyAReadItMade fixed at the other +// call site of applyPrefectConnection. The bool exists to say "nothing was +// applied"; discarding it and recording the read anyway suppresses +// ensurePostgresDiscovery, the one read that supplies the endpoint a restore +// writes into. +func TestFetchPostgresCredentialsRecordsOnlyAReadItMade(t *testing.T) { + tests := []struct { + name string + env string + read bool + addr string + }{ + { + name: "a URL that was read records the read", + env: "PREFECT_API_DATABASE_CONNECTION_URL=postgres://prefect:pw@pg.example.com:6432/prefect\n", + read: true, + addr: "pg.example.com", + }, + { + // One typo in the deployment's own variable. The URL applied + // nothing, so the question has not been answered and the external + // path must still ask it. + name: "an unparseable URL leaves the deployment read outstanding", + env: "PREFECT_API_DATABASE_CONNECTION_URL=postgres://prefect:pw@[::1:6432/prefect\n", + }, + { + // A container that exports no connection URL at all is the same + // case: there is nothing to have read. + name: "a container exporting no connection URL records nothing", + env: "PATH=/usr/bin\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iops := newTestOps() + iops.backend = &prefectEnvBackend{env: tt.env} + + if err := iops.fetchPostgresCredentials(); err != nil { + t.Fatalf("fetchPostgresCredentials() error = %v", err) + } + + if iops.discoveredPostgres.read != tt.read { + t.Errorf("discoveredPostgres.read = %t, want %t", iops.discoveredPostgres.read, tt.read) + } + if iops.discoveredPostgres.Address != tt.addr { + t.Errorf("discovered address = %q, want %q", iops.discoveredPostgres.Address, tt.addr) + } + }) + } +} + +// TestStatedPrefectSSLRootCert is the discovery half of T110: `verify-full` was +// set with no trust anchor beside it, while the anchor the deployment's own +// client uses is stated in the connection URL this tool already reads. +func TestStatedPrefectSSLRootCert(t *testing.T) { + tests := []struct { + name string + connStr string + want string + }{ + { + name: "the URL form", + connStr: "postgres://prefect:pw@pg.example.com:6432/prefect?sslmode=verify-full&sslrootcert=/etc/ssl/pg-ca.pem", + want: "/etc/ssl/pg-ca.pem", + }, + { + name: "the driver-qualified scheme Prefect writes", + connStr: "postgresql+asyncpg://prefect:pw@pg.example.com/prefect?sslrootcert=/certs/ca.crt", + want: "/certs/ca.crt", + }, + { + name: "the keyword form", + connStr: "host=pg.example.com port=6432 dbname=prefect sslrootcert='/certs/ca.crt'", + want: "/certs/ca.crt", + }, + { + name: "the keyword form with a space in the quoted path", + connStr: "host=pg.example.com sslrootcert='/certs/my ca.crt' dbname=prefect", + want: "/certs/my ca.crt", + }, + {name: "a URL naming no authority", connStr: "postgres://prefect:pw@pg.example.com/prefect"}, + {name: "nothing at all", connStr: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, _, got := statedPrefectConnection(tt.connStr); got != tt.want { + t.Errorf("statedPrefectConnection(%q) CA = %q, want %q", tt.connStr, got, tt.want) + } + }) + } + + // It reaches the configuration the client env is built from, which is the + // step that was missing: the CA was read into a log string and nowhere else. + t.Run("the discovered authority reaches the configuration", func(t *testing.T) { + iops := newTestOps() + iops.applyPrefectEndpoint("postgres://prefect:pw@pg.example.com:6432/prefect?sslrootcert=/etc/ssl/pg-ca.pem") + + if got := iops.config.ExternalDB.PostgresTLS.CAFile; got != "/etc/ssl/pg-ca.pem" { + t.Errorf("ExternalDB.PostgresTLS.CAFile = %q, want the authority the deployment states", got) + } + }) +} + +// TestStatedKeywordFields pins the keyword/value form to libpq's reading of it. +// It split on whitespace, so a quoted value holding a space was cut at the +// space and a run discovered part of a certificate path or a host. +func TestStatedKeywordFields(t *testing.T) { + tests := []struct { + name string + connStr string + want map[string]string + wantErr string + }{ + { + name: "plain values", + connStr: "host=pg.example.com port=6432 dbname=prefect", + want: map[string]string{"host": "pg.example.com", "port": "6432", "dbname": "prefect"}, + }, + { + name: "a quoted value with spaces", + connStr: "host=pg.example.com sslrootcert='/etc/ssl/my certs/ca.crt' port=6432", + want: map[string]string{"host": "pg.example.com", "sslrootcert": "/etc/ssl/my certs/ca.crt", "port": "6432"}, + }, + { + name: "whitespace around the equals sign", + connStr: "host = pg.example.com port= 6432 dbname =prefect", + want: map[string]string{"host": "pg.example.com", "port": "6432", "dbname": "prefect"}, + }, + { + name: "escaped quote and backslash inside quotes", + connStr: `password='it\'s a \\ secret' host=db`, + want: map[string]string{"password": `it's a \ secret`, "host": "db"}, + }, + { + name: "escaped space outside quotes", + connStr: `sslrootcert=/certs/my\ ca.crt`, + want: map[string]string{"sslrootcert": "/certs/my ca.crt"}, + }, + { + name: "an empty quoted value", + connStr: "host='' port=5432", + want: map[string]string{"host": "", "port": "5432"}, + }, + { + name: "keywords are lower-cased", + connStr: "HOST=pg.example.com", + want: map[string]string{"host": "pg.example.com"}, + }, + {name: "nothing at all", connStr: " ", want: map[string]string{}}, + { + name: "an unterminated quote", + connStr: "host=pg.example.com sslrootcert='/etc/ssl/my certs/ca.crt", + wantErr: "unterminated quoted value", + }, + { + name: "a keyword with no equals sign", + connStr: "host=pg.example.com prefect", + wantErr: "missing", + }, + { + name: "a value with no keyword", + connStr: "=pg.example.com", + wantErr: "no keyword", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := statedKeywordFields(tt.connStr) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("statedKeywordFields(%q) = %v, %v; want an error containing %q", tt.connStr, got, err, tt.wantErr) + } + + return + } + if err != nil { + t.Fatalf("statedKeywordFields(%q) error = %v, want nil", tt.connStr, err) + } + if !maps.Equal(got, tt.want) { + t.Errorf("statedKeywordFields(%q) = %v, want %v", tt.connStr, got, tt.want) + } + }) + } + + // A string libpq would refuse states nothing: a truncated path is worse + // than none, because the run would then verify against the wrong file. + if host, port, caFile := statedPrefectConnection("host=pg.example.com port=6432 sslrootcert='/etc/ssl/ca.crt"); host != "" || port != 0 || caFile != "" { + t.Errorf("statedPrefectConnection() = %q, %d, %q; want nothing stated for an unterminated quote", host, port, caFile) + } +} diff --git a/src/internal/app/backup.go b/src/internal/app/backup.go index e1eafd5a..73311c2c 100644 --- a/src/internal/app/backup.go +++ b/src/internal/app/backup.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ecdh" "encoding/json" + "errors" "fmt" "io" "os" @@ -73,6 +74,156 @@ func (iops *InfrahubOps) applyCreateRetention(s3UploadedThisRun bool) error { return err } +// --------------------------------------------------------------------------- +// An incomplete capture is not retained (FR-012) +// --------------------------------------------------------------------------- + +// recordIncompleteCapture remembers that one database's capture was not a +// complete one, with the account the operator will read. +// +// The capture path fails the run at the same moment. This record exists because +// failing is only half of FR-012: the other half is that no artefact survives +// the run, and that has to be decided from the verdict itself rather than from +// an error value, which any layer between the capture and the archive could +// wrap, replace or — on a path that only logs — drop. +func (iops *InfrahubOps) recordIncompleteCapture(service, detail string) { + if iops.incompleteCaptures == nil { + iops.incompleteCaptures = map[string]string{} + } + + iops.incompleteCaptures[service] = detail +} + +// capturesComplete reports whether every capture this run made reported a +// complete capture. It is true for a run that captured nothing external, which +// is every internal deployment. +func (iops *InfrahubOps) capturesComplete() bool { + return len(iops.incompleteCaptures) == 0 +} + +// incompleteCaptureDetail renders why the run's captures were not complete, in a +// stable order so two runs against the same fault say the same thing. +func (iops *InfrahubOps) incompleteCaptureDetail() string { + services := make([]string, 0, len(iops.incompleteCaptures)) + for service := range iops.incompleteCaptures { + services = append(services, service) + } + slices.Sort(services) + + details := make([]string, 0, len(services)) + for _, service := range services { + details = append(details, service+": "+iops.incompleteCaptures[service]) + } + + return strings.Join(details, "; ") +} + +// artifactReach is where this run's archive actually landed. It is the input to +// FR-012's removal, which has to cover every location the artefact reached +// rather than only the last one written to: with --s3-upload the object is in +// the bucket before retention or a restore-point listing would ever consider it, +// and one left there is one a later run can select. +type artifactReach struct { + // localNames are the names the archive has had in the backup directory, in + // the order it took them. There is more than one because encrypting renames + // it, and removing the plaintext afterwards is a warning rather than a + // failure — so a run can legitimately leave both behind, and both are + // archives retention would count. + // + // Names are not cleared when the run removes a file itself: deleting + // something already gone is the outcome the deletion asked for, and clearing + // them would make the removal depend on which flags the run was given rather + // than on where the artefact went. + localNames []string + + // s3Name is the name that reached the bucket, empty until the upload + // returned. Configuration alone never puts an object in a bucket, so it + // never authorises a deletion from one — the same rule + // retentionLegsForCreate follows for pruning. + s3Name string +} + +// reachedLocally records a name the archive now has in the backup directory. +func (r *artifactReach) reachedLocally(name string) { + if name == "" || slices.Contains(r.localNames, name) { + return + } + + r.localNames = append(r.localNames, name) +} + +// removeArtifactFrom deletes one named archive from one location, through the +// same storageLocation deletion retention uses: the name is re-checked against +// the archive pattern before anything is removed, and an archive already gone is +// not a failure. +func removeArtifactFrom(ctx context.Context, leg storageLocation, name string) error { + ref, ok := parseBackupName(name) + if !ok { + // Delete would refuse this name anyway. Saying so here is the difference + // between an operator knowing an artefact may still be out there and + // believing the run cleaned up after itself. + return fmt.Errorf("cannot remove %q from %s: it is not a backup archive name", name, leg.Name()) + } + + if err := leg.Delete(ctx, ref); err != nil { + return fmt.Errorf("failed to remove the incomplete backup from %s: %w", leg.Name(), err) + } + + logrus.Infof("Removed the incomplete backup %s from %s", name, leg.Name()) + + return nil +} + +// discardIncompleteCapture is the rest of FR-012: a capture that did not capture +// everything it was asked for must not be retained, so the run fails and the +// artefact is removed from every location it reached. +// +// Removal rather than a metadata flag is what keeps retention out of this. +// Retention selects from artefact names and recency ranks and never reads +// metadata, so an artefact merely *marked* incomplete would still occupy a keep +// slot and evict a good backup — and a previously released tool, which ignores +// fields it has never heard of, would have offered it as a restore point. An +// artefact that does not exist changes nothing either of them decides. +// +// Removal is not a prune, so unlike retentionLegsForCreate this consults no +// retention policy: a deployment that never configured retention is not thereby +// asking to keep an unusable backup. +// +// It returns the failure the run reports. Every location is attempted before it +// returns, and one that could not be cleared is named rather than swallowed — +// the artefact is then still out there, and BackupMetadata.CaptureComplete is +// what remains to identify it. +func (iops *InfrahubOps) discardIncompleteCapture(reach artifactReach) error { + failure := fmt.Errorf("the backup is incomplete and will not be kept (%s)", iops.incompleteCaptureDetail()) + + ctx := context.Background() + var problems []error + + if len(reach.localNames) > 0 { + local := newLocalLocation(iops.config.BackupDir) + for _, name := range reach.localNames { + if err := removeArtifactFrom(ctx, local, name); err != nil { + problems = append(problems, err) + } + } + } + + if reach.s3Name != "" { + client, err := NewS3Client(iops.config.S3) + if err != nil { + problems = append(problems, fmt.Errorf("failed to create S3 client to remove the incomplete backup: %w", err)) + } else if err := removeArtifactFrom(ctx, newS3Location(client), reach.s3Name); err != nil { + problems = append(problems, err) + } + } + + if len(problems) == 0 { + return failure + } + + return fmt.Errorf("%w; it may still be present: %w", failure, errors.Join(problems...)) +} + // loadEncryptionKey loads the public key for encryption. // If keyPath is empty, returns the default hardcoded key. func loadEncryptionKey(keyPath string) (*ecdh.PublicKey, error) { @@ -100,13 +251,28 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT return err } - // Detect Neo4j edition - editionInfo, err := iops.detectBackupNeo4jEdition() - if err != nil { + // The run's single exit path for every transient workload it creates + // (FR-011). Deferred before the gate below rather than after it, because the + // gate is what creates the first one — a probe that answered and then hit a + // version refusal has already put a pod in the namespace. + defer iops.releaseTransientWorkloads() + + // Where each database lives, before the edition probe reads a missing + // container as Community and before anything is stopped for it. + if err := iops.prepareDatabaseCapture(!excludeTaskManager); err != nil { + return err + } + + // Detect Neo4j edition. A probe that did not answer stops the run here, + // before the abort window and before anything is stopped: the edition + // decides whether Neo4j has to be taken offline, and a failed probe is not + // evidence that it does. + editionInfo := iops.detectNeo4jEditionInfo("backup") + if err := editionInfo.requireDeterminedEdition(); err != nil { return err } - if editionInfo.IsCommunity { - if err := iops.recordReplicaCounts(appServices...); err != nil { + if editionInfo.RequiresOfflineCapture() { + if err := iops.recordReplicaCounts(appServicesStoppedForBackup...); err != nil { return err } logrus.Warn("Neo4j Community Edition detected; Infrahub services will be stopped and restarted before the backup begins.") @@ -135,14 +301,13 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT } var servicesToRestart []string - if editionInfo.IsCommunity { + if editionInfo.RequiresOfflineCapture() { stoppedServices, stopErr := iops.stopAppContainers() if stopErr != nil { - if len(stoppedServices) > 0 { - if startErr := iops.startAppContainers(stoppedServices); startErr != nil { - logrus.Warnf("Failed to restart services after stop error: %v", startErr) - } - } + // Whatever was taken down before the failure goes back up, and + // what could not be is named (FR-013). + iops.returnAppContainersToScale(stoppedServices) + return fmt.Errorf("failed to stop services for Neo4j Community backup: %w", stopErr) } servicesToRestart = append([]string(nil), stoppedServices...) @@ -157,10 +322,50 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT } } }() + + // Asked for is not stopped, and the capture below reads the store + // offline. This is the same confirmation the restore paths make before + // their own destructive step, and it is needed here for the reason + // stopAppContainers states: `kubectl scale` returns as soon as the + // replica count is recorded, so an infrahub-server still holding a Bolt + // session is a writer attached to the store `neo4j-admin dump` is about + // to read — a torn dump reported as a successful backup. + // + // After the restart defer, so a deployment that will not quiesce is + // returned to the scale it was found at rather than left down. + if err := iops.confirmAppContainersQuiesced(servicesToRestart); err != nil { + return err + } } backupFilename := iops.generateBackupFilename() backupPath := filepath.Join(iops.config.BackupDir, backupFilename) + + // Where the archive has got to, updated as it gets there, so FR-012's + // removal covers every location rather than the one the run last touched. + var reach artifactReach + + // FR-012's backstop, registered before the captures run so that no path out + // of them can skip it: if a capture that was not complete somehow reached an + // archive, the archive does not survive the run. It is written as an + // invariant rather than as a branch of the capture's own error handling, + // because the verdict — not an error value some layer between could wrap, + // replace or drop — is what decides whether an artefact may be kept. + // + // On the ordinary path the gate after the captures has already handled it + // and cleared reach, so this does nothing. + defer func() { + if iops.capturesComplete() || (len(reach.localNames) == 0 && reach.s3Name == "") { + return + } + + if retErr != nil { + logrus.Errorf("The backup failed: %v", retErr) + } + + retErr = iops.discardIncompleteCapture(reach) + }() + workDir, err := os.MkdirTemp("", "infrahub_backup_*") if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) @@ -206,6 +411,22 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT logrus.Info("Skipping task manager database backup as requested") } + // FR-012's gate. This is the last moment the completeness verdict can + // change — only the capture paths above record one — and nothing downstream + // may advance an artefact this run must not keep. + // + // It matters that the check is here rather than only in the deferred + // backstop. Retention ranks by recency, so an archive written and then + // removed would still have been the location's newest while the prune ran, + // and would have pushed the real newest down a rank: exactly the eviction + // FR-012 exists to prevent. + if !iops.capturesComplete() { + discardErr := iops.discardIncompleteCapture(reach) + reach = artifactReach{} + + return discardErr + } + // Calculate checksums for backup files checksums, err := calculateBackupChecksums(backupDir, excludeTaskManager) if err != nil { @@ -213,6 +434,11 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT } metadata.Checksums = checksums + // What the captures reported, read now that they have run rather than when + // the metadata was built above — which was before backupDatabase (FR-012, + // see applyCaptureCompleteness). + iops.applyCaptureCompleteness(metadata) + metadataBytes, err := json.MarshalIndent(metadata, "", " ") if err != nil { return fmt.Errorf("failed to marshal metadata: %w", err) @@ -231,6 +457,8 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT return fmt.Errorf("failed to create archive: %w", err) } + reach.reachedLocally(backupFilename) + // Encrypt backup if requested if encrypt || encryptKey != "" { pubKey, loadErr := loadEncryptionKey(encryptKey) @@ -250,6 +478,7 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT backupPath = encryptedPath backupFilename = filepath.Base(encryptedPath) + reach.reachedLocally(backupFilename) } // Log backup creation with structured fields @@ -269,6 +498,7 @@ func (iops *InfrahubOps) CreateBackup(force bool, neo4jMetadata string, excludeT if err != nil { return fmt.Errorf("backup created locally but S3 upload failed: %w", err) } + reach.s3Name = backupFilename logrus.Infof("Backup uploaded to: %s", s3URI) if !s3KeepLocal { @@ -389,6 +619,12 @@ func (iops *InfrahubOps) RestoreBackup(backupFile string, excludeTaskManager boo return err } + // A database outside the deployment gets the workload the restore runs in + // at the gate below, so the workloads this run created are given back on + // every exit path from here on (FR-011). An all-internal deployment created + // none and the call does nothing. + defer iops.releaseTransientWorkloads() + if err := iops.recordReplicaCounts(restoreServices...); err != nil { return err } @@ -436,15 +672,14 @@ func (iops *InfrahubOps) RestoreBackup(backupFile string, excludeTaskManager boo "components": metadata.Components, }).Info("Backup metadata loaded") - // Detect Neo4j edition for restore - detectedEdition, detectionErr := iops.detectNeo4jEdition() - editionInfo := NewNeo4jEditionInfo(detectedEdition, detectionErr) - - neo4jEdition, err := editionInfo.ResolveRestoreEdition(metadata.Neo4jEdition) - if err != nil { + // Before the checksums, the edition resolution and everything destructive: + // an artefact whose own capture reported itself incomplete is not a restore + // point (FR-012). It is also the refusal `--latest` needs, since selection + // there has only the archive's name to go on and would pick exactly this + // artefact for being the newest. + if err := metadata.refuseIncompleteCapture(backupFile); err != nil { return err } - editionInfo.LogDetection("restore") // Determine task manager database availability taskManagerIncluded := slices.Contains(metadata.Components, "task-manager-db") @@ -453,14 +688,38 @@ func (iops *InfrahubOps) RestoreBackup(backupFile string, excludeTaskManager boo taskManagerIncluded = true } } + shouldRestoreTaskManager := taskManagerIncluded && !excludeTaskManager + + // Where each database this restore writes lives, before the edition is + // misdiagnosed, before transient data is wiped and before anything is + // scaled to zero. The discovery this gate's successor needs reads `env` out + // of infrahub-server and task-manager, so it cannot run any later than + // this: stopAppContainers has scaled both to zero by then. + // + // It runs after the metadata is read, because the archive decides which + // databases are written: an archive without the task-manager database + // writes only Neo4j, and must not be refused, or have a workload created, + // for an external PostgreSQL it never touches. Extracting and reading the + // archive is local work and changes nothing in the deployment. + if err := iops.prepareDatabaseRestore(shouldRestoreTaskManager); err != nil { + return err + } + + // Detect Neo4j edition for restore + detectedEdition, detectionErr := iops.detectNeo4jEdition() + editionInfo := NewNeo4jEditionInfo(detectedEdition, detectionErr) + + neo4jEdition, err := editionInfo.ResolveRestoreEdition(metadata.Neo4jEdition) + if err != nil { + return err + } + editionInfo.LogDetection("restore") // Validate checksums for all backup files if err := validateBackupChecksums(workDir, &metadata, excludeTaskManager); err != nil { return err } - // Determine if we should restore task manager database - shouldRestoreTaskManager := taskManagerIncluded && !excludeTaskManager prefectPath := filepath.Join(workDir, "backup", prefectDumpFilename) prefectExists := fileExists(prefectPath) validatePrefect := shouldRestoreTaskManager && prefectExists @@ -479,11 +738,40 @@ func (iops *InfrahubOps) RestoreBackup(backupFile string, excludeTaskManager boo logrus.Info("Task manager database dump detected; will restore") } + // The last check that can be made with the deployment still up and the + // databases still holding their data: for a database outside the + // deployment, the server fetches the artifact itself, so the artifact is + // staged where it can be read from, and a run that cannot establish that + // stops here (FR-007). It does nothing for an in-deployment database. + if err := iops.preflightExternalNeo4jRestore(workDir); err != nil { + return err + } + // Wipe transient data iops.wipeTransientData() // Stop application containers - if _, err := iops.stopAppContainers(); err != nil { + stopped, err := iops.stopAppContainers() + if err != nil { + // Whatever was taken down before the failure goes back up: the run is + // over, and leaving a partially quiesced deployment behind is the + // outage FR-013 exists to prevent. + iops.returnAppContainersToScale(stopped) + + return err + } + + // Every exit path from here to the ordinary restart below is a failure, and + // each of them returns the deployment to the scale it was found at (FR-013). + // The success path hands the list over by clearing it, so nothing is started + // twice. + defer func() { + iops.returnAppContainersToScale(stopped) + }() + + // Asked for is not stopped, and the next steps overwrite databases + // (FR-026). + if err := iops.confirmAppContainersQuiesced(stopped); err != nil { return err } @@ -514,12 +802,16 @@ func (iops *InfrahubOps) RestoreBackup(backupFile string, excludeTaskManager boo } } - // Restart all services - logrus.Info("Restarting Infrahub services...") - if err := iops.StartServices("infrahub-server", "task-worker"); err != nil { - return fmt.Errorf("failed to restart infrahub services: %w", err) + if err := iops.startAppServicesAfterRestore(); err != nil { + return err } + // The deployment is back, so the deferred failure-path restart has nothing + // left to do. What it would still be useful for is saying whether the + // services actually came back up, which is what this reports (FR-026). + iops.reportAppContainersRunning(stopped) + stopped = nil + logrus.Info("Restore completed successfully") logrus.Info("Infrahub should be available shortly") diff --git a/src/internal/app/backup_containers.go b/src/internal/app/backup_containers.go index 95470219..bb878912 100644 --- a/src/internal/app/backup_containers.go +++ b/src/internal/app/backup_containers.go @@ -176,16 +176,30 @@ func (iops *InfrahubOps) waitForRunningTasks() error { } } -// appServices are the application services that backup and restore stop, and -// later start again, around their database steps. -var appServices = []string{ +// appServicesStoppedForBackup are the services a run stops for an offline +// capture, and the only services IsServiceRunning is ever asked about. +// +// It is named rather than inlined because that second fact is load-bearing: +// the running check and the internal-versus-external location decision read a +// namespace by the same rule but differ on what an unanchored name means (see +// deploymentClaimsResource), and this list is why those two can never be asked +// about the same service. No database appears here, and locateService is asked +// about nothing else. +// +// That says nothing about the workload resolver, which is a different caller of +// the same rule and *is* asked about a database: a restore starts +// `task-manager-db`. So the resolver does not take its policy from this list — +// it takes it from the service (see unanchoredNamePolicyFor), because what +// holds of the services stopped here does not hold of everything that gets +// scaled. +var appServicesStoppedForBackup = []string{ "infrahub-server", "task-worker", "task-manager", "task-manager-background-svc", "cache", "message-queue", } // restoreServices are the services a restore starts: the application services // and task-manager-db, which restorePostgreSQL starts before it loads the dump. -var restoreServices = slices.Concat(appServices, []string{"task-manager-db"}) +var restoreServices = slices.Concat(appServicesStoppedForBackup, []string{"task-manager-db"}) // replicaRecorder is implemented by backends that stop a service by scaling // its workload to 0 replicas, and so must know the workload's replica count to @@ -219,11 +233,23 @@ func (iops *InfrahubOps) stopAppContainers() ([]string, error) { stopped := []string{} - for _, service := range appServices { + for _, service := range appServicesStoppedForBackup { + // A status this run could not determine is not a reason to stop + // anything, and it is not a reason to carry on either. Stopping on a + // guess is what scaled another product's `cache` or `message-queue` to + // zero, and constitution Principle II allows a backup no destructive + // effect beyond the documented stop/start of Infrahub's own containers + // — so this does not stop the workload. But every caller of this + // function is about to do something that requires the services to be + // down: an offline capture, or a restore. Warning and continuing turned + // "we do not know whether infrahub-server is running" into a cold + // `neo4j-admin dump` taken with writers still attached, reported as a + // successful backup. So the undetermined status fails the run instead, + // leaving the deployment as it was found and naming the service the + // query could not answer for. running, err := iops.IsServiceRunning(service) if err != nil { - logrus.Debugf("Could not determine status of %s: %v", service, err) - continue + return stopped, fmt.Errorf("cannot determine whether %s is running, so the deployment cannot be confirmed quiesced and this run will not read or write data as if it were: %w", service, err) } if running { @@ -244,6 +270,161 @@ func (iops *InfrahubOps) stopAppContainers() ([]string, error) { return stopped, nil } +const ( + // appQuiesceTimeout bounds the wait for the services a run stopped to + // actually be down. It is generous because a scale-down is asynchronous: on + // Kubernetes the replica count changes at once and the pods terminate + // afterwards, at whatever pace their own grace periods and finalizers + // allow. + appQuiesceTimeout = 5 * time.Minute + + // appScaleTimeout bounds the wait for them to be back up, which is a pull + // and a start rather than a stop. + appScaleTimeout = 10 * time.Minute + + // appScalePollInterval is how often the deployment is asked. It decides how + // many status queries a wait spends, not how long it waits. + appScalePollInterval = 3 * time.Second +) + +// confirmAppContainersQuiesced waits until every service this run stopped +// reports that it is down, and fails otherwise (FR-026). +// +// This is the difference between asking for a scale-down and having one. The +// two are not the same on either backend and are furthest apart on Kubernetes, +// where `kubectl scale` returns as soon as the replica count is recorded: the +// pods keep running while they terminate, and an Infrahub server still holding +// a Bolt session is a writer attached to a database the next step is about to +// overwrite. What that produces is not a failed restore, it is a restore that +// reports success over a database some other process was writing to as it +// landed — and constitution Principle II is written against exactly that class +// of quiet corruption. +// +// It takes the list the run actually stopped rather than the whole service set. +// A service that reported itself not running before the stop has already +// answered this question, and waiting on one that was never up would fail +// restores on deployments that legitimately run without a task manager. +// +// A status the deployment cannot answer for fails the run, which is the same +// answer stopAppContainers gives to the same uncertainty and for the same +// reason: a destructive step must not proceed on a guess about whether the +// writers are gone. +func (iops *InfrahubOps) confirmAppContainersQuiesced(stopped []string) error { + return iops.confirmAppContainersQuiescedWithin(stopped, appQuiesceTimeout, appScalePollInterval) +} + +// confirmAppContainersQuiescedWithin is the confirmation under an explicit +// window, which is what makes a deployment that will not quiesce assertable +// without waiting out the real one. +func (iops *InfrahubOps) confirmAppContainersQuiescedWithin(stopped []string, window, interval time.Duration) error { + if len(stopped) == 0 { + return nil + } + + logrus.Info("Confirming Infrahub application services have stopped...") + + deadline := time.Now().Add(window) + for _, service := range stopped { + for { + running, err := iops.IsServiceRunning(service) + if err != nil { + return fmt.Errorf("cannot confirm that %s has stopped, so the deployment cannot be confirmed quiesced and this run will not write data as if it were: %w", service, err) + } + if !running { + break + } + + if sleepUntilNextPoll(deadline, interval) { + return fmt.Errorf( + "%s is still running %v after it was stopped, so the deployment is not quiesced and this run will not write to a database something else may still be writing to. "+ + "Check for a pod that will not terminate, or a controller scaling it back up", + service, window) + } + } + } + + logrus.Info("Application services confirmed stopped") + + return nil +} + +// returnAppContainersToScale gives back everything a run took down (FR-013). +// +// It is the failure path's counterpart of the ordinary restart at the end of a +// restore, and it exists because that restart is only reached by a run that +// succeeded. A restore that failed after quiescing used to leave the deployment +// scaled to zero with the operator's data still in place and nothing serving +// it — the outage lasting until somebody noticed, rather than until the run +// ended. +// +// It reports what it could not do rather than returning it. The caller is on +// its way out with a failure of its own, and replacing that failure with "and +// also the restart did not work" would hide the reason the restore failed. +func (iops *InfrahubOps) returnAppContainersToScale(stopped []string) { + if len(stopped) == 0 { + return + } + + logrus.Infof("Returning %d Infrahub service(s) this run stopped to their prior scale...", len(stopped)) + + if err := iops.startAppContainers(stopped); err != nil { + logrus.Errorf("Could not return every Infrahub service this run stopped to its prior scale; %v was quiesced for the restore and needs starting by hand: %v", stopped, err) + + return + } + + iops.reportAppContainersRunning(stopped) +} + +// reportAppContainersRunning waits on the default window; see +// reportAppContainersRunningWithin. +func (iops *InfrahubOps) reportAppContainersRunning(services []string) { + iops.reportAppContainersRunningWithin(services, appScaleTimeout, appScalePollInterval) +} + +// reportAppContainersRunning waits for the services a run put back to report +// themselves up, and says so (FR-026's second half). +// +// It reports rather than fails, and the asymmetry with +// confirmAppContainersQuiesced is deliberate. That check guards a destructive +// step, so uncertainty there has to stop the run. This one runs after the data +// is already restored, where turning "the cluster is still pulling an image" +// into a non-zero exit would report a successful restore as a failed one — and +// a scheduled restore's exit status is what somebody gets paged by. What the +// operator needs here is to be told which service has not come back, which is +// what this says. +// +// Whether Infrahub then *serves* the restored data is not something this layer +// can observe: the tool speaks to the deployment and the databases, never to +// Infrahub's own API. That assertion belongs to the end-to-end suite. +func (iops *InfrahubOps) reportAppContainersRunningWithin(services []string, window, interval time.Duration) { + deadline := time.Now().Add(window) + pending := []string{} + + for _, service := range services { + for { + running, err := iops.IsServiceRunning(service) + if err == nil && running { + break + } + + if sleepUntilNextPoll(deadline, interval) { + pending = append(pending, service) + + break + } + } + } + + if len(pending) > 0 { + logrus.Warnf("The data is restored, but %v has not reported itself running again within %v; Infrahub will not serve until it does, so check those workloads in the deployment", pending, window) + + return + } + + logrus.Info("Every Infrahub service this run stopped is running again") +} + func (iops *InfrahubOps) startAppContainers(services []string) error { if len(services) == 0 { return nil @@ -276,12 +457,26 @@ func (iops *InfrahubOps) startAppContainers(services []string) error { ordered = append(ordered, svc) } + // Every service is attempted, and the failures are reported together + // (FR-013). + // + // It used to return on the first one, and the order above is what made that + // costly: `cache` is started first, so a single transient failure there left + // the other five at zero replicas — a whole deployment down on one service's + // error, reported as one line from returnAppContainersToScale. The services + // do not depend on each other for *starting*; preferredOrder is about the + // order they come up in, so one that will not start is not a reason to leave + // the rest stopped. + startErrs := []error{} for _, svc := range ordered { logrus.Infof("Starting %s...", svc) if err := iops.StartServices(svc); err != nil { - return fmt.Errorf("failed to start %s: %w", svc, err) + startErrs = append(startErrs, fmt.Errorf("failed to start %s: %w", svc, err)) } } + if len(startErrs) > 0 { + return errors.Join(startErrs...) + } logrus.Info("Application services started") return nil diff --git a/src/internal/app/backup_containers_test.go b/src/internal/app/backup_containers_test.go index 7f09a706..4224e2ac 100644 --- a/src/internal/app/backup_containers_test.go +++ b/src/internal/app/backup_containers_test.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "reflect" + "slices" "strings" "testing" "time" @@ -433,3 +434,294 @@ func TestParsePrefectMaxLimit(t *testing.T) { }) } } + +// quiescingBackend is a deployment whose services can be made to report +// themselves running or stopped, and which records every stop and start. +// +// It is what lets FR-026's "asked for is not stopped" distinction be driven +// without a cluster: a scale-down that is recorded and then not honoured is +// exactly what `kubectl scale` returning before its pods terminate looks like. +type quiescingBackend struct { + bareBackend + + // running answers the status query per service. A service absent from the + // map is running, which is the state every restore starts from. + running map[string]bool + // statusErr fails the status query, standing in for a deployment that + // cannot say whether a service is up. + statusErr error + // ignoreStop leaves a service reporting itself running after it has been + // stopped, which is a pod that will not terminate. + ignoreStop map[string]bool + // startErr fails the scale-up of one named service while every other one + // still succeeds. That is the shape a transient failure takes — one + // workload's controller, one admission webhook, one quota — and it is what + // distinguishes "this service would not start" from "the deployment is + // unreachable". + startErr map[string]error + + stopped []string + started []string +} + +func newQuiescingBackend() *quiescingBackend { + return &quiescingBackend{ + bareBackend: bareBackend{name: "kubernetes"}, + running: map[string]bool{}, + ignoreStop: map[string]bool{}, + startErr: map[string]error{}, + } +} + +func (b *quiescingBackend) IsRunning(service string) (bool, error) { + if b.statusErr != nil { + return false, b.statusErr + } + if state, ok := b.running[service]; ok { + return state, nil + } + + return true, nil +} + +func (b *quiescingBackend) Stop(services ...string) error { + b.stopped = append(b.stopped, services...) + for _, service := range services { + if !b.ignoreStop[service] { + b.running[service] = false + } + } + + return nil +} + +func (b *quiescingBackend) Start(services ...string) error { + // Recorded before the failure, because the attempt is what is being + // asserted: a run that gave up after this service must be distinguishable + // from one that carried on past it. + b.started = append(b.started, services...) + + errs := []error{} + for _, service := range services { + if err, ok := b.startErr[service]; ok { + errs = append(errs, err) + + continue + } + b.running[service] = true + } + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +func newQuiescingOps(backend *quiescingBackend) *InfrahubOps { + return &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} +} + +// TestConfirmAppContainersQuiesced is FR-026's first half. A scale-down that +// was requested and not honoured must not be read as a quiesced deployment: the +// next step overwrites a database, and a writer still attached to it produces a +// restore that reports success over data something else was writing. +func TestConfirmAppContainersQuiesced(t *testing.T) { + t.Run("services that actually stopped are confirmed", func(t *testing.T) { + backend := newQuiescingBackend() + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + if len(stopped) != len(appServicesStoppedForBackup) { + t.Fatalf("stopped = %v, want every running service", stopped) + } + if err := iops.confirmAppContainersQuiesced(stopped); err != nil { + t.Errorf("confirmAppContainersQuiesced() = %v, want nil", err) + } + }) + + t.Run("a service that will not stop fails the run", func(t *testing.T) { + backend := newQuiescingBackend() + backend.ignoreStop["infrahub-server"] = true + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil: the scale-down itself succeeds, which is the whole problem", err) + } + + if err := iops.confirmAppContainersQuiescedWithin(stopped, 50*time.Millisecond, 5*time.Millisecond); err == nil { + t.Fatal("confirmAppContainersQuiesced() = nil, want the run stopped before it writes to a database a writer may still hold") + } else if !strings.Contains(err.Error(), "infrahub-server") { + t.Errorf("err = %v, want it to name the service that is still running", err) + } + }) + + t.Run("a status the deployment cannot answer for fails the run", func(t *testing.T) { + backend := newQuiescingBackend() + backend.statusErr = errors.New(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods"`) + iops := newQuiescingOps(backend) + + err := iops.confirmAppContainersQuiescedWithin([]string{"cache"}, 50*time.Millisecond, 5*time.Millisecond) + + if err == nil { + t.Fatal("confirmAppContainersQuiesced() = nil, want the unanswered status to stop the run rather than be read as stopped") + } + if !strings.Contains(err.Error(), "cannot confirm") { + t.Errorf("err = %v, want it to say the deployment could not be confirmed quiesced", err) + } + }) + + t.Run("a run that stopped nothing has nothing to confirm", func(t *testing.T) { + backend := newQuiescingBackend() + if err := newQuiescingOps(backend).confirmAppContainersQuiesced(nil); err != nil { + t.Errorf("confirmAppContainersQuiesced(nil) = %v, want nil", err) + } + if len(backend.started) > 0 || len(backend.stopped) > 0 { + t.Errorf("the deployment was touched (%v/%v), want nothing asked of it", backend.stopped, backend.started) + } + }) +} + +// TestReturnAppContainersToScale is FR-013 on the failure path: what a restore +// took down goes back up whether or not the restore worked. Before this, a +// restore that failed after quiescing left the deployment scaled to zero with +// the operator's data still in place and nothing serving it. +func TestReturnAppContainersToScale(t *testing.T) { + t.Run("everything the run stopped is started again", func(t *testing.T) { + backend := newQuiescingBackend() + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + + iops.returnAppContainersToScale(stopped) + + for _, service := range stopped { + if !slices.Contains(backend.started, service) { + t.Errorf("%s was stopped and not started again, want the deployment returned to its prior scale", service) + } + } + }) + + // Prior scale, not "all six": a service that was already down before the + // restore is not something this run took away, and starting it would be + // this tool deciding a deployment's shape for it. + t.Run("a service that was already down is left down", func(t *testing.T) { + backend := newQuiescingBackend() + backend.running["task-manager-background-svc"] = false + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + if slices.Contains(stopped, "task-manager-background-svc") { + t.Fatalf("stopped = %v, want the service that was already down left out of it", stopped) + } + + iops.returnAppContainersToScale(stopped) + + if slices.Contains(backend.started, "task-manager-background-svc") { + t.Error("a service that was down before the restore was started by it, want its prior scale kept") + } + }) + + t.Run("a run that stopped nothing starts nothing", func(t *testing.T) { + backend := newQuiescingBackend() + newQuiescingOps(backend).returnAppContainersToScale(nil) + + if len(backend.started) > 0 { + t.Errorf("services started = %v, want none", backend.started) + } + }) +} + +// TestStartAppContainersAttemptsEveryService is FR-013's "every exit path" read +// as what it costs when only some of it holds. +// +// The restart stopped at the first failure, and preferredOrder starts with +// `cache` — so one transient failure on the first service left the other five +// at zero replicas, and returnAppContainersToScale reported it as a single +// logged line naming one of the six. The deployment was down and the message +// said so about one workload. +// +// The services do not depend on each other for *starting*: preferredOrder is +// about the order they come up in, so a service that will not start is not a +// reason to leave the rest stopped. +func TestStartAppContainersAttemptsEveryService(t *testing.T) { + t.Run("a failure on the first service does not abandon the rest", func(t *testing.T) { + backend := newQuiescingBackend() + // `cache` is first in preferredOrder, which is what made this the + // whole-deployment case rather than an edge one. + backend.startErr["cache"] = errors.New("Operation cannot be fulfilled on statefulsets.apps \"infrahub-cache\": the object has been modified") + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + + startErr := iops.startAppContainers(stopped) + if startErr == nil { + t.Fatal("startAppContainers() = nil, want the failure reported") + } + + for _, service := range stopped { + if !slices.Contains(backend.started, service) { + t.Errorf("%s was never attempted, want every service this run stopped attempted whatever the others did", service) + } + } + + // Five of the six actually came back up; only the one that failed did + // not. That is the state the operator is left in, and it is the state + // the message has to describe. + for _, service := range stopped { + if service == "cache" { + continue + } + if running, _ := backend.IsRunning(service); !running { + t.Errorf("%s is not running, want it returned to its prior scale despite cache failing", service) + } + } + }) + + t.Run("every failure is named, not just the first", func(t *testing.T) { + backend := newQuiescingBackend() + backend.startErr["cache"] = errors.New("cache would not scale") + backend.startErr["task-worker"] = errors.New("task-worker would not scale") + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + + startErr := iops.startAppContainers(stopped) + if startErr == nil { + t.Fatal("startAppContainers() = nil, want both failures reported") + } + for _, want := range []string{"cache", "task-worker"} { + if !strings.Contains(startErr.Error(), want) { + t.Errorf("err = %v, want it to name %s: what is still at zero is the whole of what the operator needs", startErr, want) + } + } + }) + + t.Run("a deployment that starts cleanly reports no error", func(t *testing.T) { + backend := newQuiescingBackend() + iops := newQuiescingOps(backend) + + stopped, err := iops.stopAppContainers() + if err != nil { + t.Fatalf("stopAppContainers() = %v, want nil", err) + } + if err := iops.startAppContainers(stopped); err != nil { + t.Errorf("startAppContainers() = %v, want nil", err) + } + }) +} diff --git a/src/internal/app/backup_deployment.go b/src/internal/app/backup_deployment.go index 10c1d79f..0eed4952 100644 --- a/src/internal/app/backup_deployment.go +++ b/src/internal/app/backup_deployment.go @@ -26,14 +26,43 @@ func (iops *InfrahubOps) resetDeploymentID() error { newUUID := uuid.NewString() updatedAt := time.Now().UTC().Format(time.RFC3339) - args := []string{ + statement := []string{ + "--param", fmt.Sprintf("new_uuid => '%s'", newUUID), + "--param", fmt.Sprintf("updated_at => '%s'", updatedAt), + "MATCH (n:Root) SET n.uuid = $new_uuid, n.updated_at = $updated_at RETURN n", + } + + args := append([]string{ "cypher-shell", "-u", iops.config.Neo4jUsername, "-p" + iops.config.Neo4jPassword, "-d", iops.config.Neo4jDatabase, - "--param", fmt.Sprintf("new_uuid => '%s'", newUUID), - "--param", fmt.Sprintf("updated_at => '%s'", updatedAt), - "MATCH (n:Root) SET n.uuid = $new_uuid, n.updated_at = $updated_at RETURN n", + }, statement...) + run := func() (string, error) { + return iops.Exec("database", args, nil) + } + + // A database outside the deployment has no `database` container to exec + // into, so the statement goes through the restore workload the gate + // prepared, over Bolt, with the credentials its secret supplies — the same + // channel the seed and its online confirmation used. + if restore := iops.externalRestores[serviceNeo4j]; restore != nil { + uri, err := restore.boltURI() + if err != nil { + return err + } + + external := append([]string{ + "cypher-shell", "--non-interactive", "--format", "plain", + "-a", uri, + "-d", restore.Endpoint.Database, + }, statement...) + run = func() (string, error) { + return iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "resetting the deployment ID", restore.Endpoint.endpointTarget(), + serviceNeo4j, external, restore.execOptions(), + ) + } } logrus.Info("Resetting deployment ID on Root node...") @@ -41,7 +70,7 @@ func (iops *InfrahubOps) resetDeploymentID() error { var lastErr error var lastOutput string for attempt := 1; attempt <= resetDeploymentIDMaxAttempts; attempt++ { - output, err := iops.Exec("database", args, nil) + output, err := run() if err == nil { logrus.WithField("new_uuid", newUUID).Info("Deployment ID reset successfully") return nil diff --git a/src/internal/app/backup_metadata.go b/src/internal/app/backup_metadata.go index 109c6441..d11d92f5 100644 --- a/src/internal/app/backup_metadata.go +++ b/src/internal/app/backup_metadata.go @@ -1,6 +1,7 @@ package app import ( + "errors" "fmt" "strings" "time" @@ -15,6 +16,15 @@ const ( neo4jEditionCommunity = "community" ) +// isCommunityEdition reads an edition string the way every gate on the +// Community mechanism does: case-insensitively, with whatever padding a server +// or a metadata file put around it. An empty string is not Community — the +// asymmetry Neo4jEditionInfo documents — so a probe that did not answer never +// selects the destructive branch. +func isCommunityEdition(edition string) bool { + return strings.EqualFold(strings.TrimSpace(edition), neo4jEditionCommunity) +} + // BackupMetadata represents the backup metadata structure type BackupMetadata struct { MetadataVersion int `json:"metadata_version"` @@ -27,9 +37,121 @@ type BackupMetadata struct { Neo4jEdition string `json:"neo4j_edition,omitempty"` Redacted bool `json:"redacted,omitempty"` Encrypted bool `json:"encrypted,omitempty"` + + // The four fields below record where a capture's data came from when a + // database lives outside the deployment (FR-005). They are additive and + // metadataVersion does not move for them (FR-024): a reader that sees none + // of them holds an internal, complete capture from an unrecorded endpoint, + // which is exactly what every previously released version wrote — no + // migration, no warning, no operator action. + // + // They are keyed by the deployment service name — serviceNeo4j and + // serviceTaskManagerDB — rather than by whatever a backend calls its + // components, because that is the name the endpoint was resolved under and + // the Plakar backend renames its components afterwards. A run that read at + // least one external database records every database it captured, including + // the internal ones, so the presence of a key never says where that database + // lives; ExternalLocation says it. + + // SourceEndpoints is what the run supplied for each database, in the order + // the capture would try them. It is not a claim that the first entry served + // the capture, and no field here claims that of any entry: with several + // endpoints supplied, no documented interface reports which one answered. + // + // The port is the one the capture actually travelled over — see + // DatabaseEndpoint.captureTargets, which is where the two services diverge. + SourceEndpoints map[string][]string `json:"source_endpoints,omitempty"` + + // SourceRoles maps each supplied endpoint to the role observed for it at + // capture time. `unknown` is a real value: a member the server named without + // a role, or a role query that failed, records unknown, and a reader knows + // the run looked. A database whose roles were never sought is absent from + // this map instead — collapsing the two would claim an observation the run + // never made. + SourceRoles map[string]map[string]string `json:"source_roles,omitempty"` + + // CaptureComplete is what the capture *reported*, not what its exit status + // implied — a backup that reached only some of several endpoints exits the + // same way as one that failed outright (research R6). + // + // It is a pointer because absent means complete: a pre-feature artefact was + // never partial by this definition. A plain bool with omitempty would encode + // false as absent and so read back as its own opposite, which is the one + // mistake this field cannot survive. + // + // It is written where the metadata is serialized rather than where it is + // built, because the captures run in between: written at build time it was + // a prediction of what they would say, and necessarily `true`. See + // applyCaptureCompleteness. + // + // A reader should never see false, because FR-012 removes an incomplete + // capture rather than retaining it. The field is what remains if that + // removal itself fails: the artefact is still identifiable as unusable, and + // a reader that does encounter it must not offer it as a restore point. + CaptureComplete *bool `json:"capture_complete,omitempty"` + + // ExternalLocation is whether each captured database lived outside the + // deployment. Diagnostic: it is what explains an artefact whose source + // endpoint is not one of the deployment's own. + ExternalLocation map[string]bool `json:"external_location,omitempty"` +} + +// refuseIncompleteCapture is CaptureComplete's reader, and the reason the field +// is worth writing. +// +// FR-012 removes an incomplete capture rather than retaining one, so this +// should never fire. It fires when that removal failed — the archive could not +// be deleted, the object could not be removed from the bucket, a snapshot could +// not be deleted from the repository — each of which is reported as a failure +// and leaves an artefact behind that the *name* does not distinguish from a good +// one. Retention ranks by recency and never reads metadata, and `--latest` +// selects by name, so both will hand exactly that artefact to a restore, being +// the newest. This is the last point at which it can be told apart, and the +// contract's rule for a reader that meets it is categorical: it must not be +// offered as a restore point. +// +// There is deliberately no --force escape, which is the one place this is +// stricter than the incomplete-*group* refusal on the Plakar path. That refusal +// is about which components are present, a shortfall an operator can see and +// reason about — no task-manager dump, restore the rest. This one is about a +// capture that reported it had not finished reading the database, so what is in +// the artefact is unknown, and there is nothing for a bypass to be a judgement +// about. An operator whose only artefact says this needs the previous one. +// +// Absent means complete, which is precisely what every previously released +// version produced (FR-024). +func (m *BackupMetadata) refuseIncompleteCapture(source string) error { + if m.CaptureComplete == nil || *m.CaptureComplete { + return nil + } + + return fmt.Errorf("refusing to restore %s: its metadata records capture_complete=false, meaning the capture that produced it did not finish reading every database it was asked for, so its contents are not a recovery point; an artefact this run had produced would have been removed rather than kept, so this one survived a failed removal — restore an earlier backup instead", source) } -// Neo4jEditionInfo encapsulates information about the detected Neo4j edition +// errNeo4jEditionUndetermined is the refusal a backup gives when the edition +// probe did not answer. Callers that need to recognise it match it with +// errors.Is. +var errNeo4jEditionUndetermined = errors.New("the Neo4j edition could not be determined") + +// Neo4jEditionInfo encapsulates information about the detected Neo4j edition. +// +// The three fields answer different questions, and the difference is the whole +// point of the type: +// +// - IsDetected says whether the edition was established at all. detectNeo4jEdition +// runs `cypher-shell` through an exec, so a pod restarting, a connection +// refused or an auth blip all end here. +// - IsCommunity is the *capture method* decision, and it is the destructive +// one: Community has no online backup, so it is answered by stopping +// Infrahub and taking Neo4j offline for a cold dump. It is therefore never +// true unless the edition was determined — a probe that failed is not +// evidence of Community, and reading it as Community turned a momentary +// `cypher-shell` failure against an internal Enterprise database into an +// unnecessary outage plus an offline dump. +// - Edition keeps the conservative *restore* fallback on an undetermined +// probe. Restoring a Community dump with the Community method is the safe +// reading of an unknown edition and does nothing to the running instance; +// the asymmetry with IsCommunity is deliberate, not an oversight. type Neo4jEditionInfo struct { Edition string IsCommunity bool @@ -44,8 +166,8 @@ type Neo4jEditionInfo struct { func NewNeo4jEditionInfo(edition string, err error) *Neo4jEditionInfo { if err != nil { return &Neo4jEditionInfo{ - Edition: neo4jEditionCommunity, // Default fallback - IsCommunity: true, + Edition: neo4jEditionCommunity, // Restore-side fallback only; see the type's doc comment. + IsCommunity: false, IsDetected: false, DetectionErr: err, } @@ -54,15 +176,43 @@ func NewNeo4jEditionInfo(edition string, err error) *Neo4jEditionInfo { normalized := strings.ToLower(edition) return &Neo4jEditionInfo{ Edition: normalized, - IsCommunity: strings.EqualFold(normalized, neo4jEditionCommunity), + IsCommunity: isCommunityEdition(normalized), IsDetected: true, } } +// RequiresOfflineCapture reports whether the backup has to stop Infrahub and +// take Neo4j offline for a cold dump, which is true only of a Community +// edition that was actually determined to be one. +func (info *Neo4jEditionInfo) RequiresOfflineCapture() bool { + return info.IsDetected && info.IsCommunity +} + +// requireDeterminedEdition refuses a backup whose edition is unknown, before +// the run has stopped anything. +// +// Neither capture method is safe to assume here: Community's offline dump costs +// an outage the deployment may not need, and Enterprise's online backup would +// silently produce an inconsistent capture of a Community database, which +// constitution Principle II forbids more strongly than it forbids failing. So +// the run stops and says what to check. +func (info *Neo4jEditionInfo) requireDeterminedEdition() error { + if info.IsDetected { + return nil + } + + return fmt.Errorf( + "cannot start the backup: %w, so this run cannot tell whether Neo4j has to be taken offline for a consistent capture; "+ + "no Infrahub service was stopped. The edition is read by running `cypher-shell` in the database container, so check that "+ + "the database service resolves to the Neo4j server, that it is running and that its credentials are right "+ + "(INFRAHUB_DB_USERNAME / INFRAHUB_DB_PASSWORD), then retry: %w", + errNeo4jEditionUndetermined, info.DetectionErr) +} + // LogDetection logs the detection result func (info *Neo4jEditionInfo) LogDetection(context string) { if !info.IsDetected { - logrus.Warnf("Could not determine Neo4j edition during %s; defaulting to community workflow: %v", context, info.DetectionErr) + logrus.Warnf("Could not determine Neo4j edition during %s; the capture method cannot be chosen from a probe that did not answer: %v", context, info.DetectionErr) } else { logrus.Infof("Detected Neo4j %s edition for %s", info.Edition, context) } @@ -97,21 +247,37 @@ func (info *Neo4jEditionInfo) ResolveRestoreEdition(backupEdition string) (strin return info.Edition, nil } -// detectBackupNeo4jEdition detects the Neo4j edition a backup is taken with. -// A backup never falls back to Community when detection fails: the Community -// workflow stops the Infrahub services, so a failed query (for example one sent -// to a pod that is not Neo4j) must not cause an outage. -func (iops *InfrahubOps) detectBackupNeo4jEdition() (*Neo4jEditionInfo, error) { +// detectNeo4jEditionInfo detects the Neo4j edition and returns structured information +func (iops *InfrahubOps) detectNeo4jEditionInfo(context string) *Neo4jEditionInfo { edition, err := iops.detectNeo4jEdition() - if err != nil { - return nil, fmt.Errorf("could not determine the Neo4j edition, so the backup was not started and no service was stopped; check that the database service resolves to the Neo4j server and that the Neo4j credentials are correct: %w", err) - } - info := NewNeo4jEditionInfo(edition, nil) - info.LogDetection("backup") - return info, nil + info := NewNeo4jEditionInfo(edition, err) + info.LogDetection(context) + return info } func (iops *InfrahubOps) detectNeo4jEdition() (string, error) { + // A database that lives outside the deployment has no container to run + // `cypher-shell` in, so the exec below would fail and be read as "edition + // undetermined" — which stops the run in requireDeterminedEdition, naming a + // container that was never going to be there. The edition of such a database + // was read over Bolt by the probe the gate ran, before anything was stopped; + // this returns that answer rather than asking a second time. Both gates + // probe — a capture's and a restore's — and this decision is reached from + // both directions, so it reads the record of either rather than the + // capture's alone, which is what left a restore against an external Neo4j + // with no edition and no way to proceed (T148). + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + return "", err + } + if source != nil { + if source.Facts.Edition == "" { + return "", fmt.Errorf("the external %s server did not report its edition", serviceNeo4j) + } + + return source.Facts.Edition, nil + } + output, err := iops.Exec("database", []string{ "cypher-shell", "-u", iops.config.Neo4jUsername, @@ -166,7 +332,7 @@ func (iops *InfrahubOps) createBackupMetadata(backupID string, includeTaskManage components = append(components, "task-manager-db") } - return &BackupMetadata{ + metadata := &BackupMetadata{ MetadataVersion: metadataVersion, BackupID: backupID, CreatedAt: time.Now().UTC().Format(time.RFC3339), @@ -175,4 +341,93 @@ func (iops *InfrahubOps) createBackupMetadata(backupID string, includeTaskManage Components: components, Neo4jEdition: strings.ToLower(neo4jEdition), } + + iops.applySourceProvenance(metadata, components) + + return metadata +} + +// applyCaptureCompleteness records what this run's captures reported (FR-012). +// +// It is called where the metadata is *serialized*, not where it is built, and +// that is the whole of it. createBackupMetadata runs before backupDatabase and +// backupTaskManagerDB, so incompleteCaptures is necessarily empty when it runs +// and the field it wrote was a prediction rather than a report — `true` in +// every artefact ever produced, whatever the captures went on to say. Read from +// here the captures have finished, so a verdict any of them recorded is already +// in. +// +// Both backends still gate on the same verdict before reaching serialization: +// the tarball path discards rather than archiving, and the Plakar path refuses +// before the metadata component that would carry the field is created. So this +// tool does not write `false` — which is FR-012's actual requirement, that an +// incomplete capture leaves nothing behind rather than something labelled. What +// this makes true is that the label cannot disagree with the gate, whatever a +// later change does to where the gate sits; and refuseIncompleteCapture, its +// reader, is left guarding the artefacts a *failed removal* or an older tool can +// leave behind, against a value that now means what it says. +// +// A run with no external database writes nothing, exactly as +// applySourceProvenance writes nothing, and for the same reason (FR-015). +func (iops *InfrahubOps) applyCaptureCompleteness(metadata *BackupMetadata) { + if len(iops.externalSources) == 0 { + return + } + + complete := iops.capturesComplete() + metadata.CaptureComplete = &complete +} + +// applySourceProvenance records where this capture's data came from, for a run +// that read at least one database living outside the deployment (FR-005). +// +// A run whose databases are all internal returns here having written nothing, so +// its metadata is byte-for-byte what it has always been (FR-015). That is also +// the honest encoding rather than a convenience: the provenance fields exist to +// describe an endpoint the artefact does not otherwise account for, and an +// internal deployment has none — which is precisely the reading +// contracts/artifact-and-metadata.md gives to their absence. +// +// Within a run that did read an external database, every database is recorded, +// internal ones included, so the fields themselves never signal externality. +// +// services are the deployment service names this run captured, which is what +// endpoints resolve under. They are taken as an argument rather than read from +// metadata.Components because the Plakar backend replaces that list with its own +// component names after this has run, and a key nothing resolves under describes +// nothing. +func (iops *InfrahubOps) applySourceProvenance(metadata *BackupMetadata, services []string) { + if len(iops.externalSources) == 0 { + return + } + + metadata.ExternalLocation = make(map[string]bool, len(services)) + + for _, service := range services { + source := iops.externalSourceFor(service) + metadata.ExternalLocation[service] = source != nil + if source == nil { + continue + } + + if endpoints := renderHostPortList(source.captureTargets()); len(endpoints) > 0 { + if metadata.SourceEndpoints == nil { + metadata.SourceEndpoints = map[string][]string{} + } + metadata.SourceEndpoints[service] = endpoints + } + + // Absent rather than a map of unknowns for a database whose roles this + // run never asked for; see probedFacts.RolesObserved. + if !source.Facts.RolesObserved { + continue + } + + if roles := source.observedRoles(); len(roles) > 0 { + if metadata.SourceRoles == nil { + metadata.SourceRoles = map[string]map[string]string{} + } + metadata.SourceRoles[service] = roles + } + } } diff --git a/src/internal/app/backup_metadata_test.go b/src/internal/app/backup_metadata_test.go index 3dab18d9..d12eef70 100644 --- a/src/internal/app/backup_metadata_test.go +++ b/src/internal/app/backup_metadata_test.go @@ -3,12 +3,648 @@ package app import ( "encoding/json" "errors" + "fmt" "os" "path/filepath" + "slices" "strings" "testing" ) +// probeFailure is what a transient edition probe failure looks like: the +// wrapped exec error detectNeo4jEdition returns when `cypher-shell` could not +// be run or could not connect. +func probeFailure() error { + return fmt.Errorf("failed to query neo4j edition: %w", errors.New("exit status 1: Connection refused")) +} + +// TestNeo4jEditionInfo_ErroredProbeDoesNotTakeTheOfflineBranch is the defect +// this test file exists for. Community has no online backup, so the Community +// branch stops Infrahub and takes Neo4j offline for a cold dump — and the +// constructor used to report a probe that merely *errored* as Community. A +// momentary `cypher-shell` failure against an internal Enterprise database +// therefore produced an outage the deployment did not need, and an offline dump. +func TestNeo4jEditionInfo_ErroredProbeDoesNotTakeTheOfflineBranch(t *testing.T) { + info := NewNeo4jEditionInfo("", probeFailure()) + + if info.IsDetected { + t.Error("IsDetected = true for a probe that failed") + } + if info.IsCommunity { + t.Error("IsCommunity = true for a probe that failed: a failure is not evidence of Community") + } + if info.RequiresOfflineCapture() { + t.Error("RequiresOfflineCapture = true for a probe that failed: the run would stop Infrahub and dump Neo4j cold") + } +} + +// TestNeo4jEditionInfo_RequireDeterminedEdition covers the refusal the backup +// paths make before anything is stopped. +func TestNeo4jEditionInfo_RequireDeterminedEdition(t *testing.T) { + t.Run("an undetermined edition refuses the run", func(t *testing.T) { + err := NewNeo4jEditionInfo("", probeFailure()).requireDeterminedEdition() + if err == nil { + t.Fatal("requireDeterminedEdition allowed a backup whose edition is unknown") + } + if !errors.Is(err, errNeo4jEditionUndetermined) { + t.Errorf("err = %v, want errNeo4jEditionUndetermined so callers can recognise the refusal", err) + } + // The operator has to be able to tell an aborted run from an outage. + if !strings.Contains(err.Error(), "no Infrahub service was stopped") { + t.Errorf("err = %v, want it to state that nothing was stopped", err) + } + if !strings.Contains(err.Error(), "INFRAHUB_DB_USERNAME") { + t.Errorf("err = %v, want it to name what to check", err) + } + }) + + t.Run("a determined edition allows the run", func(t *testing.T) { + for _, edition := range []string{neo4jEditionCommunity, neo4jEditionEnterprise} { + if err := NewNeo4jEditionInfo(edition, nil).requireDeterminedEdition(); err != nil { + t.Errorf("requireDeterminedEdition refused a determined %s edition: %v", edition, err) + } + } + }) +} + +// TestNeo4jEditionInfo_DeterminedEditions keeps the detected cases behaving +// exactly as the release does: a determined Community edition still takes the +// offline branch, and Enterprise still does not. +func TestNeo4jEditionInfo_DeterminedEditions(t *testing.T) { + tests := []struct { + name string + detected string + wantEdition string + wantOffline bool + }{ + {"community", "community", neo4jEditionCommunity, true}, + {"community reported in mixed case", "Community", neo4jEditionCommunity, true}, + {"enterprise", "enterprise", neo4jEditionEnterprise, false}, + {"enterprise reported in upper case", "ENTERPRISE", neo4jEditionEnterprise, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info := NewNeo4jEditionInfo(tt.detected, nil) + if !info.IsDetected { + t.Error("IsDetected = false for a probe that answered") + } + if info.Edition != tt.wantEdition { + t.Errorf("Edition = %q, want %q", info.Edition, tt.wantEdition) + } + if got := info.RequiresOfflineCapture(); got != tt.wantOffline { + t.Errorf("RequiresOfflineCapture = %t, want %t", got, tt.wantOffline) + } + }) + } +} + +// TestNeo4jEditionInfo_RestoreFallbackIsUnchanged guards the asymmetry the +// type documents. The stop/dump decision now requires a determined edition, +// but the restore side keeps its conservative fallback: a Community dump is +// still restored with the Community method when the probe did not answer, +// which is what an existing deployment already relies on and does nothing to +// the running instance. +// TestDetectNeo4jEditionReadsTheGateOfEitherDirection is T148. The edition of +// an external database is read by the gate's probe, and detectNeo4jEdition +// returned that answer — but only from the capture's record. A restore records +// a workload instead, so against an external Neo4j the decision found nothing, +// and the restore stopped in requireDeterminedEdition against a database whose +// edition its own gate had established minutes earlier. Both restore entry +// points reach this decision after prepareDatabaseRestore, so the feature's +// restore path never ran. +// +// The fake answers `cypher-shell` with Community, so a fall-through to the +// absent container is visible twice over: as the wrong edition, and as a +// recorded exec. +func TestDetectNeo4jEditionReadsTheGateOfEitherDirection(t *testing.T) { + external := func(t *testing.T) (*InfrahubOps, *gatedBackend) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + + return newGatedOps(t, backend, BackendPlakar), backend + } + noProbeExec := func(t *testing.T, backend *gatedBackend) { + t.Helper() + for _, call := range backend.execs { + if strings.HasPrefix(call, "cypher-shell") { + t.Errorf("the edition was asked of a container that does not exist (%q), want the gate's probe read instead", call) + } + } + } + + t.Run("a restore against an external Neo4j reads the edition its gate probed", func(t *testing.T) { + iops, backend := external(t) + iops.recordExternalRestore(serviceNeo4j, &externalRestore{externalSource: &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j}, + Facts: probedFacts{Edition: neo4jEditionEnterprise}, + }}) + + edition, err := iops.detectNeo4jEdition() + if err != nil { + t.Fatalf("detectNeo4jEdition() failed: %v; want the edition the restore's gate established (T148)", err) + } + if edition != neo4jEditionEnterprise { + t.Errorf("edition = %q, want %q from the probe's facts", edition, neo4jEditionEnterprise) + } + noProbeExec(t, backend) + }) + + t.Run("a capture against an external Neo4j is unchanged", func(t *testing.T) { + iops, backend := external(t) + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j}, + Facts: probedFacts{Edition: neo4jEditionEnterprise}, + }) + + edition, err := iops.detectNeo4jEdition() + if err != nil { + t.Fatalf("detectNeo4jEdition() failed: %v", err) + } + if edition != neo4jEditionEnterprise { + t.Errorf("edition = %q, want %q from the probe's facts", edition, neo4jEditionEnterprise) + } + noProbeExec(t, backend) + }) + + t.Run("a restore whose probe reported no edition is undetermined, not Community", func(t *testing.T) { + iops, backend := external(t) + iops.recordExternalRestore(serviceNeo4j, &externalRestore{externalSource: &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j}, + }}) + + edition, err := iops.detectNeo4jEdition() + if err == nil { + t.Fatalf("detectNeo4jEdition() = %q, nil; want an edition the server did not report left undetermined", edition) + } + if !strings.Contains(err.Error(), "did not report its edition") { + t.Errorf("err = %v, want it to say the server did not report an edition", err) + } + noProbeExec(t, backend) + }) +} + +func TestNeo4jEditionInfo_RestoreFallbackIsUnchanged(t *testing.T) { + undetermined := NewNeo4jEditionInfo("", probeFailure()) + + if undetermined.Edition != neo4jEditionCommunity { + t.Errorf("Edition = %q, want the community restore fallback", undetermined.Edition) + } + + edition, err := undetermined.ResolveRestoreEdition(neo4jEditionCommunity) + if err != nil { + t.Fatalf("ResolveRestoreEdition failed for a community backup: %v", err) + } + if edition != neo4jEditionCommunity { + t.Errorf("edition = %q, want %q", edition, neo4jEditionCommunity) + } + + if _, err := undetermined.ResolveRestoreEdition(neo4jEditionEnterprise); err == nil { + t.Error("ResolveRestoreEdition accepted an Enterprise backup against an unknown edition") + } + + // And the detected paths are untouched. + enterprise := NewNeo4jEditionInfo(neo4jEditionEnterprise, nil) + edition, err = enterprise.ResolveRestoreEdition(neo4jEditionCommunity) + if err != nil { + t.Fatalf("ResolveRestoreEdition failed for a community backup on enterprise: %v", err) + } + if edition != neo4jEditionCommunity { + t.Errorf("edition = %q, want the community restore method for a community backup", edition) + } +} + +// --------------------------------------------------------------------------- +// T042/T044: the provenance fields, and the compatibility they must not break +// --------------------------------------------------------------------------- + +// releasedBackupMetadata is BackupMetadata as a previously released version of +// the tool declares it: every field that shipped before 007, and none of the +// provenance fields it added. +// +// It is a declared copy rather than the live struct on purpose. Once those +// fields exist on BackupMetadata, unmarshalling a new artefact into it stops +// testing the old-reads-new direction at all — the reader would know every field +// it is supposed to be ignorant of, and FR-024's untested direction would go on +// being untested while looking covered. This is the only shape in the repository +// that can fail when an addition turns out not to be ignorable. +type releasedBackupMetadata struct { + MetadataVersion int `json:"metadata_version"` + BackupID string `json:"backup_id"` + CreatedAt string `json:"created_at"` + ToolVersion string `json:"tool_version"` + InfrahubVersion string `json:"infrahub_version"` + Components []string `json:"components"` + Checksums map[string]string `json:"checksums,omitempty"` + Neo4jEdition string `json:"neo4j_edition,omitempty"` + Redacted bool `json:"redacted,omitempty"` + Encrypted bool `json:"encrypted,omitempty"` +} + +// newProvenanceArtefact is an artefact carrying every field 007 added, in the +// shape the writer produces them, plus a field no version has ever defined. +// +// TestProvenanceArtefactMatchesWhatTheWriterProduces keeps this fixture honest: +// a fixture that guessed the shape would pin compatibility for an artefact the +// tool never writes. +func newProvenanceArtefact() []byte { + return []byte(`{ + "metadata_version": 2025111200, + "backup_id": "20260101_120000", + "created_at": "2026-01-01T12:00:00Z", + "tool_version": "9.9.9", + "infrahub_version": "1.4.0", + "components": ["database", "task-manager-db"], + "checksums": {"neo4j.dump": "abc123"}, + "neo4j_edition": "enterprise", + "source_endpoints": { + "database": ["db-a.example.net:6362", "db-b.example.net:6362"], + "task-manager-db": ["pg-a.example.net:5432"] + }, + "source_roles": { + "database": {"db-a.example.net:6362": "primary", "db-b.example.net:6362": "unknown"} + }, + "capture_complete": true, + "external_location": {"database": true, "task-manager-db": true}, + "a_field_no_version_has_ever_defined": {"nested": [1, 2, 3]} + }`) +} + +// externalProvenanceOps is a run that read a Neo4j cluster and a PostgreSQL +// server, both outside the deployment, and observed roles for the Neo4j members +// only — which is the shape the two probes actually produce. +func externalProvenanceOps() *InfrahubOps { + iops := &InfrahubOps{config: &Configuration{}} + + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Database: "neo4j", + Hosts: []HostPort{{Host: "db-a.example.net", Port: 7687}, {Host: "db-b.example.net", Port: 7687}}, + ClientPort: 7687, + BackupPort: 6362, + }, + Facts: probedFacts{ + RolesObserved: true, + Members: []observedMember{{Address: "db-a.example.net:7687", Role: "primary"}}, + Answered: HostPort{Host: "db-b.example.net", Port: 7687}, + }, + }) + + iops.recordExternalSource(serviceTaskManagerDB, &externalSource{ + Endpoint: &DatabaseEndpoint{ + Service: serviceTaskManagerDB, + Location: EndpointLocationExternal, + Database: "prefect", + Hosts: []HostPort{{Host: "pg-a.example.net", Port: 5432}}, + ClientPort: 5432, + }, + // PostgreSQL is never asked for member roles, so it observed none. + Facts: probedFacts{Answered: HostPort{Host: "pg-a.example.net", Port: 5432}}, + }) + + return iops +} + +// artefactMetadata is the metadata an artefact actually carries: built, and +// then completed where both write paths complete it. +// +// The two steps are separate in production for a reason FR-012 turns on. +// createBackupMetadata runs *before* backupDatabase and backupTaskManagerDB, so +// a completeness verdict read there is a prediction of what the captures will +// say — and necessarily "complete", since nothing has run. It is read again at +// serialization, which is what the writers do and what this mirrors; a test that +// exercised only the first half could not tell the two apart, which is how the +// field came to be unfalsifiable (see applyCaptureCompleteness). +func artefactMetadata(iops *InfrahubOps) *BackupMetadata { + metadata := iops.createBackupMetadata("20260101_120000", true, "1.4.0", "enterprise") + iops.applyCaptureCompleteness(metadata) + + return metadata +} + +// TestProvenanceArtefactMatchesWhatTheWriterProduces ties the compatibility +// fixture to the writer. Every provenance key the fixture carries must be the +// key and the JSON shape the writers actually emit, so that the compatibility +// this file pins is compatibility with a real artefact. +func TestProvenanceArtefactMatchesWhatTheWriterProduces(t *testing.T) { + written, err := json.Marshal(artefactMetadata(externalProvenanceOps())) + if err != nil { + t.Fatalf("marshalling the written metadata = %v, want nil", err) + } + + var writtenShape, fixtureShape map[string]json.RawMessage + if err := json.Unmarshal(written, &writtenShape); err != nil { + t.Fatalf("unmarshalling the written metadata = %v, want nil", err) + } + if err := json.Unmarshal(newProvenanceArtefact(), &fixtureShape); err != nil { + t.Fatalf("unmarshalling the fixture = %v, want nil", err) + } + + for _, key := range []string{"source_endpoints", "source_roles", "capture_complete", "external_location"} { + if _, ok := writtenShape[key]; !ok { + t.Errorf("the writer emitted no %q for an external run; the fixture pins a shape nothing produces", key) + + continue + } + + // Same JSON kind — object against object, not object against array. + // This is the mismatch a hand-written fixture actually gets wrong. + if got, want := fixtureShape[key][0], writtenShape[key][0]; got != want { + t.Errorf("fixture %q starts with %q, the writer's with %q: the fixture is a different JSON shape", key, got, want) + } + } +} + +// TestBackupMetadataRecordsWhereAnExternalCaptureCameFrom is T042 against the +// contract: what each field says, and — for source_roles — what its absence +// says, which is a different statement from unknown. +func TestBackupMetadataRecordsWhereAnExternalCaptureCameFrom(t *testing.T) { + metadata := artefactMetadata(externalProvenanceOps()) + + t.Run("the declared metadata version does not move", func(t *testing.T) { + // FR-024: these fields are additive, so a released tool that compares + // the version must still accept the artefact. + if metadata.MetadataVersion != 2025111200 { + t.Errorf("MetadataVersion = %d, want it unchanged", metadata.MetadataVersion) + } + }) + + t.Run("the endpoints are recorded in try order", func(t *testing.T) { + want := []string{"db-a.example.net:6362", "db-b.example.net:6362"} + if got := metadata.SourceEndpoints[serviceNeo4j]; !slices.Equal(got, want) { + t.Errorf("SourceEndpoints[%q] = %v, want %v in try order", serviceNeo4j, got, want) + } + }) + + t.Run("PostgreSQL provenance names the port the dump travelled over", func(t *testing.T) { + // An earlier version of captureTargets stamped Neo4j's backup port on + // every service, which recorded :6362 for a dump that went over :5432 — + // a false statement about where the data came from. + want := []string{"pg-a.example.net:5432"} + if got := metadata.SourceEndpoints[serviceTaskManagerDB]; !slices.Equal(got, want) { + t.Errorf("SourceEndpoints[%q] = %v, want %v", serviceTaskManagerDB, got, want) + } + }) + + t.Run("no field claims which endpoint served the capture", func(t *testing.T) { + // FR-005 forbids it: with several endpoints supplied, no documented + // interface reports it. The probe's Answered is deliberately not here. + blob, err := json.Marshal(metadata) + if err != nil { + t.Fatalf("json.Marshal() = %v, want nil", err) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(blob, &fields); err != nil { + t.Fatalf("json.Unmarshal() = %v, want nil", err) + } + for key := range fields { + if strings.Contains(key, "answer") || strings.Contains(key, "served") || strings.Contains(key, "captured_from") { + t.Errorf("metadata carries %q: no field may claim which endpoint served the capture (FR-005)", key) + } + } + }) + + t.Run("an observed role that no member matched is unknown, not absent", func(t *testing.T) { + roles := metadata.SourceRoles[serviceNeo4j] + if got := roles["db-a.example.net:6362"]; got != "primary" { + t.Errorf("role of db-a = %q, want primary as the server reported it", got) + } + if got, ok := roles["db-b.example.net:6362"]; !ok || got != roleUnknown { + t.Errorf("role of db-b = %q (present %t), want %q as a recorded value", got, ok, roleUnknown) + } + }) + + t.Run("a database whose roles were never sought carries none", func(t *testing.T) { + // Absent means "roles not observed"; unknown means "observed, and the + // server named no role". Recording unknown for PostgreSQL, which is + // never asked, would claim an observation the run never made. + if roles, ok := metadata.SourceRoles[serviceTaskManagerDB]; ok { + t.Errorf("SourceRoles[%q] = %v, want it absent: its roles were never observed", serviceTaskManagerDB, roles) + } + }) + + t.Run("capture_complete states what the capture reported", func(t *testing.T) { + if metadata.CaptureComplete == nil || !*metadata.CaptureComplete { + t.Errorf("CaptureComplete = %v, want true for a run whose captures all reported complete", metadata.CaptureComplete) + } + }) + + t.Run("external_location covers every database, so the other fields do not signal externality", func(t *testing.T) { + // Keyed by the deployment service name, which is what an endpoint + // resolves under — not by whatever a backend calls its components. + for _, service := range []string{serviceNeo4j, serviceTaskManagerDB} { + if external, ok := metadata.ExternalLocation[service]; !ok || !external { + t.Errorf("ExternalLocation[%q] = %t (present %t), want true", service, external, ok) + } + } + }) + + t.Run("an internal database in an external run is recorded as internal", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j, Database: "neo4j", Hosts: []HostPort{{Host: "db-a", Port: 7687}}, BackupPort: 6362}, + Facts: probedFacts{RolesObserved: true}, + }) + + mixed := iops.createBackupMetadata("20260101_120000", true, "1.4.0", "enterprise") + if external, ok := mixed.ExternalLocation[serviceTaskManagerDB]; !ok || external { + t.Errorf("ExternalLocation[%q] = %t (present %t), want false: it is recorded, and recorded as internal", serviceTaskManagerDB, external, ok) + } + if _, ok := mixed.SourceEndpoints[serviceTaskManagerDB]; ok { + t.Errorf("SourceEndpoints[%q] = %v, want none for a database inside the deployment", serviceTaskManagerDB, mixed.SourceEndpoints[serviceTaskManagerDB]) + } + }) + + t.Run("an incomplete capture is recorded as incomplete", func(t *testing.T) { + // A reader should never meet this, because FR-012 removes such an + // artefact rather than retaining it. It is what remains if the removal + // itself failed, so it has to be writable and it has to survive a + // round trip as false rather than as absent. + // + // The order below is production's order, and that is the assertion. + // The metadata is built first, the capture reports its verdict second — + // createBackupMetadata runs before backupDatabase — and the field is + // written last. Built and read in one step, as it was, the map is always + // empty when it is read and this case cannot arise at all: `true` went + // into every artefact ever written, and the reader that refuses `false` + // guarded a value nothing could produce. + iops := externalProvenanceOps() + metadata := iops.createBackupMetadata("20260101_120000", true, "1.4.0", "enterprise") + + iops.recordIncompleteCapture(serviceNeo4j, "the operation produced an artifact and then reported a problem") + iops.applyCaptureCompleteness(metadata) + + blob, err := json.Marshal(metadata) + if err != nil { + t.Fatalf("json.Marshal() = %v, want nil", err) + } + + var reread BackupMetadata + if err := json.Unmarshal(blob, &reread); err != nil { + t.Fatalf("json.Unmarshal() = %v, want nil", err) + } + if reread.CaptureComplete == nil { + t.Fatal("CaptureComplete = absent after a round trip, want false: absent means complete, so this would read as its own opposite") + } + if *reread.CaptureComplete { + t.Error("CaptureComplete = true, want false") + } + }) +} + +// TestInternalDeploymentMetadataIsUnchanged is FR-015. A deployment whose +// databases are all internal must produce the metadata it always has, and the +// check is on the serialized bytes rather than on the struct: a field that +// appears only in JSON is still a change to the artefact. +func TestInternalDeploymentMetadataIsUnchanged(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + + // Through both steps a writer performs, so the completeness step is held to + // the same guarantee: an internal run writes no provenance field, and that + // includes the one now written at serialization. + metadata := artefactMetadata(iops) + + blob, err := json.Marshal(metadata) + if err != nil { + t.Fatalf("json.Marshal() = %v, want nil", err) + } + + var fields map[string]json.RawMessage + if err := json.Unmarshal(blob, &fields); err != nil { + t.Fatalf("json.Unmarshal() = %v, want nil", err) + } + + for _, key := range []string{"source_endpoints", "source_roles", "capture_complete", "external_location"} { + if _, ok := fields[key]; ok { + t.Errorf("an all-internal run wrote %q; its metadata must be what it has always been (FR-015)", key) + } + } + + // And the keys it does write are exactly the pre-feature set. + want := []string{"backup_id", "components", "created_at", "infrahub_version", "metadata_version", "neo4j_edition", "tool_version"} + got := make([]string, 0, len(fields)) + for key := range fields { + got = append(got, key) + } + slices.Sort(got) + if !slices.Equal(got, want) { + t.Errorf("metadata keys = %v, want %v", got, want) + } +} + +// TestReleasedReaderIsUnaffectedByTheProvenanceFields is FR-024's untested +// direction: a previously released tool reads an artefact carrying all four new +// fields. It must neither refuse it nor act incorrectly on the fields it can +// see, and it has no way to notice the ones it cannot. +func TestReleasedReaderIsUnaffectedByTheProvenanceFields(t *testing.T) { + var released releasedBackupMetadata + if err := json.Unmarshal(newProvenanceArtefact(), &released); err != nil { + t.Fatalf("a released reader refused a new artefact: %v", err) + } + + if released.MetadataVersion != metadataVersion { + t.Errorf("MetadataVersion = %d, want %d: a released tool comparing this must still accept the artefact", released.MetadataVersion, metadataVersion) + } + if released.Neo4jEdition != neo4jEditionEnterprise { + t.Errorf("Neo4jEdition = %q, want enterprise: the restore's edition decision reads this", released.Neo4jEdition) + } + if got := released.Components; !slices.Equal(got, []string{serviceNeo4j, serviceTaskManagerDB}) { + t.Errorf("Components = %v, want both databases: this is what decides which are restored", got) + } + if released.Checksums["neo4j.dump"] != "abc123" { + t.Errorf("Checksums = %v, want the artefact's own", released.Checksums) + } + if released.Redacted || released.Encrypted { + t.Errorf("Redacted/Encrypted = %t/%t, want both false: neither is stated by this artefact", released.Redacted, released.Encrypted) + } +} + +// TestNewReaderTreatsAPreFeatureArtefactAsInternalAndComplete is FR-016 and the +// reader rule in contracts/artifact-and-metadata.md: an artefact stating none of +// the four fields is an internal, complete capture from an unrecorded endpoint — +// which is exactly what every previously released version produced. No warning, +// no migration, no operator action. +func TestNewReaderTreatsAPreFeatureArtefactAsInternalAndComplete(t *testing.T) { + preFeature := []byte(`{ + "metadata_version": 2025111200, + "backup_id": "20250101_120000", + "created_at": "2025-01-01T12:00:00Z", + "tool_version": "1.0.0", + "infrahub_version": "1.0.0", + "components": ["database"], + "neo4j_edition": "enterprise" + }`) + + var metadata BackupMetadata + if err := json.Unmarshal(preFeature, &metadata); err != nil { + t.Fatalf("the current reader refused a pre-feature artefact: %v", err) + } + + if metadata.CaptureComplete != nil { + t.Errorf("CaptureComplete = %v, want absent: absence is what says complete", *metadata.CaptureComplete) + } + if metadata.SourceEndpoints != nil || metadata.SourceRoles != nil { + t.Errorf("SourceEndpoints/SourceRoles = %v/%v, want both absent rather than empty maps that would read as 'recorded nothing'", + metadata.SourceEndpoints, metadata.SourceRoles) + } + if metadata.ExternalLocation != nil { + t.Errorf("ExternalLocation = %v, want absent: an unstated location is internal", metadata.ExternalLocation) + } + + // The distinction the reader rule turns on: absent is not false. A reader + // that collapsed the two would refuse every artefact written before this + // feature as an incomplete capture. + if metadata.CaptureComplete != nil && !*metadata.CaptureComplete { + t.Error("a pre-feature artefact read as an incomplete capture") + } +} + +// TestRefuseIncompleteCaptureIsTheReaderOfTheField is T102. The contract calls +// capture_complete the last-resort identifier of an unusable artefact — "a +// reader that does encounter it MUST NOT offer it as a restore point" — and it +// had no reader on any restore or restore-point-selection path, so the field +// never rejected anything. `--latest` selects by name and recency, so the +// artefact a failed removal left behind is exactly the one it picks. +func TestRefuseIncompleteCaptureIsTheReaderOfTheField(t *testing.T) { + complete := true + incomplete := false + + tests := []struct { + name string + field *bool + wantErr bool + }{ + {"a pre-feature artefact states nothing, and absence means complete", nil, false}, + {"a complete capture restores", &complete, false}, + {"an incomplete capture is refused", &incomplete, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metadata := &BackupMetadata{CaptureComplete: tt.field} + err := metadata.refuseIncompleteCapture("infrahub_backup_20260903_120000.tar.gz") + if tt.wantErr != (err != nil) { + t.Fatalf("refuseIncompleteCapture() = %v, want an error: %t", err, tt.wantErr) + } + if err == nil { + return + } + if !strings.Contains(err.Error(), "infrahub_backup_20260903_120000.tar.gz") { + t.Errorf("err = %v, want the artefact named", err) + } + if !strings.Contains(err.Error(), "capture_complete=false") { + t.Errorf("err = %v, want the field that decided it named", err) + } + if !strings.Contains(err.Error(), "restore an earlier backup") { + t.Errorf("err = %v, want the operator action named (SC-008)", err) + } + }) + } +} + // editionProbeFailureBackend is a deployment whose database service does not // answer the Neo4j edition query, as when the query lands on a pod that is not // Neo4j. It records every service it is asked to stop. @@ -55,7 +691,7 @@ func TestBackupStopsWhenNeo4jEditionIsUnknown(t *testing.T) { if err == nil { t.Fatal("backup succeeded although the Neo4j edition could not be determined") } - if !strings.Contains(err.Error(), "could not determine the Neo4j edition") { + if !errors.Is(err, errNeo4jEditionUndetermined) { t.Errorf("err = %v, want it to say the Neo4j edition could not be determined", err) } if !strings.Contains(err.Error(), "executable file not found") { diff --git a/src/internal/app/backup_neo4j.go b/src/internal/app/backup_neo4j.go index eef43a67..a6830e90 100644 --- a/src/internal/app/backup_neo4j.go +++ b/src/internal/app/backup_neo4j.go @@ -1,6 +1,7 @@ package app import ( + "errors" "fmt" "io" "os" @@ -41,6 +42,18 @@ const ( // The backup is created with --compress=false for better Plakar deduplication. func (iops *InfrahubOps) backupNeo4jEnterpriseStream(backupMetadata string) (func() (io.ReadCloser, error), error) { return func() (io.ReadCloser, error) { + // The branch is here rather than where the factory is built, so that the + // transient workload's life covers the stream rather than starting when + // the factory was merely constructed. A nil source is an internal + // database, and everything below is the path it has always taken. + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + return nil, err + } + if source != nil { + return iops.streamNeo4jEnterpriseExternal(backupMetadata) + } + cleanupBackupDir := func() { if _, err := iops.Exec("database", []string{"rm", "-rf", neo4jTempBackupDir}, nil); err != nil { logrus.Warnf("Failed to remove temporary Neo4j backup directory: %v", err) @@ -55,14 +68,12 @@ func (iops *InfrahubOps) backupNeo4jEnterpriseStream(backupMetadata string) (fun } // Run backup command separately so its stdout logs don't contaminate the data stream - if output, err := iops.Exec("database", []string{ - "neo4j-admin", "database", "backup", - "--expand-commands", - "--include-metadata=" + backupMetadata, - "--compress=false", - "--to-path=" + neo4jTempBackupDir, - iops.config.Neo4jDatabase, - }, nil); err != nil { + if output, err := iops.Exec("database", neo4jCaptureCommand(neo4jCaptureRequest{ + Database: iops.config.Neo4jDatabase, + BackupMetadata: backupMetadata, + ToPath: neo4jTempBackupDir, + Uncompressed: true, + }), nil); err != nil { cleanupBackupDir() return nil, fmt.Errorf("failed to backup neo4j: %w\nOutput: %v", err, output) } @@ -124,24 +135,86 @@ func (iops *InfrahubOps) backupNeo4jCommunityStream() (func() (io.ReadCloser, er // is considered stalled if no data has been read. const defaultStreamIdleTimeout = 30 * time.Minute -// execReadCloser wraps an exec stdout pipe with cleanup logic and idle timeout. +// execReadCloser wraps an exec stdout pipe with cleanup logic, an idle timeout +// and, optionally, a deadline on the whole stream. +// +// The consumer is kloset, which reads a record to EOF and then closes it with +// its error discarded. So the outcome of the command producing the stream has +// to be reported by Read, not by Close: a command that dies part-way through — +// a transient pod removed at its activeDeadlineSeconds, a dump that fails — ends +// its stdout like a command that finished, and an EOF taken at face value is a +// truncated artifact stored as a complete one. On EOF, Read waits for the +// command and returns its failure in place of the EOF, which kloset records as +// a source error and commitComponentSnapshot refuses. type execReadCloser struct { reader io.ReadCloser wait func() error cleanup func() closed bool - timedOut bool idleTimeout time.Duration // 0 = no timeout - timer *time.Timer // reusable timer for idle timeout + timer *time.Timer // reusable timer for idle timeout and deadline + + // deadline is when the stream must have ended, or the zero time for none. + // It exists for a producer that runs in a pod with its own + // activeDeadlineSeconds: the idle timeout cannot stop a slow transfer that + // keeps making progress, so without this the cluster, not the tool, would + // be what ends the stream. + deadline time.Time + + // failure is the error every Read returns once the stream has failed. + failure error + + // waited records that wait has been called, and waitErr what it returned: + // wait is cmd.Wait, which may be called once. + waited bool + waitErr error } var errStreamIdleTimeout = fmt.Errorf("stream idle timeout") +// errStreamDeadline is what a Read returns once the stream has run past its +// deadline. +var errStreamDeadline = fmt.Errorf("stream deadline exceeded") + func (e *execReadCloser) Read(p []byte) (int, error) { - if e.timedOut { - return 0, errStreamIdleTimeout + if e.failure != nil { + return 0, e.failure + } + // Waiting for the command closed its pipe, and kloset reads once more after + // an EOF: reading the closed pipe would report "file already closed" and + // turn a stream that completed into one kloset records as failed. + if e.waited { + return 0, io.EOF } - if e.idleTimeout <= 0 { + + n, err := e.read(p) + if errors.Is(err, io.EOF) { + if waitErr := e.finish(); waitErr != nil { + e.failure = fmt.Errorf("the stream ended because the command producing it failed, so what was read is incomplete: %w", waitErr) + + return n, e.failure + } + } + + return n, err +} + +// read is one Read of the underlying pipe, under the idle timeout and the +// deadline, whichever is nearer. +func (e *execReadCloser) read(p []byte) (int, error) { + limit := e.idleTimeout + deadlineIsNearer := false + if !e.deadline.IsZero() { + remaining := time.Until(e.deadline) + if remaining <= 0 { + return 0, e.fail(fmt.Errorf("%w: the stream did not end within its bound", errStreamDeadline)) + } + if limit <= 0 || remaining < limit { + limit = remaining + deadlineIsNearer = true + } + } + if limit <= 0 { return e.reader.Read(p) } @@ -157,7 +230,7 @@ func (e *execReadCloser) Read(p []byte) (int, error) { // Lazily create the timer on first use, reset on subsequent calls if e.timer == nil { - e.timer = time.NewTimer(e.idleTimeout) + e.timer = time.NewTimer(limit) } else { if !e.timer.Stop() { select { @@ -165,18 +238,41 @@ func (e *execReadCloser) Read(p []byte) (int, error) { default: } } - e.timer.Reset(e.idleTimeout) + e.timer.Reset(limit) } select { case res := <-ch: return res.n, res.err case <-e.timer.C: - e.timedOut = true - // Close the underlying reader to unblock the goroutine - e.reader.Close() - return 0, fmt.Errorf("stream idle timeout after %v with no data", e.idleTimeout) + if deadlineIsNearer { + return 0, e.fail(fmt.Errorf("%w: the stream did not end within its bound", errStreamDeadline)) + } + + return 0, e.fail(fmt.Errorf("%w after %v with no data", errStreamIdleTimeout, e.idleTimeout)) + } +} + +// fail records err as the stream's outcome and closes the underlying reader, +// which unblocks a Read still in flight and tells the producer to stop. +func (e *execReadCloser) fail(err error) error { + e.failure = err + e.reader.Close() + + return err +} + +// finish waits for the producing command, once. +func (e *execReadCloser) finish() error { + if e.waited { + return e.waitErr + } + e.waited = true + if e.wait != nil { + e.waitErr = e.wait() } + + return e.waitErr } func (e *execReadCloser) Close() error { @@ -190,14 +286,17 @@ func (e *execReadCloser) Close() error { e.timer.Stop() } - // Close the reader first (may signal EOF to the process) + // Close the reader first (may signal EOF to the process). Once the command + // has been waited for, its pipe is already closed and the error from closing + // it again says nothing about the stream. + alreadyWaited := e.waited readErr := e.reader.Close() + if alreadyWaited { + readErr = nil + } // Wait for the process to finish - var waitErr error - if e.wait != nil { - waitErr = e.wait() - } + waitErr := e.finish() // Run cleanup if e.cleanup != nil { @@ -221,6 +320,14 @@ func (iops *InfrahubOps) backupDatabase(backupDir string, backupMetadata string, } func (iops *InfrahubOps) backupNeo4jEnterprise(backupDir string, backupMetadata string) error { + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + return err + } + if source != nil { + return iops.backupNeo4jEnterpriseExternal(backupDir, backupMetadata) + } + logrus.Info("Backing up Neo4j database (Enterprise Edition online backup)...") if _, err := iops.Exec("database", []string{"mkdir", "-p", neo4jTempBackupDir}, nil); err != nil { @@ -234,7 +341,11 @@ func (iops *InfrahubOps) backupNeo4jEnterprise(backupDir string, backupMetadata if output, err := iops.Exec( "database", - []string{"neo4j-admin", "database", "backup", "--expand-commands", "--include-metadata=" + backupMetadata, "--to-path=/tmp/infrahubops", iops.config.Neo4jDatabase}, + neo4jCaptureCommand(neo4jCaptureRequest{ + Database: iops.config.Neo4jDatabase, + BackupMetadata: backupMetadata, + ToPath: neo4jTempBackupDir, + }), nil, ); err != nil { return fmt.Errorf("failed to backup neo4j: %w\nOutput: %v", err, output) @@ -374,6 +485,19 @@ func (iops *InfrahubOps) backupNeo4jCommunity(backupDir string) (retErr error) { } func (iops *InfrahubOps) restoreNeo4j(workDir, neo4jEdition string, restoreMigrateFormat bool) error { + // Before the copy into the container, because for a database outside the + // deployment there is no container to copy into and nothing to copy: the + // server fetches the artifact itself from where the pre-flight staged it. + // A nil restore is an in-deployment database and takes the path below + // unchanged (FR-015). + restore, err := iops.externalRestoreFor(serviceNeo4j) + if err != nil { + return err + } + if restore != nil { + return iops.restoreNeo4jExternal(restore) + } + backupPath := filepath.Join(workDir, "backup", "database") // Clear the destination before copying into it, and abort if it cannot be cleared. @@ -422,7 +546,11 @@ func (iops *InfrahubOps) restoreNeo4jEnterprise(restoreMigrateFormat bool) error opts := iops.getNeo4jExecOptions() // Check if Neo4j is running in cluster mode - if iops.isNeo4jCluster() { + clustered, err := iops.isNeo4jCluster() + if err != nil { + return err + } + if clustered { return iops.restoreNeo4jCluster(opts) } @@ -845,6 +973,24 @@ func (iops *InfrahubOps) restoreNeo4jCommunity(restoreMigrateFormat bool) (retEr // directly from the provided reader into `neo4j-admin database load --from-stdin`. // This avoids copying dump files to a temporary directory on the container. func (iops *InfrahubOps) restoreNeo4jCommunityStream(reader io.ReadCloser, restoreMigrateFormat bool) (retErr error) { + // The Plakar paths reach this without going through restoreNeo4j, so the + // dispatch that sends a database outside the deployment to the seed path is + // not on this route. Everything below suspends the database's own process + // and writes into its store, which needs a container this database does not + // have. + // + // Both Plakar entry points now refuse this at their gate, before anything + // is stopped, so this is the backstop rather than the refusal — it is the + // last place the guarantee can be held, and a path added later could reach + // it without passing a gate. See refuseExternalCommunityDumpRestore. + restore, err := iops.externalRestoreFor(serviceNeo4j) + if err != nil { + return err + } + if restore != nil { + return refuseExternalDumpRestoreAfterQuiescing(restore.Endpoint) + } + logrus.Info("Restoring Neo4j database (Community Edition streamed load)...") pidStr, err := iops.readNeo4jPID() @@ -934,8 +1080,30 @@ func (iops *InfrahubOps) getNeo4jExecOptions() *ExecOptions { return &ExecOptions{User: "neo4j"} } -// isNeo4jCluster checks if Neo4j is running in cluster mode by counting servers +// redactDatabase replaces every attribute value in the database with a random +// UUID. It runs `cypher-shell` in the `database` container, which is what makes +// it an in-deployment operation: it modifies the source database in place, +// before the capture reads it. +// +// A database that lives outside the deployment has no such container, and the +// exec used to be attempted anyway — producing "no pods found for service +// database in namespace …", which is the exact misdiagnosis FR-008's refusal +// exists to remove. So the refusal is stated here too, in the terms an operator +// can act on: nothing is missing from the deployment, and this build has no way +// to redact a database it does not host. func (iops *InfrahubOps) redactDatabase() error { + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + return err + } + if source != nil { + return fmt.Errorf( + "cannot redact %s: redaction rewrites the database in place through the %s container, and there is none here because the database is managed outside this deployment. "+ + "Nothing is missing from the deployment. Take the backup without --redact, or redact the data at its source. "+ + "No Infrahub service was stopped and no data was changed", + source.Endpoint.endpointTarget(), serviceNeo4j) + } + logrus.Warn("Redacting attribute values in the database. This operation is destructive and irreversible!") query := `MATCH (av:AttributeValue) WITH av.value AS av_value, collect(av) AS av_verts WITH av_value, av_verts, randomUUID() as new_value CALL (av_value, new_value, av_verts) { UNWIND av_verts AS av SET av.value = new_value } IN TRANSACTIONS` @@ -955,7 +1123,28 @@ func (iops *InfrahubOps) redactDatabase() error { return nil } -func (iops *InfrahubOps) isNeo4jCluster() bool { +// isNeo4jCluster reports whether Neo4j runs in cluster mode. +// +// For a database inside the deployment it counts the servers the system +// database reports, and a query that fails is read as "not clustered" — the +// answer only selects between two restore procedures, and degrading is what +// this has always done. +// +// For one outside it, the count was already taken: the probe enumerated the +// members before anything was stopped, and asking again would mean exec'ing +// `cypher-shell` in a `database` container that is not there — the same +// misdiagnosis redactDatabase carried. Only the location failing is fatal here, +// because that is the one thing this cannot degrade around: it decides which of +// the two answers is even being computed. +func (iops *InfrahubOps) isNeo4jCluster() (bool, error) { + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + return false, err + } + if source != nil { + return len(source.Facts.Members) > 1, nil + } + output, err := iops.Exec("database", []string{ "cypher-shell", "-u", iops.config.Neo4jUsername, @@ -965,13 +1154,615 @@ func (iops *InfrahubOps) isNeo4jCluster() bool { "SHOW SERVERS YIELD * RETURN count(*) as serverCount", }, nil) if err != nil { - return false // Assume not clustered if query fails + return false, nil // Assume not clustered if query fails } // Parse server count - if > 1, it's a cluster lines := strings.Split(strings.TrimSpace(output), "\n") if len(lines) >= 2 { count, _ := strconv.Atoi(strings.TrimSpace(lines[len(lines)-1])) - return count > 1 + + return count > 1, nil + } + + return false, nil +} + +// --------------------------------------------------------------------------- +// Capturing a Neo4j database that lives outside the deployment +// --------------------------------------------------------------------------- + +// The online backup command is a client: the vendor's own documented topology +// is to run it "from a server on the same network as the database, but that is +// not part of the cluster" (research R1). The transient workload is that +// machine, so the capture below is the same command the in-container path runs, +// with the one addition that turns it from a local operation into a remote one — +// the ordered --from list of members to try. +// +// Everything else about the two paths is deliberately identical, including the +// order the flags are written in and the name of the directory the artifact +// lands in, because an artifact taken from an external database has to be +// restorable into an internal deployment and vice versa (FR-015, and the +// artifact contract). neo4jCaptureCommand is what makes that a shared fact +// rather than two implementations that agree today. + +const ( + // externalNeo4jCaptureName is the directory the artifact lands in, and it + // is the internal path's basename rather than a name of its own: the + // archive assembled from it is a tar of this directory, so a different name + // here would produce an archive whose entries a restore does not recognise. + externalNeo4jCaptureName = "infrahubops" + + // externalNeo4jCaptureParent is the scratch volume the capture directory + // sits in — the volume FR-022 sizes — rather than the node's own disk, + // which a large database filling would take neighbouring workloads with it. + externalNeo4jCaptureParent = transientScratchPath + + externalNeo4jCaptureDir = externalNeo4jCaptureParent + "/" + externalNeo4jCaptureName + + // externalNeo4jStagingDir is the --temp-path the command stages the store + // and transaction logs into before producing the artifact beside them. It + // is named explicitly because the vendor's default is the working + // directory, and "the temporary directory must therefore have enough free + // space to hold the entire backup" — so leaving it implicit puts the + // staging copy somewhere this run has not sized (research R5). + externalNeo4jStagingDir = transientScratchPath + "/staging" +) + +// neo4jCaptureRequest is everything the online backup command needs. It exists +// so the argv is built once for both the internal and the external path. +type neo4jCaptureRequest struct { + // Database is the database to capture, which names the artifact. + Database string + + // BackupMetadata is the --include-metadata value. + BackupMetadata string + + // ToPath is the directory the artifact is placed in. + ToPath string + + // TempPath is the staging directory. Empty leaves the flag off, which is + // what the in-container path does — it stages inside the database's own + // container, where the vendor's default is already the right place. + TempPath string + + // From is the ordered member list. Empty leaves the flag off entirely, + // which is what makes the command an in-container capture of the local + // store; a member list makes the same command a remote one (FR-005). + From []HostPort + + // Uncompressed emits --compress=false, which the streaming path asks for + // so the artifact deduplicates. + Uncompressed bool +} + +// neo4jCaptureCommand builds the online backup argv. +// +// The flag order is the order the in-container path has always written, and each +// addition is appended only when its field is set, so a request with no member +// list, no staging path and no compression override produces the exact argv that +// path produced before this feature existed. That equality is asserted by test +// rather than left as an intention (FR-015). +// +// `--expand-commands` is emitted only for the in-container capture, and the +// reason is a property of the vendor's own tooling rather than a preference. +// Given that flag, `neo4j-admin` refuses to read any configuration file whose +// permissions are looser than 0640 — and it reads both `neo4j.conf` and +// `neo4j-admin.conf`, so tightening one is not enough. The vendor image ships +// them world-writable — `neo4j:5-enterprise` has `/var/lib/neo4j/conf/neo4j.conf` +// at 0777 — and the infrahub-backup image the transient workload runs copies +// that conf directory from `neo4j:2025.10.1-enterprise` unchanged, so +// `/opt/neo4j/conf/neo4j.conf` is 0777 there too. Passing the flag aborts the +// capture *before the database is contacted*, reporting a file permission the +// operator never set, in an image the operator did not build. +// +// Nothing is lost by omitting it. The flag exists so that a deployment's own +// configuration may hold command-valued settings for `neo4j-admin` to expand, +// and the transient workload has no such configuration — its conf is the stock +// one, and it holds only the credentials it was given and the endpoint it was +// told to read. The in-container path does run against the deployment's config, +// so it keeps the flag, which is also what keeps its argv byte-identical +// (FR-015). +// +// Verified against real servers on 2026-09-03: with the flag, `neo4j:5-enterprise` +// fails on config permissions; without it, the same argv completes a full remote +// capture from a live backup listener and writes +// `neo4j-.backup`. +func neo4jCaptureCommand(req neo4jCaptureRequest) []string { + command := []string{ + "neo4j-admin", "database", "backup", + } + + if len(req.From) == 0 { + command = append(command, "--expand-commands") + } + + command = append(command, "--include-metadata="+req.BackupMetadata) + + if req.Uncompressed { + command = append(command, "--compress=false") + } + if len(req.From) > 0 { + command = append(command, "--from="+renderHostPorts(req.From)) + } + if req.TempPath != "" { + command = append(command, "--temp-path="+req.TempPath) + } + + return append(command, "--to-path="+req.ToPath, req.Database) +} + +// externalNeo4jCaptureRequest is the request for a capture from outside the +// deployment. +// +// It carries no credential, and there is nothing missing: the backup service is +// reached over its own protocol on its own port, and where the probe statements +// need to authenticate they read NEO4J_USERNAME and NEO4J_PASSWORD from the +// environment the transient workload's secret provides. So no path here puts a +// password on a command line (FR-014). +func externalNeo4jCaptureRequest(capture *externalCapture, backupMetadata string, uncompressed bool) neo4jCaptureRequest { + return neo4jCaptureRequest{ + Database: capture.Endpoint.Database, + BackupMetadata: backupMetadata, + ToPath: externalNeo4jCaptureDir, + TempPath: externalNeo4jStagingDir, + From: capture.captureTargets(), + Uncompressed: uncompressed, + } +} + +// errExternalCommunityCapture is the FR-008 refusal, as a value callers and +// tests can identify rather than a message they have to match. +var errExternalCommunityCapture = errors.New("the Community Edition backup mechanism requires direct access to the database's own storage") + +// externalCommunityCaptureRefusal refuses to capture an external Community +// database, saying why the mechanism is unavailable. +// +// The wording matters more than usual here. Before this feature the same +// situation produced "service not found", and a customer reasonably concluded +// their deployment was broken and went looking for a missing container — so the +// message states that the mechanism needs the database's own storage, that the +// remote alternative is an Enterprise feature, and explicitly that nothing is +// missing from the deployment (FR-008, and the failure-message contract). +func externalCommunityCaptureRefusal(endpoint *DatabaseEndpoint) error { + return fmt.Errorf( + "cannot back up %s: %w, because it is an offline dump of the store files, and this run has no access to the storage of a server it does not host. "+ + "The online backup that does work remotely is a Neo4j Enterprise Edition feature. "+ + "Nothing is missing from the deployment — the %s service has no container here because the database is managed outside it. "+ + "No Infrahub service was stopped", + endpoint.endpointTarget(), errExternalCommunityCapture, serviceNeo4j, + ) +} + +// refuseExternalCommunityCapture refuses an external capture whose server +// reported an edition the remote mechanism cannot serve (FR-008). +// +// It is called from the gate, on the edition the probe read over Bolt, which is +// the only place the answer is both known and still free: one exec into a probe +// pod has been spent, and no Infrahub service has been stopped, no abort window +// announced, and no capture workload built. Deciding later would mean deciding +// after detectNeo4jEdition had already turned Community into +// RequiresOfflineCapture and scaled six services to zero for a dump this run was +// never going to be able to take. +// +// An edition the server did not report is not read as Community: the same +// asymmetry Neo4jEditionInfo documents applies here, and guessing the +// destructive branch from a probe that did not answer is what Principle II +// forbids. requireDeterminedEdition is what stops such a run, one step later. +func refuseExternalCommunityCapture(endpoint *DatabaseEndpoint, edition string) error { + if !isCommunityEdition(edition) { + return nil + } + + return externalCommunityCaptureRefusal(endpoint) +} + +// --------------------------------------------------------------------------- +// Deciding whether a capture captured everything (FR-012, research R6) +// --------------------------------------------------------------------------- + +// The backup command's exit codes conflate two outcomes. Code 1 is "Backup +// failed, or succeeded but encountered problems such as some servers being +// uncontactable", and for several databases "One or several backups failed, or +// succeeded with problems". There is no code that separates them, and the +// vendor documents no machine-readable marker in the output either — the exit +// code table says only "See logs for more details". +// +// So completeness is established from two things the run can observe for itself: +// whether the command reported success, and whether the artifact it was asked +// for is actually there. Both are required. A non-zero exit is never complete, +// and a zero exit that produced no artifact is not complete either, which is the +// case a signal read from exit status alone would have called a success. +// +// What the output is used for is telling the operator which of the two outcomes +// they hit. Where the output names a server it could not reach, that line is +// quoted in the failure; where it does not, the failure says so rather than +// implying an outright failure it did not establish. The markers below therefore +// never decide anything — inventing a completeness signal the command does not +// emit is the failure mode this whole section exists to avoid. + +// neo4jCaptureVerdict is what a capture established about itself. +type neo4jCaptureVerdict struct { + // Complete is true only for a capture that reported success and left the + // artifact it was asked for. + Complete bool + + // ArtifactProduced records whether the artifact is there, which is what + // separates "succeeded with problems" from "failed outright" on the + // identical exit status they share. + ArtifactProduced bool + + // Detail is the operator-facing account of an incomplete capture. + Detail string +} + +// neo4jUncontactableMarkers are phrasings that indicate the command could not +// reach a member. They corroborate a verdict already reached from the exit +// status and the artifact; none of them is the verdict, because the vendor +// documents no output contract for this and a phrasing that changes between +// releases must not be able to turn an incomplete capture into a complete one. +// +// None of them is attribution either. Every one of these phrasings is also how +// a cluster's own control plane reports that *it* is unreachable, so a marker +// counts only on a line that names an endpoint the run supplied — see +// uncontactableEvidence. +var neo4jUncontactableMarkers = []string{ + "uncontactable", + "could not be contacted", + "unable to connect", + "connection refused", + "did not respond", + "not responding", + "unreachable", + "failed to connect", +} + +// uncontactableEvidence returns the first line that names one of the endpoints +// the capture was asked to try and says it could not be reached, or the empty +// string when nothing does. +// +// It reads the command's stdout *and* the error, because the two carry +// different halves of what the command said: bounded execution returns stdout +// alone, and a failed command's stderr is folded into its error by withStderr. +// Reading only stdout — which is what an earlier version of this did, back when +// the two streams were merged — would look for the evidence in the one place a +// failing neo4j-admin does not write it. +// +// The endpoints are what make reading the error safe, and this is why they are +// a parameter. kubectl writes its *own* stderr into the same stream the remote +// command's goes to, and its transport failures are worded exactly like a +// database that will not answer: an API server that has stopped answering +// produces "Unable to connect to the server: dial tcp …: connection refused", +// which matches two of the markers below. So the whole of an API-server outage +// was reported to the operator as a named Neo4j endpoint the capture could not +// reach — a diagnosis pointing at the wrong system entirely, on a run that +// failed for a reason the cluster could have told them. +// +// Attribution is therefore positive rather than a list of transport phrasings +// to ignore: a line is evidence about this database only if it names a host the +// run supplied. On the host, not the whole address, because the supplied +// endpoint carries the backup listener's port and the server names its own. A +// genuine failure whose wording omits the host is not read as evidence, and +// that is the safe direction: the markers only ever corroborate a verdict +// already reached from the exit status and the artifact, so losing one costs +// the operator the endpoint's name in the message, never a capture reported +// complete when it was not. +func uncontactableEvidence(output string, runErr error, targets []HostPort) string { + for _, line := range nonEmptyLines(output) { + if namesAnUncontactableEndpoint(line, targets) { + return strings.TrimSpace(line) + } + } + + if runErr == nil { + return "" + } + + for _, line := range nonEmptyLines(runErr.Error()) { + if namesAnUncontactableEndpoint(line, targets) { + return strings.TrimSpace(line) + } + } + + return "" +} + +// namesAnUncontactableEndpoint reports whether one line says a supplied +// endpoint could not be reached. +func namesAnUncontactableEndpoint(line string, targets []HostPort) bool { + lowered := strings.ToLower(strings.TrimSpace(line)) + + named := false + for _, target := range targets { + host := strings.ToLower(strings.TrimSpace(target.Host)) + if host != "" && strings.Contains(lowered, host) { + named = true + + break + } + } + if !named { + return false } + + for _, marker := range neo4jUncontactableMarkers { + if strings.Contains(lowered, marker) { + return true + } + } + + return false +} + +// neo4jArtifactProduced reports whether a listing of the capture directory shows +// an artifact for the database that was asked for. +// +// The artifact is named `-.backup`, and this reads all +// three parts. Matching the bare name as a prefix — which is what this did — +// lets a sibling database's `2-….backup` satisfy the check, so a run +// that captured nothing of the database it was asked for records a complete +// capture. Requiring the separator fixes that case but not `-staging`, +// whose own artifact also begins `-`; so what follows the separator +// must additionally begin a timestamp rather than more name. +// +// Both of the assumptions here — the `.backup` suffix and a timestamp starting +// with its year — fail in the same direction if the vendor ever changes the +// name: the artifact is not recognised, the capture is reported incomplete, and +// the run fails. That is the direction Principle II asks for. The opposite +// reading is the one this whole section exists to prevent: a complete, loadable +// artifact of something else, sitting where the reader looks, reported as the +// backup that was asked for. +func neo4jArtifactProduced(database, listing string) bool { + prefix := database + "-" + + for _, entry := range nonEmptyLines(listing) { + if !strings.HasPrefix(entry, prefix) || !strings.HasSuffix(entry, ".backup") { + continue + } + + if remainder := entry[len(prefix):]; remainder != "" && remainder[0] >= '0' && remainder[0] <= '9' { + return true + } + } + return false } + +// classifyNeo4jCapture decides whether a capture captured everything it was +// asked for, from what the operation reported rather than from its exit status +// alone (FR-012). +// +// targets are the endpoints the capture was asked to try, and they are what +// make the "could not reach an endpoint" wording attributable to this database +// rather than to the cluster the command ran through (uncontactableEvidence). +func classifyNeo4jCapture(runErr error, output string, artifactProduced bool, targets []HostPort) neo4jCaptureVerdict { + verdict := neo4jCaptureVerdict{ArtifactProduced: artifactProduced} + evidence := uncontactableEvidence(output, runErr, targets) + + switch { + case runErr == nil && artifactProduced: + verdict.Complete = true + + // Reported, not acted on: a successful capture that also mentions an + // unreachable member is a capture the vendor called successful, and + // overriding that on a phrase match would refuse work the server did. + if evidence != "" { + logrus.Warnf("The capture reported success but its output mentions an endpoint it could not reach: %s", evidence) + } + + case runErr == nil && !artifactProduced: + verdict.Detail = "the operation reported success but left no artifact in the capture directory, so there is nothing to record as a backup" + if line := firstOutputLine(output); line != "" { + verdict.Detail += " (the command said: " + line + ")" + } + + case artifactProduced && evidence != "": + verdict.Detail = "the operation produced an artifact and then reported a problem, and its output names an endpoint it could not reach: " + evidence + ". A capture that reached only some of the endpoints supplied is not a complete capture, and its exit status is the same as an outright failure's, so it is treated as incomplete" + + case artifactProduced: + verdict.Detail = "the operation produced an artifact and then reported a problem. Its exit status means either that the capture failed or that it succeeded against only some of the endpoints supplied, and no exit status distinguishes the two, so it is treated as incomplete. The server's own logs say which" + + case evidence != "": + verdict.Detail = "the operation produced no artifact and its output names an endpoint it could not reach: " + evidence + + default: + verdict.Detail = "the operation produced no artifact" + } + + return verdict +} + +// firstOutputLine is what the command said, for the one verdict that has no +// endpoint to name: a success that produced nothing should not be reported as a +// bare assertion when the command's own first line is there to quote. +func firstOutputLine(output string) string { + lines := nonEmptyLines(output) + if len(lines) == 0 { + return "" + } + + return lines[0] +} + +// --------------------------------------------------------------------------- +// The external capture itself +// --------------------------------------------------------------------------- + +// prepareExternalCaptureDirs clears and creates the capture and staging +// directories in the transient workload's scratch volume. +func (iops *InfrahubOps) prepareExternalCaptureDirs(capture *externalCapture) error { + command := []string{"sh", "-c", fmt.Sprintf("rm -rf %s %s && mkdir -p %s %s", + externalNeo4jCaptureDir, externalNeo4jStagingDir, externalNeo4jCaptureDir, externalNeo4jStagingDir)} + + if _, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "preparing the capture directory", + transientWorkloadTarget(serviceNeo4j), serviceNeo4j, command, capture.execOptions(nil), + ); err != nil { + return fmt.Errorf("failed to prepare the external Neo4j capture directory for %s: %w", capture.Endpoint.endpointTarget(), err) + } + + return nil +} + +// cleanupExternalCaptureDirs removes what the capture staged. It is best-effort: +// the workload is about to be given back and its scratch volume goes with it, so +// a failure here costs nothing (FR-011, FR-023). +func (iops *InfrahubOps) cleanupExternalCaptureDirs(capture *externalCapture) { + iops.removeTransientScratch(serviceNeo4j, "the capture directory", capture.execOptions(nil), externalNeo4jCaptureDir, externalNeo4jStagingDir) +} + +// externalCaptureProducedArtifact asks the capture directory whether the +// artifact is there, which is the observation that separates a partial capture +// from a failed one. +// +// A listing that cannot be read reports "no artifact", which is the safe +// direction: it makes the run fail as an incomplete capture rather than record a +// completeness it did not observe. +func (iops *InfrahubOps) externalCaptureProducedArtifact(capture *externalCapture) bool { + command := []string{"sh", "-c", "ls -1 " + externalNeo4jCaptureDir + " 2>/dev/null"} + + output, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "listing the capture directory", + transientWorkloadTarget(serviceNeo4j), serviceNeo4j, command, capture.execOptions(nil), + ) + if err != nil { + logrus.Warnf("Could not list the external Neo4j capture directory, so the capture is treated as having produced nothing: %v", err) + + return false + } + + return neo4jArtifactProduced(capture.Endpoint.Database, output) +} + +// runExternalNeo4jCapture runs the online backup against the resolved members +// and fails the run for anything short of a complete capture (FR-012). +// +// It is bounded by the operator's --external-db-timeout rather than run through +// the unbounded Exec (FR-025). The internal path is deliberately left unbounded: +// it is the path every existing deployment takes, and a bound that has never +// been there is a behaviour change to a working flow (FR-015). +func (iops *InfrahubOps) runExternalNeo4jCapture(capture *externalCapture, backupMetadata string, uncompressed bool) error { + command := neo4jCaptureCommand(externalNeo4jCaptureRequest(capture, backupMetadata, uncompressed)) + + output, runErr := iops.execBoundedAgainst( + capture.Bound, "the online backup", capture.Endpoint.endpointTarget(), serviceNeo4j, command, capture.execOptions(nil), + ) + + verdict := classifyNeo4jCapture(runErr, output, iops.externalCaptureProducedArtifact(capture), capture.captureTargets()) + if verdict.Complete { + return nil + } + + // Recorded before the error is returned, so that FR-012's other half — no + // artefact survives this run — is decided from the verdict itself rather + // than from an error value a caller between here and the archive could + // replace. See InfrahubOps.discardIncompleteCapture. + iops.recordIncompleteCapture(serviceNeo4j, verdict.Detail) + + if runErr != nil { + return fmt.Errorf("the Neo4j capture from %s is incomplete and will not be kept: %s: %w", capture.Endpoint.endpointTarget(), verdict.Detail, runErr) + } + + return fmt.Errorf("the Neo4j capture from %s is incomplete and will not be kept: %s", capture.Endpoint.endpointTarget(), verdict.Detail) +} + +// reportExternalCapture announces the capture about to run: the members it will +// try, in order, and the role each was observed in at this moment (FR-005). +// +// It states no member as the one that will serve the capture, because --from +// tries its list in order and reports no such thing. What the operator gets +// instead is enough to see that a follower could serve it, which is the fact +// that bears on the recovery point. +func (capture *externalCapture) reportExternalCapture(action string) { + targets := capture.captureTargets() + + logrus.Infof("%s the external Neo4j database (Enterprise Edition online backup, endpoints tried in order: %s; roles observed now: %s; the member that serves the capture is not reported by the backup command)", + action, renderHostPorts(targets), describeCaptureRoles(targets, capture.observedRoles())) +} + +// backupNeo4jEnterpriseExternal is the directory-copy capture from a database +// outside the deployment. It is the internal path's shape with the container +// replaced by the transient workload, which is why the destination and the +// artifact layout are unchanged. +func (iops *InfrahubOps) backupNeo4jEnterpriseExternal(backupDir, backupMetadata string) error { + capture, err := iops.openExternalCapture(serviceNeo4j) + if err != nil { + return err + } + defer capture.Release() + + capture.reportExternalCapture("Backing up") + + if err := iops.prepareExternalCaptureDirs(capture); err != nil { + return err + } + defer iops.cleanupExternalCaptureDirs(capture) + + if err := iops.runExternalNeo4jCapture(capture, backupMetadata, false); err != nil { + return err + } + + if err := iops.copyFromTransientWorkload(capture.Pod, externalNeo4jCaptureDir, filepath.Join(backupDir, "database")); err != nil { + return fmt.Errorf("failed to copy the external database backup: %w", err) + } + + logrus.Info("Neo4j backup completed") + + return nil +} + +// streamNeo4jEnterpriseExternal is the streaming capture from a database outside +// the deployment. +// +// The tar is taken with the same -C and the same member name the internal path +// uses, so the archive is byte-compatible with an internal one and restores by +// the same code (FR-015). The transient workload is given back when the stream +// is closed, and on every failure path before that (FR-011). +func (iops *InfrahubOps) streamNeo4jEnterpriseExternal(backupMetadata string) (io.ReadCloser, error) { + capture, err := iops.openExternalCapture(serviceNeo4j) + if err != nil { + return nil, err + } + + capture.reportExternalCapture("Streaming") + + if err := iops.prepareExternalCaptureDirs(capture); err != nil { + capture.Release() + + return nil, err + } + + // Once the directories exist, giving the workload back means clearing them + // first, on every path out of here — including the stream's own close. + teardown := func() { + iops.cleanupExternalCaptureDirs(capture) + capture.Release() + } + + if err := iops.runExternalNeo4jCapture(capture, backupMetadata, true); err != nil { + teardown() + + return nil, err + } + + stdout, wait, err := iops.ExecStreamPipe(serviceNeo4j, + []string{"tar", "cf", "-", "-C", externalNeo4jCaptureParent, externalNeo4jCaptureName}, capture.execOptions(nil)) + if err != nil { + teardown() + + return nil, fmt.Errorf("failed to start the external neo4j enterprise stream: %w", err) + } + + // The capture workload's deadline budgets a whole operation bound for the + // artifact to leave the pod (externalDBCaptureFullBoundCalls), and on this + // path the stream is how it leaves. Bounding the stream by the same amount + // keeps the tool's own deadline ahead of the cluster's, so a slow transfer + // that is still progressing is stopped and reported by the tool rather + // than cut short by the pod being removed. + return &execReadCloser{ + reader: stdout, + wait: wait, + idleTimeout: defaultStreamIdleTimeout, + deadline: time.Now().Add(capture.Bound), + cleanup: teardown, + }, nil +} diff --git a/src/internal/app/backup_neo4j_test.go b/src/internal/app/backup_neo4j_test.go index b2e6f343..5e4ff7c3 100644 --- a/src/internal/app/backup_neo4j_test.go +++ b/src/internal/app/backup_neo4j_test.go @@ -2,11 +2,13 @@ package app import ( "errors" + "io" "os" "path/filepath" "slices" "strings" "testing" + "time" ) // neo4jScriptedBackend answers the queries the Neo4j backup and restore paths make against @@ -245,3 +247,206 @@ func TestRestoreNeo4jFailsWhenTheDestinationCannotBeCleared(t *testing.T) { t.Errorf("a restore ran despite the destination not being cleared; calls: %v", backend.calls) } } + +// TestRedactDatabaseRefusesAnExternalDatabase is F6. --redact rewrites the +// database in place through `cypher-shell` in the `database` container, and it +// ran unconditionally — so a run against an external database failed with "no +// pods found for service database in namespace …", which is the exact +// misdiagnosis FR-008's refusal was written to eliminate. +func TestRedactDatabaseRefusesAnExternalDatabase(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "neo4j.example", Port: 7687}}, + }, + }) + + err := iops.redactDatabase() + if err == nil { + t.Fatal("redactDatabase() = nil, want a refusal naming the database it cannot reach") + } + if !strings.Contains(err.Error(), "Nothing is missing from the deployment") { + t.Errorf("err = %v, want the message that stops an operator hunting a missing container", err) + } + if !strings.Contains(err.Error(), "no data was changed") { + t.Errorf("err = %v, want it to state that nothing was changed", err) + } + for _, call := range backend.execs { + if strings.HasPrefix(call, "cypher-shell") { + t.Errorf("redaction ran %q against a database with no container here", call) + } + } +} + +// TestIsNeo4jClusterReadsWhatTheProbeObserved is the other half of F6: the +// member count was already taken by the probe, before anything was stopped, so +// asking again means exec'ing `cypher-shell` in a container that is not there. +func TestIsNeo4jClusterReadsWhatTheProbeObserved(t *testing.T) { + external := func(members []observedMember) *InfrahubOps { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j, Location: EndpointLocationExternal}, + Facts: probedFacts{Members: members}, + }) + + return iops + } + + t.Run("several observed members is a cluster", func(t *testing.T) { + clustered, err := external([]observedMember{{Address: "a"}, {Address: "b"}}).isNeo4jCluster() + if err != nil { + t.Fatalf("isNeo4jCluster failed: %v", err) + } + if !clustered { + t.Error("isNeo4jCluster() = false, want true for a database the probe saw two members of") + } + }) + + t.Run("one observed member is not", func(t *testing.T) { + iops := external([]observedMember{{Address: "a"}}) + clustered, err := iops.isNeo4jCluster() + if err != nil { + t.Fatalf("isNeo4jCluster failed: %v", err) + } + if clustered { + t.Error("isNeo4jCluster() = true, want false for a single-member database") + } + for _, call := range iops.backend.(*gatedBackend).execs { + if strings.HasPrefix(call, "cypher-shell") { + t.Errorf("the cluster check ran %q against a database with no container here", call) + } + } + }) + + t.Run("an internal database keeps degrading on a failed query", func(t *testing.T) { + backend := newGatedBackend() + backend.cypherErr = errors.New("Connection refused") + iops := newGatedOps(t, backend, BackendTarball) + clustered, err := iops.isNeo4jCluster() + if err != nil { + t.Fatalf("isNeo4jCluster failed: %v", err) + } + if clustered { + t.Error("isNeo4jCluster() = true, want the shipped assume-not-clustered reading of a failed query") + } + // The query has to have been issued and failed: without that, a false + // answer is the parse of a successful one, and the failure branch this + // subtest is named for goes unexercised. + queried := false + for _, call := range backend.execs { + if strings.Contains(call, "SHOW SERVERS") { + queried = true + } + } + if !queried { + t.Errorf("the cluster query was never issued against the internal database; execs = %v", backend.execs) + } + }) +} + +// A producer that dies part-way through ends its stdout like one that +// finished. kloset reads to EOF and discards Close's error, so the failure has +// to come out of Read, or a truncated stream is stored as a complete one. +func TestExecReadCloser_EOFFromAFailedCommandIsAReadError(t *testing.T) { + podRemoved := errors.New("command terminated with exit code 137") + waits := 0 + stream := &execReadCloser{ + reader: io.NopCloser(strings.NewReader("partial")), + wait: func() error { waits++; return podRemoved }, + idleTimeout: time.Second, + } + + data, err := io.ReadAll(stream) + if !errors.Is(err, podRemoved) { + t.Fatalf("ReadAll error = %v, want it to carry the command's failure", err) + } + if string(data) != "partial" { + t.Fatalf("ReadAll data = %q, want %q", data, "partial") + } + if closeErr := stream.Close(); !errors.Is(closeErr, podRemoved) { + t.Fatalf("Close error = %v, want the command's failure", closeErr) + } + if waits != 1 { + t.Fatalf("wait called %d times, want 1", waits) + } +} + +func TestExecReadCloser_EOFFromASucceededCommandIsEOF(t *testing.T) { + waits := 0 + cleaned := false + stream := &execReadCloser{ + reader: io.NopCloser(strings.NewReader("complete")), + wait: func() error { waits++; return nil }, + cleanup: func() { cleaned = true }, + idleTimeout: time.Second, + deadline: time.Now().Add(time.Minute), + } + + data, err := io.ReadAll(stream) + if err != nil || string(data) != "complete" { + t.Fatalf("ReadAll = %q, %v; want %q, nil", data, err, "complete") + } + if err := stream.Close(); err != nil { + t.Fatalf("Close error = %v, want nil", err) + } + if waits != 1 || !cleaned { + t.Fatalf("waits = %d, cleaned = %v; want 1, true", waits, cleaned) + } +} + +// A transfer that keeps making progress never trips the idle timeout. The +// deadline is what ends it, so that the tool and not the transient pod's +// activeDeadlineSeconds decides when it is over. +func TestExecReadCloser_DeadlineEndsASlowButProgressingStream(t *testing.T) { + pr, pw := io.Pipe() + go func() { + for { + if _, err := pw.Write([]byte("x")); err != nil { + return + } + time.Sleep(10 * time.Millisecond) + } + }() + + stream := &execReadCloser{ + reader: pr, + wait: func() error { return nil }, + idleTimeout: time.Minute, + deadline: time.Now().Add(150 * time.Millisecond), + } + + data, err := io.ReadAll(stream) + if !errors.Is(err, errStreamDeadline) { + t.Fatalf("ReadAll error = %v, want errStreamDeadline", err) + } + if len(data) == 0 { + t.Fatal("the stream was making progress, so some data should have been read before the deadline") + } + if _, err := stream.Read(make([]byte, 1)); !errors.Is(err, errStreamDeadline) { + t.Fatalf("Read after the deadline = %v, want errStreamDeadline", err) + } + _ = stream.Close() +} + +func TestExecReadCloser_IdleTimeoutStillApplies(t *testing.T) { + pr, pw := io.Pipe() + defer pw.Close() + + stream := &execReadCloser{ + reader: pr, + wait: func() error { return nil }, + idleTimeout: 50 * time.Millisecond, + deadline: time.Now().Add(time.Minute), + } + + if _, err := stream.Read(make([]byte, 1)); !errors.Is(err, errStreamIdleTimeout) { + t.Fatalf("Read error = %v, want errStreamIdleTimeout", err) + } + _ = stream.Close() +} diff --git a/src/internal/app/backup_retention_test.go b/src/internal/app/backup_retention_test.go index df2cd3d6..25be835d 100644 --- a/src/internal/app/backup_retention_test.go +++ b/src/internal/app/backup_retention_test.go @@ -56,6 +56,7 @@ type fakeS3Endpoint struct { uploadCode int listCode int uploaded []string + deleted []string listed int } @@ -84,6 +85,9 @@ func (e *fakeS3Endpoint) handle(w http.ResponseWriter, r *http.Request) { _, _ = io.Copy(io.Discard, r.Body) e.uploaded = append(e.uploaded, r.URL.Path) w.Header().Set("ETag", `"fake-etag"`) + case http.MethodDelete: + e.deleted = append(e.deleted, r.URL.Path) + w.WriteHeader(http.StatusNoContent) case http.MethodGet: e.listed++ if e.listCode != http.StatusOK { @@ -287,3 +291,179 @@ func TestCreateBackupReportsRetentionFailureAfterSuccess(t *testing.T) { t.Errorf("backup directory = %v, want the local leg pruned %v", got, want) } } + +// --------------------------------------------------------------------------- +// T045: an incomplete capture is not retained anywhere (FR-012) +// --------------------------------------------------------------------------- + +// retentionDecision is what the policy would do at one location, as the two name +// sets it splits a listing into. Comparing decisions rather than directories is +// what FR-012's verification asks for: the requirement is that an incomplete run +// leaves retention deciding exactly what it would have decided had the run +// failed before writing anything. +type retentionDecision struct { + keep []string + prune []string +} + +func decideRetention(t *testing.T, dir string, policy RetentionPolicy, now time.Time) retentionDecision { + t.Helper() + + refs, err := newLocalLocation(dir).List(t.Context()) + if err != nil { + t.Fatalf("listing %s = %v, want nil", dir, err) + } + + keep, prune := selectPrunable(refs, policy, now) + + decision := retentionDecision{keep: make([]string, 0, len(keep)), prune: make([]string, 0, len(prune))} + for _, ref := range keep { + decision.keep = append(decision.keep, ref.Name) + } + for _, ref := range prune { + decision.prune = append(decision.prune, ref.Name) + } + slices.Sort(decision.keep) + slices.Sort(decision.prune) + + return decision +} + +// TestIncompleteCaptureIsRetainedNowhere is FR-012's verification: fail one +// component, and the run fails leaving no artefact at any location and leaving +// retention's decisions identical to a run that failed before anything was +// written. +// +// The two failures are deliberately different in kind. The copy failure never +// produces an archive; the incomplete capture is the one that could — it is the +// verdict a backup command reaches after writing something, on the exit status +// it shares with an outright failure — and the point of the requirement is that +// the two are indistinguishable from retention's side afterwards. +func TestIncompleteCaptureIsRetainedNowhere(t *testing.T) { + now := time.Now() + + // The run that failed before writing anything: the reference decision. + failedEarly := newCreateRetentionFixture(t) + failedEarly.backend.copyFromErr = errors.New("no space left on device") + if _, err := failedEarly.run(t, false); err == nil { + t.Fatal("CreateBackup() = nil, want the failed database copy reported") + } + reference := decideRetention(t, failedEarly.dir, failedEarly.iops.config.Retention.Policy(), now) + + // The run whose capture reported something short of a complete capture. + // runExternalNeo4jCapture records exactly this before it returns its error; + // recording it here stands in for a cluster that could not be reached. + incomplete := newCreateRetentionFixture(t) + before := pruneDirNames(t, incomplete.dir) + incomplete.iops.recordIncompleteCapture(serviceNeo4j, + "the operation produced an artifact and then reported a problem") + + output, err := incomplete.run(t, false) + if err == nil { + t.Fatal("CreateBackup() = nil, want an incomplete capture to fail the run") + } + if !strings.Contains(err.Error(), "incomplete") { + t.Errorf("error = %q, want it to say the backup is incomplete", err) + } + + t.Run("no artefact is left behind", func(t *testing.T) { + if got := pruneDirNames(t, incomplete.dir); !slices.Equal(got, before) { + t.Errorf("backup directory after an incomplete capture = %v, want it untouched %v", got, before) + } + }) + + t.Run("retention never runs", func(t *testing.T) { + // An incomplete run must not prune. Retention ranks by recency, so an + // archive written and then removed would still have been the newest + // while the prune ran and would have evicted the real newest. + if strings.Contains(output, "Applying retention policy") { + t.Errorf("log output = %q, want no retention pass after an incomplete capture", output) + } + }) + + t.Run("retention decides exactly what it would have decided", func(t *testing.T) { + got := decideRetention(t, incomplete.dir, incomplete.iops.config.Retention.Policy(), now) + if !slices.Equal(got.keep, reference.keep) || !slices.Equal(got.prune, reference.prune) { + t.Errorf("retention after an incomplete capture keeps %v and prunes %v;\nafter a run that failed before writing anything it keeps %v and prunes %v", + got.keep, got.prune, reference.keep, reference.prune) + } + }) +} + +// TestIncompleteCaptureIsRemovedFromEveryLocationItReached covers the half the +// run above cannot reach: an artefact that had already been written locally and +// uploaded before the capture was judged incomplete. +// +// The ordinary path fails at the gate, before an archive exists, so nothing is +// there to remove — which is why the removal is written as an invariant over +// wherever the archive got to rather than as a step of one code path. This is +// what exercises it, and it is the S3 leg that matters: an object left in the +// bucket is one a later `restore --latest --s3` can select, and neither +// retention nor a released reader would know not to. +func TestIncompleteCaptureIsRemovedFromEveryLocationItReached(t *testing.T) { + endpoint := newFakeS3Endpoint(t) + + archive := backupNameAt(time.Now()) + encrypted := archive + ".enc" + dir := seedPruneDir(t, archive, encrypted) + + cfg := createRetentionConfig(dir, RetentionConfig{Days: 7}) + cfg.S3.Endpoint = endpoint.server.URL + iops := &InfrahubOps{config: cfg, executor: NewCommandExecutor()} + iops.recordIncompleteCapture(serviceNeo4j, "some servers were uncontactable") + + // The archive reached the backup directory under both names — encrypting + // renames it, and removing the plaintext afterwards is only a warning — and + // the encrypted one reached the bucket. + reach := artifactReach{s3Name: encrypted} + reach.reachedLocally(archive) + reach.reachedLocally(encrypted) + + err := iops.discardIncompleteCapture(reach) + if err == nil { + t.Fatal("discardIncompleteCapture() = nil, want the incomplete backup reported as a failure") + } + if !strings.Contains(err.Error(), "some servers were uncontactable") { + t.Errorf("error = %q, want it to carry the capture's own account", err) + } + if strings.Contains(err.Error(), "may still be present") { + t.Errorf("error = %q, want no leftover reported: every location was cleared", err) + } + + t.Run("both local names are gone", func(t *testing.T) { + want := slices.Clone(pruneDecoys) + slices.Sort(want) + if got := pruneDirNames(t, dir); !slices.Equal(got, want) { + t.Errorf("backup directory = %v, want only the non-archive decoys %v", got, want) + } + }) + + t.Run("the object is deleted under the key the upload built", func(t *testing.T) { + want := []string{"/infrahub-backups/prod/" + encrypted} + if !slices.Equal(endpoint.deleted, want) { + t.Errorf("S3 deletions = %v, want %v: a base name where the full key belongs succeeds silently against real S3", + endpoint.deleted, want) + } + }) + + t.Run("a location the archive never reached is left alone", func(t *testing.T) { + // Configuration alone never authorises a deletion, which is the rule + // retentionLegsForCreate follows for pruning and the reason an S3 + // bucket that this run never uploaded to is not touched. + local := newFakeS3Endpoint(t) + localOnly := createRetentionConfig(seedPruneDir(t, archive), RetentionConfig{Days: 7}) + localOnly.S3.Endpoint = local.server.URL + + iops := &InfrahubOps{config: localOnly, executor: NewCommandExecutor()} + iops.recordIncompleteCapture(serviceNeo4j, "some servers were uncontactable") + + notUploaded := artifactReach{} + notUploaded.reachedLocally(archive) + if err := iops.discardIncompleteCapture(notUploaded); err == nil { + t.Fatal("discardIncompleteCapture() = nil, want the failure reported") + } + if len(local.deleted) != 0 { + t.Errorf("S3 deletions = %v, want none: this run never uploaded there", local.deleted) + } + }) +} diff --git a/src/internal/app/backup_taskmanager.go b/src/internal/app/backup_taskmanager.go index d369a74f..9e7d9e63 100644 --- a/src/internal/app/backup_taskmanager.go +++ b/src/internal/app/backup_taskmanager.go @@ -4,22 +4,140 @@ import ( "fmt" "io" "path/filepath" + "strconv" "strings" + "time" "github.com/sirupsen/logrus" ) +// A PostgreSQL dump is a client operation either way, so the external path +// differs from the internal one in exactly three places: the host it connects +// to, where the credentials come from, and whether it verifies the server's +// certificate. +// +// The host is the reason the internal command cannot simply be reused — +// `-h localhost` inside the transient workload names the workload, not the +// database. The credentials move because ExecOptions.Env becomes an +// `env PGPASSWORD=…` prefix on the exec command line, which is readable by +// anyone who can watch the deployment's processes or its API-server audit log +// (FR-014, research R7); the workload's own secret supplies PGUSER and PGPASSWORD +// as real environment instead, so the external command carries neither -U nor a +// password. And the certificate is verified unless the operator opted out +// (FR-029). + +// postgresDumpRequest is everything the dump command needs. +type postgresDumpRequest struct { + // Host is the server to connect to. A nil host means the internal path, + // which connects to localhost inside the database's own container. + Host *HostPort + + // User is the role to connect as. It is empty on the external path, where + // PGUSER comes from the workload's secret; naming it on the command line + // there would be redundant, and the password that goes with it must not be + // there at all. + User string + + // Database is the database to dump. + Database string + + // File is where the dump is written. Empty streams it to stdout. + File string + + // Uncompressed emits -Z0, which the streaming path asks for so the dump + // deduplicates. + Uncompressed bool +} + +// postgresDumpCommand builds the pg_dump argv. +// +// The flag order is the order the in-container path has always written, and every +// difference is driven by a field rather than by a branch, so an internal +// request produces the exact argv that path produced before this feature existed +// (FR-015). It never carries a password: the only credential pg_dump needs +// beyond the role name is PGPASSWORD, which is environment, and on the path where +// that matters it is environment the workload's secret provides. +func postgresDumpCommand(req postgresDumpRequest) []string { + command := []string{"pg_dump", "-Fc"} + if req.Uncompressed { + command = append(command, "-Z0") + } + + host := "localhost" + if req.Host != nil { + host = req.Host.Host + } + command = append(command, "-h", host) + + if req.Host != nil && req.Host.Port > 0 { + command = append(command, "-p", strconv.Itoa(req.Host.Port)) + } + if req.User != "" { + command = append(command, "-U", req.User) + } + + command = append(command, "-d", req.Database) + + if req.File != "" { + command = append(command, "-f", req.File) + } + + return command +} + +// externalPostgresDumpRequest is the request for a dump from a server outside +// the deployment: the member the probe reached, no role on the command line, and +// no password anywhere. +// +// The host is the one the probe got an answer from, not the first in the list. +// The two differ exactly when they matter — the probe walks the members until +// one responds, so taking Hosts[0] here, as an earlier version of this did, +// throws away the only reachability evidence the run has and points the dump at +// a member it has just watched fail. +func externalPostgresDumpRequest(capture *externalCapture, file string, uncompressed bool) postgresDumpRequest { + host := capture.Facts.Answered + + return postgresDumpRequest{ + Host: &host, + Database: capture.Endpoint.Database, + File: file, + Uncompressed: uncompressed, + } +} + +// internalPostgresDumpRequest is the request the in-container path has always +// made. +func internalPostgresDumpRequest(cfg *Configuration, file string, uncompressed bool) postgresDumpRequest { + return postgresDumpRequest{ + User: cfg.PostgresUsername, + Database: cfg.PostgresDatabase, + File: file, + Uncompressed: uncompressed, + } +} + // backupTaskManagerDBStream returns a data factory that streams the PostgreSQL dump // in custom format without compression (-Fc -Z0) from the container via exec stdout. func (iops *InfrahubOps) backupTaskManagerDBStream() (func() (io.ReadCloser, error), error) { return func() (io.ReadCloser, error) { + // Resolved when the factory runs rather than when it was built, so the + // transient workload's life covers the stream. A nil source is an + // internal database and takes the path below unchanged. + source, err := iops.externalDatabaseFor(serviceTaskManagerDB) + if err != nil { + return nil, err + } + if source != nil { + return iops.streamTaskManagerDBExternal() + } + opts := &ExecOptions{Env: map[string]string{ "PGPASSWORD": iops.config.PostgresPassword, }} stdout, wait, err := iops.ExecStreamPipe( "task-manager-db", - []string{"pg_dump", "-Fc", "-Z0", "-h", "localhost", "-U", iops.config.PostgresUsername, "-d", iops.config.PostgresDatabase}, + postgresDumpCommand(internalPostgresDumpRequest(iops.config, "", true)), opts, ) if err != nil { @@ -30,7 +148,86 @@ func (iops *InfrahubOps) backupTaskManagerDBStream() (func() (io.ReadCloser, err }, nil } +// externalPostgresDumpFile is where an external dump is written: the transient +// workload's scratch volume, which is the volume FR-022 sized, rather than a +// temp directory on the node. +const externalPostgresDumpFile = transientScratchPath + "/infrahubops_prefect.dump" + +// streamTaskManagerDBExternal streams the dump from a server outside the +// deployment. The transient workload is given back when the stream closes, and +// on every failure path before that (FR-011). +func (iops *InfrahubOps) streamTaskManagerDBExternal() (io.ReadCloser, error) { + capture, err := iops.openExternalCapture(serviceTaskManagerDB) + if err != nil { + return nil, err + } + + logrus.Infof("Streaming the external PostgreSQL dump from %s (of the endpoints supplied, %s answered the probe)...", + capture.Endpoint.endpointTarget(), capture.Facts.Answered) + + stdout, wait, err := iops.ExecStreamPipe( + serviceTaskManagerDB, + postgresDumpCommand(externalPostgresDumpRequest(capture, "", true)), + capture.execOptions(capture.ClientEnv), + ) + if err != nil { + capture.Release() + + return nil, fmt.Errorf("failed to start the external postgres stream from %s: %w", capture.Endpoint.endpointTarget(), err) + } + + // Bounded by the capture's own bound, which the capture workload's deadline + // budgets for, so that the tool rather than the cluster ends a transfer that + // is slow but still progressing (see streamNeo4jEnterpriseExternal). + return &execReadCloser{ + reader: stdout, + wait: wait, + idleTimeout: defaultStreamIdleTimeout, + deadline: time.Now().Add(capture.Bound), + cleanup: capture.Release, + }, nil +} + +// backupTaskManagerDBExternal takes the dump from a server outside the +// deployment. The destination and the entry name are the internal path's, so the +// artifact is byte-compatible either way (FR-015). +func (iops *InfrahubOps) backupTaskManagerDBExternal(backupDir string) error { + capture, err := iops.openExternalCapture(serviceTaskManagerDB) + if err != nil { + return err + } + defer capture.Release() + + logrus.Infof("Backing up the external PostgreSQL database from %s (of the endpoints supplied, %s answered the probe)...", + capture.Endpoint.endpointTarget(), capture.Facts.Answered) + + if output, err := iops.execBoundedAgainst( + capture.Bound, "the database dump", capture.Endpoint.endpointTarget(), serviceTaskManagerDB, + postgresDumpCommand(externalPostgresDumpRequest(capture, externalPostgresDumpFile, false)), + capture.execOptions(capture.ClientEnv), + ); err != nil { + return fmt.Errorf("failed to dump the external PostgreSQL database at %s: %w\nOutput: %v", capture.Endpoint.endpointTarget(), err, output) + } + defer iops.removeTransientScratch(serviceTaskManagerDB, "the dump", capture.execOptions(nil), externalPostgresDumpFile) + + if err := iops.copyFromTransientWorkload(capture.Pod, externalPostgresDumpFile, filepath.Join(backupDir, "prefect.dump")); err != nil { + return fmt.Errorf("failed to copy the external postgresql dump: %w", err) + } + + logrus.Info("PostgreSQL backup completed") + + return nil +} + func (iops *InfrahubOps) backupTaskManagerDB(backupDir string) error { + source, err := iops.externalDatabaseFor(serviceTaskManagerDB) + if err != nil { + return err + } + if source != nil { + return iops.backupTaskManagerDBExternal(backupDir) + } + logrus.Info("Backing up PostgreSQL database...") // Determine writable temp directory @@ -43,7 +240,7 @@ func (iops *InfrahubOps) backupTaskManagerDB(backupDir string) error { }} if output, err := iops.Exec( "task-manager-db", - []string{"pg_dump", "-Fc", "-h", "localhost", "-U", iops.config.PostgresUsername, "-d", iops.config.PostgresDatabase, "-f", dumpFile}, + postgresDumpCommand(internalPostgresDumpRequest(iops.config, dumpFile, false)), opts, ); err != nil { return fmt.Errorf("failed to create postgresql dump: %w\nOutput: %v", err, output) @@ -63,7 +260,76 @@ func (iops *InfrahubOps) backupTaskManagerDB(backupDir string) error { return nil } +// postgresRestoreRequest is everything the restore command needs, and it is +// the dump request's mirror: the same three fields differ between a client +// running inside the database's own container and one running in a transient +// workload beside an external server. +type postgresRestoreRequest struct { + // Host is the server to connect to. A nil host is the internal path's Unix + // socket branch, which names no host at all; a host with no port is its TCP + // branch, which has always connected to localhost without naming one. + Host *HostPort + + // User is the role to connect as. It is empty on the external path, where + // PGUSER comes from the workload's secret, and empty on the socket branch, + // where the container's own user is the role. + User string + + // File is the dump to read. + File string + + // Socket asks for the connection the internal path uses where the + // container's user matches the database role: no host, no user, no + // password. + Socket bool +} + +// postgresRestoreCommand builds the pg_restore argv. +// +// The flag order is the order the in-container path has always written, and +// every difference is driven by a field rather than by a branch, so an internal +// request produces the exact argv that path produced before this feature +// existed (FR-015). It never carries a password: the only credential +// pg_restore needs beyond the role name is PGPASSWORD, which is environment — +// and on the path where that matters it is environment the workload's secret +// provides (FR-014). +func postgresRestoreCommand(req postgresRestoreRequest) []string { + command := []string{"pg_restore"} + + if !req.Socket { + host := "localhost" + if req.Host != nil { + host = req.Host.Host + } + command = append(command, "-h", host) + + if req.Host != nil && req.Host.Port > 0 { + command = append(command, "-p", strconv.Itoa(req.Host.Port)) + } + } + + command = append(command, "-d", "postgres") + + if req.User != "" { + command = append(command, "-U", req.User) + } + + return append(command, "--clean", "--create", req.File) +} + func (iops *InfrahubOps) restorePostgreSQL(workDir string) error { + // Resolved before anything is started or copied: a nil restore is an + // in-deployment database and takes the path below unchanged, and a database + // the gate established is external with nothing prepared for it is an error + // rather than a silent fall-through into a container it does not live in. + restore, err := iops.externalRestoreFor(serviceTaskManagerDB) + if err != nil { + return err + } + if restore != nil { + return iops.restorePostgreSQLExternal(workDir, restore) + } + logrus.Info("Restoring PostgreSQL database...") // Start task-manager-db @@ -99,7 +365,7 @@ func (iops *InfrahubOps) restorePostgreSQL(workDir string) error { useUnixSocket := err == nil && !strings.Contains(strings.TrimSpace(containerUser), "cannot find name") if useUnixSocket { // Use Unix socket connection (no host, user, or password) - restoreCmd = []string{"pg_restore", "-d", "postgres", "--clean", "--create", dumpFile} + restoreCmd = postgresRestoreCommand(postgresRestoreRequest{Socket: true, File: dumpFile}) // On Docker, run as the postgres user so Unix socket auth succeeds backend, backendErr := iops.ensureBackend() if backendErr == nil && backend.Name() == "docker" { @@ -112,7 +378,7 @@ func (iops *InfrahubOps) restorePostgreSQL(workDir string) error { opts = &ExecOptions{Env: map[string]string{ "PGPASSWORD": iops.config.PostgresPassword, }} - restoreCmd = []string{"pg_restore", "-h", "localhost", "-d", "postgres", "-U", iops.config.PostgresUsername, "--clean", "--create", dumpFile} + restoreCmd = postgresRestoreCommand(postgresRestoreRequest{User: iops.config.PostgresUsername, File: dumpFile}) } if output, err := iops.Exec( "task-manager-db", diff --git a/src/internal/app/cli.go b/src/internal/app/cli.go index d10fc28b..f0867406 100644 --- a/src/internal/app/cli.go +++ b/src/internal/app/cli.go @@ -3,7 +3,10 @@ package app import ( "context" "fmt" + "os" + "strconv" "strings" + "sync" "infrahub-ops/src/internal/updater" @@ -12,6 +15,66 @@ import ( "github.com/spf13/viper" ) +const ( + // AllowExternalRestoreFlag is the interactive form of the authorisation to + // restore into a database the deployment does not manage (FR-009). + AllowExternalRestoreFlag = "allow-external-restore" + // AllowExternalRestoreEnvVar is its configuration form, and the only channel + // through which an unattended restore can be authorised (FR-010). + AllowExternalRestoreEnvVar = "INFRAHUB_ALLOW_EXTERNAL_RESTORE" + + // The endpoint overrides FR-004 requires for a deployment whose own + // configuration does not expose where its database is. Named because the + // refusal an undiscoverable endpoint produces has to state the flag that + // supplies it (see undiscoverableEndpoint), and a message naming a flag this + // command never registered is worse than no message at all. + neo4jAddressFlag = "neo4j-address" + postgresAddressFlag = "postgres-address" + + // externalDBTimeoutFlag bounds a single operation against an external + // database, and externalDBTimeoutEnvVar is its configuration form — the one + // an unattended run has (FR-010). Named once because + // resolveExternalDBTimeout has to tell the operator which of the two + // carried the value it refused. + externalDBTimeoutFlag = "external-db-timeout" + externalDBTimeoutEnvVar = "INFRAHUB_EXTERNAL_DB_TIMEOUT" +) + +// resolveExternalRestoreAuth decides whether an external restore is authorised +// and which channel authorised it. The channel matters on its own: FR-010 allows +// an unattended restore only where the authorisation came from configuration, +// which is a distinction the boolean alone cannot carry. +// +// An explicit flag wins over the environment, in either direction, so an +// operator can refuse on the command line what a scheduled job's configuration +// permits. A malformed environment value authorises nothing: the safe reading of +// an unparseable authorisation is that none was given, and it is logged so the +// refusal that follows is explainable. +func resolveExternalRestoreAuth(flagValue, flagSet bool, envValue string) ExternalRestoreAuth { + if flagSet { + if !flagValue { + return ExternalRestoreAuth{} + } + return ExternalRestoreAuth{Source: ExternalRestoreAuthFlag} + } + + // Unset and set-but-empty are the same absence of an authorisation. + if envValue == "" { + return ExternalRestoreAuth{} + } + + allowed, err := strconv.ParseBool(envValue) + if err != nil { + logrus.Warnf("%s is not a boolean (%q): external restore stays unauthorised", AllowExternalRestoreEnvVar, envValue) + return ExternalRestoreAuth{} + } + if !allowed { + return ExternalRestoreAuth{} + } + + return ExternalRestoreAuth{Source: ExternalRestoreAuthConfig} +} + // ConfigureRootCommand wires shared flags, environment variables, and logging for CLI binaries. func ConfigureRootCommand(cmd *cobra.Command, app *InfrahubOps) { cfg := app.Config() @@ -35,6 +98,84 @@ func ConfigureRootCommand(cmd *cobra.Command, app *InfrahubOps) { cmd.PersistentFlags().StringVar(&cfg.S3.Endpoint, "s3-endpoint", cfg.S3.Endpoint, "Custom S3 endpoint URL (for S3-compatible storage)") cmd.PersistentFlags().StringVar(&cfg.S3.Region, "s3-region", cfg.S3.Region, "AWS region for S3 bucket") + // External database flags. Every one of them is inert for a deployment whose + // databases are all internal (FR-015): each either overrides a value + // discovery supplies or applies only once a database has been positively + // established as external. + // + // The plain-string settings among them are registered, bound to viper and + // read back from this one table, so a setting cannot be registered without + // being bound, or bound without being read — three steps that used to be + // spelled out per flag and checked by nothing. + externalDBStrings := []struct { + name string + dst *string + usage string + }{ + {neo4jAddressFlag, &cfg.ExternalDB.Neo4jAddress, "Override the Neo4j host list as host[:port][,host[:port]...] (default: discovered from the deployment)"}, + {postgresAddressFlag, &cfg.ExternalDB.PostgresAddress, "Override the PostgreSQL endpoint as host[:port] (default: discovered from the deployment)"}, + {"external-db-scratch-size", &cfg.ExternalDB.ScratchSize, "Scratch space for the transient backup workload, e.g. 50Gi (default: derived from the database size)"}, + // What the operator's cluster requires of a pod it did not author. + // These are what make the image flags below usable: mirroring the + // tooling image into a private registry — the reason those flags exist + // — needs the pull secret for it to be nameable, or the pod waits in + // ImagePullBackOff until the readiness bound gives up. Same for a + // namespace whose capable nodes are tainted, or which admits pods only + // under a named service account. + // + // Unlike the image flags, these *are* bound to viper. FR-027's reason + // for keeping the image out of ambient environment is that the tool + // holds workload-creation rights and an environment-settable image + // turns those into arbitrary code execution in the customer's + // namespace. None of these can do that while the image stays pinned: + // they say where a pod may run and which credentials fetch its image, + // not what it runs. The pod still mounts no service-account token, so + // naming an account grants it no API access either. And an unattended + // backup is configured by environment as often as by argv, so making + // these flag-only would leave a scheduled run in a mirrored-registry + // cluster with no way to say so. + {"external-db-image-pull-secrets", &cfg.ExternalDB.Scheduling.ImagePullSecrets, "Comma-separated secrets in the deployment's namespace holding registry credentials for the transient workload's image"}, + {"external-db-service-account", &cfg.ExternalDB.Scheduling.ServiceAccount, "Service account the transient workload runs under (it still mounts no service-account token)"}, + {"external-db-node-selector", &cfg.ExternalDB.Scheduling.NodeSelector, "Comma-separated key=value node labels the transient workload must be scheduled onto"}, + {"external-db-tolerations", &cfg.ExternalDB.Scheduling.Tolerations, "Comma-separated taints the transient workload tolerates, written as key=value:Effect, key:Effect or a bare key"}, + {"external-db-priority-class", &cfg.ExternalDB.Scheduling.PriorityClass, "PriorityClass the transient workload is admitted under"}, + } + for _, setting := range externalDBStrings { + cmd.PersistentFlags().StringVar(setting.dst, setting.name, *setting.dst, setting.usage) + } + cmd.PersistentFlags().IntVar(&cfg.ExternalDB.Neo4jBackupPort, "neo4j-backup-port", cfg.ExternalDB.Neo4jBackupPort, "Port the Neo4j backup service listens on (not the Bolt port)") + // The only way certificate verification is skipped for a connection this + // tool opens. A deployment that runs Infrahub with tls_insecure does not + // grant it: FR-029 requires the opt-out to be the operator's own decision + // about this tool, made here, rather than one inherited from a setting that + // was made about something else. + // + // This replaces --external-db-tls-policy, which T095 removed for having no + // consumer. The channel that landed with the capture path does not take a + // policy name: cypher-shell's verification is chosen by the Bolt URI scheme + // and libpq's by PGSSLMODE, and both are two-valued here — verify, or the + // operator's opt-out. A string naming a policy would have had no value to + // carry that this does not (see the CLI-surface contract). + cmd.PersistentFlags().BoolVar(&cfg.ExternalDB.InsecureTLS, insecureTLSFlagName, cfg.ExternalDB.InsecureTLS, "Do not protect connections to an external database server: skip certificate verification, and accept an unencrypted channel where the deployment records TLS as disabled (a discovered insecure-TLS setting does not imply this)") + cmd.PersistentFlags().DurationVar(&cfg.ExternalDB.Timeout, externalDBTimeoutFlag, cfg.ExternalDB.Timeout, "Bound on any single operation against an external database (e.g. 30m, 2h)") + // Registered without a flag variable on purpose: resolveExternalRestoreAuth + // needs to know which channel supplied the authorisation, and reading the + // flag through a variable loses that. + cmd.PersistentFlags().Bool(AllowExternalRestoreFlag, false, "Authorise a destructive restore into a database the deployment does not manage (distinct from --force)") + + // The image flag is deliberately not bound to viper below, so it resolves + // from its default or an explicitly supplied flag and never from ambient + // environment (FR-027): the tool holds workload-creation rights in the + // deployment, and an environment-settable image would turn those rights into + // a means of running an arbitrary image in the customer's namespace. + // + // One flag for both databases, because both run the same image: this tool's + // own, which carries every database tool the external path calls (FR-031, + // ADR 0017). Its default is that image at this binary's version, and a + // development build has none — the help text says so rather than showing an + // empty default an operator would read as "unset is fine". + cmd.PersistentFlags().StringVar(&cfg.ExternalDB.Image, ExternalDBImageFlag, cfg.ExternalDB.Image, "Image the transient workload runs to reach an external database; required on a development build, which has no published image") + bind := func(name string) { if err := viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name)); err != nil { panic(err) @@ -53,8 +194,45 @@ func ConfigureRootCommand(cmd *cobra.Command, app *InfrahubOps) { bind("s3-prefix") bind("s3-endpoint") bind("s3-region") + for _, setting := range externalDBStrings { + bind(setting.name) + } + bind("neo4j-backup-port") + bind(insecureTLSFlagName) + bind(externalDBTimeoutFlag) + // --allow-external-restore is deliberately not bound: viper would report the + // flag and INFRAHUB_ALLOW_EXTERNAL_RESTORE through one key, and FR-010 turns + // on telling them apart, so resolveExternalRestoreAuth reads each channel + // itself. + // --external-db-image is not bound; see FR-027 at its registration above. - cobra.OnInitialize(func() { + // The resolution hangs on the command it configures rather than going + // through cobra.OnInitialize. + // + // cobra.OnInitialize appends to a package-global slice that nothing can + // unregister from, and the closure captures this call's Configuration. A + // binary calls this once and never notices. The tests call it once per + // case, so every case ran every earlier case's resolution as well — against + // its own environment, writing into Configuration values those earlier + // cases were still holding — and the work grew with the square of the case + // count. A per-command hook is registered on the command and goes when it + // does. + // + // Only the first persistent pre-run found walking up from the executed + // command runs, unless cobra.EnableTraverseRunHooks is set: a subcommand + // that defined one of its own would silently take the whole of this + // resolution with it. Nothing under these roots defines one, and + // TestNoSubcommandShadowsTheRootPersistentPreRun is what keeps that true. + // + // The resolution is also held as a value, reachable through + // app.ResolveConfiguration, because cobra validates a command's arguments + // *before* it runs any persistent pre-run. A command whose argument + // contract depends on a resolved value — `restore`, whose arity rule + // differs per backend — cannot wait for the hook, so it asks for the + // resolution by name and the hook's later call is the no-op. Running it at + // most once is what makes that safe: the body is otherwise repeat-safe, but + // its two warnings about malformed configuration would be emitted twice. + resolve := func() error { viper.SetEnvPrefix("INFRAHUB") viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) viper.AutomaticEnv() @@ -92,6 +270,37 @@ func ConfigureRootCommand(cmd *cobra.Command, app *InfrahubOps) { if viper.IsSet("s3-region") { cfg.S3.Region = viper.GetString("s3-region") } + for _, setting := range externalDBStrings { + if viper.IsSet(setting.name) { + *setting.dst = viper.GetString(setting.name) + } + } + if viper.IsSet("neo4j-backup-port") { + cfg.ExternalDB.Neo4jBackupPort = viper.GetInt("neo4j-backup-port") + } + if viper.IsSet(insecureTLSFlagName) { + cfg.ExternalDB.InsecureTLS = viper.GetBool(insecureTLSFlagName) + } + // Read as a string and parsed here rather than through + // viper.GetDuration, which reads a unit-less value as nanoseconds: see + // resolveExternalDBTimeout. A value it refuses leaves the bound as it + // was, and says so. + if viper.IsSet(externalDBTimeoutFlag) { + bound, err := resolveExternalDBTimeout(viper.GetString(externalDBTimeoutFlag)) + switch { + case err != nil: + logrus.Warn(err) + case bound > 0: + cfg.ExternalDB.Timeout = bound + } + } + + flagValue, _ := cmd.PersistentFlags().GetBool(AllowExternalRestoreFlag) + cfg.ExternalDB.Restore = resolveExternalRestoreAuth( + flagValue, + cmd.PersistentFlags().Changed(AllowExternalRestoreFlag), + os.Getenv(AllowExternalRestoreEnvVar), + ) switch viper.GetString("log-format") { case "json": @@ -99,7 +308,40 @@ func ConfigureRootCommand(cmd *cobra.Command, app *InfrahubOps) { default: logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) } - }) + + return nil + } + + var once sync.Once + var resolveErr error + app.resolveConfiguration = func() error { + once.Do(func() { resolveErr = resolve() }) + return resolveErr + } + + cmd.PersistentPreRunE = func(*cobra.Command, []string) error { + return app.ResolveConfiguration() + } +} + +// ResolveConfiguration resolves this app's configuration from the flags and +// environment of the root command ConfigureRootCommand was given, if it has not +// been resolved already, and reports what that resolution refused. +// +// It exists for the one caller that cannot wait for the persistent pre-run: +// cobra validates arguments before running it, so a command whose argument +// contract reads a resolved value has to ask for the resolution itself. Calling +// it is idempotent — the second and later calls return the first call's result +// without re-reading anything — so the pre-run's own call stays correct for +// every command that does not need it early. +// +// An app whose root command was never configured has nothing to resolve and +// keeps the defaults NewInfrahubOps gave it. +func (iops *InfrahubOps) ResolveConfiguration() error { + if iops.resolveConfiguration == nil { + return nil + } + return iops.resolveConfiguration() } // AttachEnvironmentCommands wires the environment detection subcommands onto a root command. @@ -115,9 +357,13 @@ func AttachEnvironmentCommands(rootCmd *cobra.Command, app *InfrahubOps) { detectCmd := &cobra.Command{ Use: "detect", - Short: "Detect the active deployment environment", + Short: "Detect the active deployment environment and where each database lives", RunE: func(cmd *cobra.Command, args []string) error { - return app.DetectEnvironment() + // ReportEnvironment rather than DetectEnvironment: this is the + // command an operator reaches for when a backup fails, so it + // reports where each database was resolved to as well as which + // deployment was found (US2 scenario 5). + return app.ReportEnvironment() }, } diff --git a/src/internal/app/cli_test.go b/src/internal/app/cli_test.go new file mode 100644 index 00000000..6808c036 --- /dev/null +++ b/src/internal/app/cli_test.go @@ -0,0 +1,549 @@ +package app + +import ( + "io" + "os" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// externalDBEnvVars are the environment variables that can reach the external +// database configuration, including the one that must not (FR-027). Every test +// starts with all of them absent, so a value in the developer's or CI runner's +// environment cannot decide the outcome. +var externalDBEnvVars = []string{ + "INFRAHUB_NEO4J_ADDRESS", + "INFRAHUB_NEO4J_BACKUP_PORT", + "INFRAHUB_POSTGRES_ADDRESS", + "INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE", + "INFRAHUB_EXTERNAL_DB_TIMEOUT", + "INFRAHUB_EXTERNAL_DB_INSECURE_TLS", + "INFRAHUB_EXTERNAL_DB_IMAGE", + "INFRAHUB_EXTERNAL_DB_IMAGE_PULL_SECRETS", + "INFRAHUB_EXTERNAL_DB_SERVICE_ACCOUNT", + "INFRAHUB_EXTERNAL_DB_NODE_SELECTOR", + "INFRAHUB_EXTERNAL_DB_TOLERATIONS", + "INFRAHUB_EXTERNAL_DB_PRIORITY_CLASS", + AllowExternalRestoreEnvVar, +} + +// newConfiguredRootCommand wires a root command exactly as a binary's main does +// and hands back both it and the app it resolves into, without executing it — +// which is what lets a test observe the configuration before and after the +// resolution runs. viper is global, so each call resets it and restores it +// afterwards. +func newConfiguredRootCommand(t *testing.T, env map[string]string) (*cobra.Command, *InfrahubOps) { + t.Helper() + + viper.Reset() + t.Cleanup(viper.Reset) + + for _, name := range externalDBEnvVars { + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatalf("unsetting %s = %v, want nil", name, err) + } + } + for name, value := range env { + t.Setenv(name, value) + } + + iops := NewInfrahubOps() + cmd := &cobra.Command{Use: "root", RunE: func(*cobra.Command, []string) error { return nil }} + ConfigureRootCommand(cmd, iops) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + + return cmd, iops +} + +// configureRootCommand wires a root command exactly as a binary's main does, +// parses the given command line, and returns the configuration the flags and +// environment resolved to. +func configureRootCommand(t *testing.T, env map[string]string, args ...string) *Configuration { + t.Helper() + + cmd, iops := newConfiguredRootCommand(t, env) + cmd.SetArgs(args) + + if err := cmd.Execute(); err != nil { + t.Fatalf("executing %v = %v, want nil", args, err) + } + + return iops.Config() +} + +// TestResolveConfigurationIsCallableBeforeArgumentValidation covers the ordering +// T123 turned on. Cobra validates a command's arguments before it runs any +// persistent pre-run, so a command whose argument contract reads a resolved +// value — `restore`, whose arity rule differs per backend — cannot wait for the +// hook and has to ask for the resolution by name. +func TestResolveConfigurationIsCallableBeforeArgumentValidation(t *testing.T) { + cmd, iops := newConfiguredRootCommand(t, map[string]string{"INFRAHUB_NEO4J_ADDRESS": "early.example.internal"}) + + if got := iops.Config().ExternalDB.Neo4jAddress; got != "" { + t.Fatalf("Neo4jAddress = %q before anything ran, want the unresolved default", got) + } + + if err := iops.ResolveConfiguration(); err != nil { + t.Fatalf("ResolveConfiguration() = %v, want nil", err) + } + if got := iops.Config().ExternalDB.Neo4jAddress; got != "early.example.internal" { + t.Errorf("Neo4jAddress = %q after ResolveConfiguration, want the configured value: nothing that runs before the persistent pre-run can read the configuration", got) + } + + cmd.SetArgs(nil) + if err := cmd.Execute(); err != nil { + t.Fatalf("executing the command afterwards = %v, want nil", err) + } + if got := iops.Config().ExternalDB.Neo4jAddress; got != "early.example.internal" { + t.Errorf("Neo4jAddress = %q after the command ran, want the configured value", got) + } +} + +// TestResolveConfigurationRunsOnce pins what makes calling the resolution early +// safe: the second and later calls read nothing again. The body is otherwise +// repeat-safe, but it warns about a malformed timeout and a malformed +// authorisation, and an operator must not be told twice about one mistake. +// +// The second call is given a different environment, so a resolution that ran +// again would be visible in the value it produced. +func TestResolveConfigurationRunsOnce(t *testing.T) { + cmd, iops := newConfiguredRootCommand(t, map[string]string{"INFRAHUB_NEO4J_ADDRESS": "first.example.internal"}) + + if err := iops.ResolveConfiguration(); err != nil { + t.Fatalf("ResolveConfiguration() = %v, want nil", err) + } + + t.Setenv("INFRAHUB_NEO4J_ADDRESS", "second.example.internal") + + if err := iops.ResolveConfiguration(); err != nil { + t.Fatalf("second ResolveConfiguration() = %v, want nil", err) + } + if got := iops.Config().ExternalDB.Neo4jAddress; got != "first.example.internal" { + t.Errorf("Neo4jAddress = %q after a second ResolveConfiguration, want the first call's value: the resolution ran twice", got) + } + + cmd.SetArgs(nil) + if err := cmd.Execute(); err != nil { + t.Fatalf("executing the command afterwards = %v, want nil", err) + } + if got := iops.Config().ExternalDB.Neo4jAddress; got != "first.example.internal" { + t.Errorf("Neo4jAddress = %q after the persistent pre-run, want the first call's value: the pre-run's call must be the no-op", got) + } +} + +// TestResolveConfigurationWithoutARootCommand covers the app every test in this +// package builds without configuring a command: it has nothing to resolve and +// keeps the defaults NewInfrahubOps gave it. +func TestResolveConfigurationWithoutARootCommand(t *testing.T) { + iops := NewInfrahubOps() + + if err := iops.ResolveConfiguration(); err != nil { + t.Fatalf("ResolveConfiguration() = %v, want nil", err) + } + if got := iops.Config().Backend; got != BackendTarball { + t.Errorf("Backend = %q, want %q", got, BackendTarball) + } +} + +// TestConfigureRootCommandLeavesNoResolutionBehind is the pollution that made +// this file's own cases interfere with each other. +// +// Registering the resolution through cobra.OnInitialize put a closure holding +// this call's Configuration into package-global state nothing can unregister +// from, so every later Execute in the process ran it again — against whatever +// environment that later case had set, writing into a Configuration the earlier +// case was still holding. +func TestConfigureRootCommandLeavesNoResolutionBehind(t *testing.T) { + first := configureRootCommand(t, map[string]string{"INFRAHUB_NEO4J_ADDRESS": "first.example.internal"}) + if first.ExternalDB.Neo4jAddress != "first.example.internal" { + t.Fatalf("Neo4jAddress = %q, want the value this command was given", first.ExternalDB.Neo4jAddress) + } + + configureRootCommand(t, map[string]string{"INFRAHUB_NEO4J_ADDRESS": "second.example.internal"}) + + if first.ExternalDB.Neo4jAddress != "first.example.internal" { + t.Errorf("the first command's Neo4jAddress became %q when a second command ran: its resolution outlived the command it was registered for", first.ExternalDB.Neo4jAddress) + } +} + +// TestNoSubcommandShadowsTheRootPersistentPreRun protects the one property the +// move to a per-command hook depends on. Cobra runs only the first persistent +// pre-run it finds walking up from the executed command, so a subcommand that +// defined one of its own would silently take the whole of ConfigureRootCommand's +// resolution with it — every flag and environment variable would read as unset. +// +// It walks the tree this package builds. A binary's own subcommands are +// constructed inside its main, out of reach from here, so this is the shared +// subtree rather than the whole of any binary's. +func TestNoSubcommandShadowsTheRootPersistentPreRun(t *testing.T) { + root := &cobra.Command{Use: "root", RunE: func(*cobra.Command, []string) error { return nil }} + iops := NewInfrahubOps() + ConfigureRootCommand(root, iops) + AttachEnvironmentCommands(root, iops) + t.Cleanup(viper.Reset) + + if root.PersistentPreRunE == nil { + t.Fatal("ConfigureRootCommand registered no persistent pre-run, so nothing resolves the configuration") + } + + var walk func(cmd *cobra.Command) + walk = func(cmd *cobra.Command) { + for _, child := range cmd.Commands() { + if child.PersistentPreRunE != nil || child.PersistentPreRun != nil { + t.Errorf("%q defines a persistent pre-run, which shadows the root's and drops the whole configuration resolution; fold it into the root's hook or set cobra.EnableTraverseRunHooks", child.CommandPath()) + } + walk(child) + } + } + walk(root) +} + +// TestExternalDBFlagDefaults pins the defaults an internal-only deployment sees. +// They are what makes the whole surface inert for it (FR-015): the two values +// discovery cannot supply have defaults, and nothing else is set. +func TestExternalDBFlagDefaults(t *testing.T) { + cfg := configureRootCommand(t, nil) + + if cfg.ExternalDB.Neo4jBackupPort != defaultNeo4jBackupPort { + t.Errorf("Neo4jBackupPort = %d, want %d", cfg.ExternalDB.Neo4jBackupPort, defaultNeo4jBackupPort) + } + if cfg.ExternalDB.Timeout != defaultExternalDBTimeout { + t.Errorf("Timeout = %v, want %v", cfg.ExternalDB.Timeout, defaultExternalDBTimeout) + } + if cfg.ExternalDB.Image != defaultExternalDBImage() { + t.Errorf("Image = %q, want %q", cfg.ExternalDB.Image, defaultExternalDBImage()) + } + if cfg.ExternalDB.Neo4jAddress != "" || cfg.ExternalDB.PostgresAddress != "" { + t.Errorf("addresses = %q/%q, want both empty", cfg.ExternalDB.Neo4jAddress, cfg.ExternalDB.PostgresAddress) + } + if cfg.ExternalDB.ScratchSize != "" { + t.Errorf("ScratchSize = %q, want it empty", cfg.ExternalDB.ScratchSize) + } + if cfg.ExternalDB.Restore != (ExternalRestoreAuth{}) { + t.Errorf("Restore = %+v, want the zero value (unauthorised)", cfg.ExternalDB.Restore) + } +} + +// TestExternalDBFlagsFromCommandLine asserts each flag reaches the field it +// configures. The address overrides are asserted together because FR-001 +// requires overriding one to leave the other's resolution alone. +func TestExternalDBFlagsFromCommandLine(t *testing.T) { + cfg := configureRootCommand(t, nil, + "--neo4j-address", "db-a:7688,db-b", + "--neo4j-backup-port", "7000", + "--postgres-address", "pg.example.net:5433", + "--external-db-scratch-size", "80Gi", + "--external-db-timeout", "45m", + "--external-db-image", "registry.example.net/infrahub-backup:2.3.0", + ) + + tests := []struct { + field string + got string + want string + }{ + {"Neo4jAddress", cfg.ExternalDB.Neo4jAddress, "db-a:7688,db-b"}, + {"PostgresAddress", cfg.ExternalDB.PostgresAddress, "pg.example.net:5433"}, + {"ScratchSize", cfg.ExternalDB.ScratchSize, "80Gi"}, + {"Image", cfg.ExternalDB.Image, "registry.example.net/infrahub-backup:2.3.0"}, + } + for _, tt := range tests { + if tt.got != tt.want { + t.Errorf("%s = %q, want %q", tt.field, tt.got, tt.want) + } + } + + if cfg.ExternalDB.Neo4jBackupPort != 7000 { + t.Errorf("Neo4jBackupPort = %d, want 7000", cfg.ExternalDB.Neo4jBackupPort) + } + if cfg.ExternalDB.Timeout != 45*time.Minute { + t.Errorf("Timeout = %v, want 45m", cfg.ExternalDB.Timeout) + } +} + +// TestExternalDBFlagsFromEnvironment asserts the flags follow the existing +// INFRAHUB_-prefixed binding convention, which is what makes the feature usable +// from a scheduled job's configuration. +func TestExternalDBFlagsFromEnvironment(t *testing.T) { + cfg := configureRootCommand(t, map[string]string{ + "INFRAHUB_NEO4J_ADDRESS": "db.example.net", + "INFRAHUB_NEO4J_BACKUP_PORT": "7001", + "INFRAHUB_POSTGRES_ADDRESS": "pg.example.net", + "INFRAHUB_EXTERNAL_DB_SCRATCH_SIZE": "120Gi", + "INFRAHUB_EXTERNAL_DB_TIMEOUT": "90m", + }) + + if cfg.ExternalDB.Neo4jAddress != "db.example.net" { + t.Errorf("Neo4jAddress = %q, want %q", cfg.ExternalDB.Neo4jAddress, "db.example.net") + } + if cfg.ExternalDB.Neo4jBackupPort != 7001 { + t.Errorf("Neo4jBackupPort = %d, want 7001", cfg.ExternalDB.Neo4jBackupPort) + } + if cfg.ExternalDB.PostgresAddress != "pg.example.net" { + t.Errorf("PostgresAddress = %q, want %q", cfg.ExternalDB.PostgresAddress, "pg.example.net") + } + if cfg.ExternalDB.ScratchSize != "120Gi" { + t.Errorf("ScratchSize = %q, want %q", cfg.ExternalDB.ScratchSize, "120Gi") + } + if cfg.ExternalDB.Timeout != 90*time.Minute { + t.Errorf("Timeout = %v, want 90m", cfg.ExternalDB.Timeout) + } +} + +// TestExternalDBImagesIgnoreEnvironment is FR-027's flag-layer guarantee: the +// tool holds workload-creation rights in the customer's namespace, so the image +// it runs there resolves from its default or an explicit flag and never from +// ambient environment. The image flag breaks the binding convention every +// other flag follows, which is exactly why it is asserted rather than assumed. +// +// The environment carries the variable the binding convention would give the +// one flag and the two the flags it replaced (FR-031) once had, so a binding +// added back under either name is caught. +func TestExternalDBImagesIgnoreEnvironment(t *testing.T) { + withBuildVersion(t, "v2.3.0") + + env := map[string]string{ + "INFRAHUB_EXTERNAL_DB_IMAGE": "attacker.example.net/anything:latest", + "INFRAHUB_EXTERNAL_DB_IMAGE_NEO4J": "attacker.example.net/anything:latest", + "INFRAHUB_EXTERNAL_DB_IMAGE_POSTGRES": "attacker.example.net/anything:latest", + } + const released = "registry.opsmill.io/opsmill/infrahub-backup:2.3.0" + + t.Run("environment alone leaves the default", func(t *testing.T) { + cfg := configureRootCommand(t, env) + + if cfg.ExternalDB.Image != released { + t.Errorf("Image = %q, want the release's own %q", cfg.ExternalDB.Image, released) + } + }) + + t.Run("an explicit flag still decides", func(t *testing.T) { + cfg := configureRootCommand(t, env, + "--external-db-image", "registry.example.net/infrahub-backup:2.3.0", + ) + + if cfg.ExternalDB.Image != "registry.example.net/infrahub-backup:2.3.0" { + t.Errorf("Image = %q, want the flag's value", cfg.ExternalDB.Image) + } + }) + + t.Run("a development build gets no image from the environment either", func(t *testing.T) { + withBuildVersion(t, "") + cfg := configureRootCommand(t, env) + + if cfg.ExternalDB.Image != "" { + t.Errorf("Image = %q on a development build, want it empty so the external gate refuses", cfg.ExternalDB.Image) + } + }) +} + +// TestExternalDBImageIsOneUnboundFlag is T164: one --external-db-image replaces +// the per-database pair (FR-031), and it is the flag viper does not know about. +// Asserting viper's own registry rather than only the resolved value is what +// catches a binding that some later default happens to mask. +func TestExternalDBImageIsOneUnboundFlag(t *testing.T) { + root, _ := newConfiguredRootCommand(t, nil) + + if root.PersistentFlags().Lookup(ExternalDBImageFlag) == nil { + t.Fatalf("--%s is not registered", ExternalDBImageFlag) + } + for _, gone := range []string{"external-db-image-neo4j", "external-db-image-postgres"} { + if root.PersistentFlags().Lookup(gone) != nil { + t.Errorf("--%s is still registered; FR-031 replaces it with --%s", gone, ExternalDBImageFlag) + } + } + for _, key := range viper.AllKeys() { + if strings.HasPrefix(key, "external-db-image") && key != "external-db-image-pull-secrets" { + t.Errorf("viper key %q is bound; the workload image must not be settable from the environment (FR-027)", key) + } + } + if viper.IsSet(ExternalDBImageFlag) { + t.Errorf("viper reports %q as set; it must not be bound at all (FR-027)", ExternalDBImageFlag) + } +} + +// TestExternalDBSchedulingFlags is T111 at the flag layer. The image flags are +// unusable on their own: an operator who mirrors the tooling image into their +// own registry — the reason those flags exist — has a registry that needs +// credentials, and a pod that can name no pull secret waits in ImagePullBackOff +// until readiness gives up. +// +// Unlike the image flags these follow the binding convention, and that is +// asserted rather than assumed: an unattended backup is configured by +// environment as often as by argv, and none of these can run code — they say +// where a pod may be placed and which credentials fetch its image, while the +// image itself stays pinned or flag-only. +func TestExternalDBSchedulingFlags(t *testing.T) { + t.Run("nothing is configured by default", func(t *testing.T) { + cfg := configureRootCommand(t, nil) + + if cfg.ExternalDB.Scheduling != (ExternalDBScheduling{}) { + t.Errorf("Scheduling = %+v, want the zero value", cfg.ExternalDB.Scheduling) + } + }) + + t.Run("from the command line", func(t *testing.T) { + cfg := configureRootCommand(t, nil, + "--external-db-image-pull-secrets", "mirror-creds,backup-mirror-creds", + "--external-db-service-account", "infrahub-backup", + "--external-db-node-selector", "node-role=backup", + "--external-db-tolerations", "dedicated=backup:NoSchedule", + "--external-db-priority-class", "backup-critical", + ) + + want := ExternalDBScheduling{ + ImagePullSecrets: "mirror-creds,backup-mirror-creds", + ServiceAccount: "infrahub-backup", + NodeSelector: "node-role=backup", + Tolerations: "dedicated=backup:NoSchedule", + PriorityClass: "backup-critical", + } + if cfg.ExternalDB.Scheduling != want { + t.Errorf("Scheduling = %+v, want %+v", cfg.ExternalDB.Scheduling, want) + } + }) + + t.Run("from the environment", func(t *testing.T) { + cfg := configureRootCommand(t, map[string]string{ + "INFRAHUB_EXTERNAL_DB_IMAGE_PULL_SECRETS": "mirror-creds", + "INFRAHUB_EXTERNAL_DB_SERVICE_ACCOUNT": "infrahub-backup", + "INFRAHUB_EXTERNAL_DB_NODE_SELECTOR": "node-role=backup,disk=ssd", + "INFRAHUB_EXTERNAL_DB_TOLERATIONS": "dedicated:NoExecute", + "INFRAHUB_EXTERNAL_DB_PRIORITY_CLASS": "backup-critical", + }) + + want := ExternalDBScheduling{ + ImagePullSecrets: "mirror-creds", + ServiceAccount: "infrahub-backup", + NodeSelector: "node-role=backup,disk=ssd", + Tolerations: "dedicated:NoExecute", + PriorityClass: "backup-critical", + } + if cfg.ExternalDB.Scheduling != want { + t.Errorf("Scheduling = %+v, want %+v", cfg.ExternalDB.Scheduling, want) + } + }) +} + +// TestExternalRestoreAuthorisationChannels covers FR-010 end to end at this +// layer: the configuration channel authorises an unattended restore, the flag +// channel is recorded as the interactive one, and each is distinguishable from +// the other in the resolved configuration. +func TestExternalRestoreAuthorisationChannels(t *testing.T) { + tests := []struct { + name string + env map[string]string + args []string + want ExternalRestoreAuth + }{ + { + name: "absent", + want: ExternalRestoreAuth{}, + }, + { + name: "configuration", + env: map[string]string{AllowExternalRestoreEnvVar: "true"}, + want: ExternalRestoreAuth{Source: ExternalRestoreAuthConfig}, + }, + { + name: "flag", + args: []string{"--" + AllowExternalRestoreFlag}, + want: ExternalRestoreAuth{Source: ExternalRestoreAuthFlag}, + }, + { + name: "flag refuses what configuration permits", + env: map[string]string{AllowExternalRestoreEnvVar: "true"}, + args: []string{"--" + AllowExternalRestoreFlag + "=false"}, + want: ExternalRestoreAuth{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := configureRootCommand(t, tt.env, tt.args...) + + if cfg.ExternalDB.Restore != tt.want { + t.Errorf("Restore = %+v, want %+v", cfg.ExternalDB.Restore, tt.want) + } + }) + } +} + +// TestResolveExternalRestoreAuth covers the resolution rules directly, including +// the values a command line cannot produce. The refusals matter more than the +// authorisations: this is the only gate in front of overwriting a database the +// deployment does not manage (FR-009). +func TestResolveExternalRestoreAuth(t *testing.T) { + tests := []struct { + name string + flagValue bool + flagSet bool + envValue string + want ExternalRestoreAuth + }{ + { + name: "nothing supplied", + want: ExternalRestoreAuth{}, + }, + { + name: "flag set", + flagValue: true, + flagSet: true, + want: ExternalRestoreAuth{Source: ExternalRestoreAuthFlag}, + }, + { + name: "flag explicitly false", + flagSet: true, + want: ExternalRestoreAuth{}, + }, + { + name: "flag wins over the environment", + flagValue: true, + flagSet: true, + envValue: "false", + want: ExternalRestoreAuth{Source: ExternalRestoreAuthFlag}, + }, + { + name: "environment true", + envValue: "true", + want: ExternalRestoreAuth{Source: ExternalRestoreAuthConfig}, + }, + { + name: "environment 1", + envValue: "1", + want: ExternalRestoreAuth{Source: ExternalRestoreAuthConfig}, + }, + { + name: "environment false", + envValue: "false", + want: ExternalRestoreAuth{}, + }, + { + name: "environment empty", + envValue: "", + want: ExternalRestoreAuth{}, + }, + { + name: "environment malformed", + envValue: "ture", + want: ExternalRestoreAuth{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := resolveExternalRestoreAuth(tt.flagValue, tt.flagSet, tt.envValue) + + if got != tt.want { + t.Errorf("resolveExternalRestoreAuth(%t, %t, %q) = %+v, want %+v", + tt.flagValue, tt.flagSet, tt.envValue, got, tt.want) + } + }) + } +} diff --git a/src/internal/app/collect.go b/src/internal/app/collect.go index dea43687..21f67b8e 100644 --- a/src/internal/app/collect.go +++ b/src/internal/app/collect.go @@ -157,6 +157,13 @@ func (e *collectSkipError) Error() string { return e.reason } // run (FR-009); only a missing environment, an unwritable output directory, // or an archiving failure returns an error. func (iops *InfrahubOps) CollectBundle(opts CollectOptions) error { + // Marked for the whole run rather than inside the --include-backup + // collector, because the property belongs to the binary and not to one of + // its steps: nothing collection does may create, delete or scale a + // workload of the deployment's (ADR-0003). It is what stops --include-backup + // reaching the external-database capture path, which would. + iops.forbidExternalCapture() + backend, err := iops.ensureBackend() if err != nil { return fmt.Errorf("bundle collection requires a usable Infrahub environment: %w", err) diff --git a/src/internal/app/collect_extras_test.go b/src/internal/app/collect_extras_test.go index b7a42e4f..0cc14a5e 100644 --- a/src/internal/app/collect_extras_test.go +++ b/src/internal/app/collect_extras_test.go @@ -470,3 +470,83 @@ func TestNewestNewArchive(t *testing.T) { }) } } + +// TestCollectionRefusesAnExternalDatabaseCapture is T118's boundary, asserted +// on both halves of it. +// +// ADR-0003 says without qualification that `infrahub-collect` never stops, +// restarts or scales a workload of the deployment's, and it is the reason the +// tool can be handed to a customer to run against their own production cluster. +// `--include-backup` delegates to CreateBackup (ADR-0006), and CreateBackup +// against a database that lives outside the deployment creates a pod and a +// secret in the operator's namespace. So the external path is refused from +// collection rather than the ADR being weakened — the cost being that such a +// deployment gets no backup inside its bundle, which the message says. +func TestCollectionRefusesAnExternalDatabaseCapture(t *testing.T) { + t.Run("a collection run is marked as one that may not create a workload", func(t *testing.T) { + iops := NewInfrahubOps() + iops.backend = newFakeCollectBackend() + opts := CollectOptions{OutputDir: filepath.Join(t.TempDir(), "bundles"), LogLines: 100} + + if err := iops.CollectBundle(opts); err != nil { + t.Fatalf("CollectBundle() = %v, want the bundle produced", err) + } + + if !iops.externalCaptureForbidden { + t.Error("CollectBundle left the run permitted to capture an external database; --include-backup would then create and delete a pod and a secret (ADR-0003)") + } + }) + + t.Run("the refusal is raised where a transient workload becomes possible", func(t *testing.T) { + iops := NewInfrahubOps() + iops.backend = newTestKubernetesBackend() + iops.forbidExternalCapture() + + _, err := iops.externalCaptureOps() + if err == nil { + t.Fatal("externalCaptureOps() = nil, want the refusal: this is the only function a transient object can come from") + } + for _, want := range []string{"namespace infrahub", "infrahub-backup create", "--include-backup"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to name %q (condition, resource and operator action)", err, want) + } + } + }) + + t.Run("the backup a collection delegates stops at the gate, before anything is stopped", func(t *testing.T) { + queries := deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + if service == serviceNeo4j { + return EndpointLocationExternal, nil + } + + return EndpointLocationInternal, nil + }, + discover: func(string) error { return nil }, + } + + iops := NewInfrahubOps() + iops.backend = newTestKubernetesBackend() + iops.forbidExternalCapture() + + err := iops.prepareDatabaseCaptureWith(queries, true) + if err == nil { + t.Fatal("prepareDatabaseCaptureWith() = nil, want the collection refused before the capture path") + } + if !strings.Contains(err.Error(), "infrahub-collect") { + t.Errorf("err = %v, want it to say which tool declined", err) + } + if !strings.Contains(err.Error(), "infrahub-backup create") { + t.Errorf("err = %v, want it to name the tool that does perform the capture", err) + } + }) + + t.Run("a backup run is unaffected", func(t *testing.T) { + iops := NewInfrahubOps() + iops.backend = newTestKubernetesBackend() + + if err := iops.refuseForbiddenExternalCapture("infrahub"); err != nil { + t.Errorf("refuseForbiddenExternalCapture() = %v on a backup run, want nil: only collection is refused", err) + } + }) +} diff --git a/src/internal/app/command_executor.go b/src/internal/app/command_executor.go index db0214e7..a89c212e 100644 --- a/src/internal/app/command_executor.go +++ b/src/internal/app/command_executor.go @@ -65,6 +65,22 @@ func (ce *CommandExecutor) runCommand(name string, args ...string) (string, erro return strings.TrimSpace(string(output)), err } +// runCommandSeparate is runCommand with the output streams kept apart: stdout +// first, stderr second. It is the unbounded sibling of +// runCommandSeparateContext, for the same callers — those that parse stdout as +// data — when no bound applies; see that function for what a merged stream does +// to such a caller. +func (ce *CommandExecutor) runCommandSeparate(name string, args ...string) (string, string, error) { + var stdout, stderr bytes.Buffer + cmd := exec.Command(name, args...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + + return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err +} + // timeoutError marks a command that exceeded its allotted execution time. Its // message is exactly "timed out after " so orchestrators can surface // it verbatim (e.g. as a bundle manifest failure reason). @@ -90,10 +106,32 @@ func formatCommandTimeout(d time.Duration) string { // is killed once timeout elapses (or ctx is cancelled) and the returned error // is a *timeoutError when the timeout expired. func (ce *CommandExecutor) runCommandContext(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + return ce.runBounded(ctx, timeout, false, name, args...) +} + +// runCommandContextKillGroup is runCommandContext for a command whose +// children must not outlive its timeout: the command runs in a process group +// of its own and the whole group is killed when the timeout expires, so a +// `docker compose start` reported as timed out has stopped. On Windows only +// the command itself is killed. The command no longer receives the +// terminal's Ctrl-C, so it is kept to the bounded docker calls of the +// database readiness wait rather than applied to every bounded command. +func (ce *CommandExecutor) runCommandContextKillGroup(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + return ce.runBounded(ctx, timeout, true, name, args...) +} + +func (ce *CommandExecutor) runBounded(ctx context.Context, timeout time.Duration, killGroup bool, name string, args ...string) (string, error) { cctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() cmd := exec.CommandContext(cctx, name, args...) + if killGroup { + killProcessGroupOnCancel(cmd) + } + // Killing the command does not close the output pipe that a child it + // spawned still holds (the docker CLI runs `compose` as a plugin child), so + // without a bound on that wait the call outlives its timeout. + cmd.WaitDelay = commandWaitDelay output, err := cmd.CombinedOutput() if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { return strings.TrimSpace(string(output)), &timeoutError{timeout: timeout} @@ -101,6 +139,10 @@ func (ce *CommandExecutor) runCommandContext(ctx context.Context, timeout time.D return strings.TrimSpace(string(output)), err } +// commandWaitDelay is how long runBounded waits for a killed command's +// output pipe to close before it abandons it. +const commandWaitDelay = 2 * time.Second + // runCommandSeparateContext is the timeout-bounded variant of runCommand that // keeps the command's output streams apart: stdout is returned first, stderr // second. Callers that write a command's stdout to a file need this — merged @@ -126,20 +168,6 @@ func (ce *CommandExecutor) runCommandSeparateContext(ctx context.Context, timeou return out, errOut, err } -// runCommandSeparate is the unbounded variant of runCommandSeparateContext: -// stdout is returned first, stderr second. Callers that parse a command's -// stdout need it, because a warning the command writes to stderr would -// otherwise land in the value they parse. -func (ce *CommandExecutor) runCommandSeparate(name string, args ...string) (string, string, error) { - var stdout, stderr bytes.Buffer - cmd := exec.Command(name, args...) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err -} - // runCommandPipeContext is the timeout-bounded variant of runCommandPipe. The // caller must read from stdout and then call wait() to get the exit status; // wait() returns a *timeoutError when the timeout expired before the command @@ -302,6 +330,49 @@ func (ce *CommandExecutor) runCommandWritePipe(stdin io.Reader, name string, arg return wait, nil } +// runCommandWritePipeContext is the timeout-bounded variant of +// runCommandWritePipe: the command is killed once timeout elapses (or ctx is +// cancelled) and wait() returns a *timeoutError when the timeout expired. The +// internal context is released when wait() is called. +// +// It exists because a manifest piped to `kubectl create` is how a transient +// external-database workload is created, and that call had no bound at all +// (FR-025). +func (ce *CommandExecutor) runCommandWritePipeContext(ctx context.Context, timeout time.Duration, stdin io.Reader, name string, args ...string) (func() error, error) { + cctx, cancel := context.WithTimeout(ctx, timeout) + + cmd := exec.CommandContext(cctx, name, args...) + logrus.Debugf("exec write-pipe (timeout %s): %s %s", timeout, name, strings.Join(args, " ")) + + cmd.Stdin = stdin + + // Capture stderr for error reporting + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + + if err := cmd.Start(); err != nil { + cancel() + return nil, err + } + + wait := func() error { + defer cancel() + if err := cmd.Wait(); err != nil { + if errors.Is(cctx.Err(), context.DeadlineExceeded) { + return &timeoutError{timeout: timeout} + } + stderrStr := strings.TrimSpace(stderrBuf.String()) + if stderrStr != "" { + return fmt.Errorf("%w: %s", err, stderrStr) + } + return err + } + return nil + } + + return wait, nil +} + func (ce *CommandExecutor) runCommandWithStream(name string, args ...string) (string, error) { cmd := exec.Command(name, args...) diff --git a/src/internal/app/command_executor_test.go b/src/internal/app/command_executor_test.go index 5c641760..ec9d9d17 100644 --- a/src/internal/app/command_executor_test.go +++ b/src/internal/app/command_executor_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "io" + "os" + "path/filepath" "strings" "testing" "time" @@ -116,13 +118,27 @@ func TestRunCommandPipeContext_Success(t *testing.T) { } } +// TestRunCommandPipeContext_Timeout is the stdout-only sibling of +// TestRunCommandCombinedPipeContext_Timeout, and carried the identical race: +// the same 100ms shared by process spawn, `exec` and a pipe flush, with the +// same assertion that the output had arrived. It fails under -count=50 for the +// same reason and is made deterministic the same way. func TestRunCommandPipeContext_Timeout(t *testing.T) { + const bound = time.Hour + + sentinel := filepath.Join(t.TempDir(), "written") + parent := &deadlineOnDemand{done: make(chan struct{})} + ce := NewCommandExecutor() - stdout, wait, err := ce.runCommandPipeContext(context.Background(), 100*time.Millisecond, "sh", "-c", "echo started; exec sleep 5") + stdout, wait, err := ce.runCommandPipeContext(parent, bound, + "sh", "-c", `echo started; : > "$0"; exec sleep 300`, sentinel) if err != nil { t.Fatalf("runCommandPipeContext failed to start: %v", err) } + waitForFile(t, sentinel) + parent.expire() + content, err := io.ReadAll(stdout) if err != nil { t.Fatalf("reading stdout failed: %v", err) @@ -139,8 +155,10 @@ func TestRunCommandPipeContext_Timeout(t *testing.T) { if !errors.As(err, &timeout) { t.Fatalf("error = %v (%T), want *timeoutError", err, err) } - if err.Error() != "timed out after 100ms" { - t.Errorf("Error() = %q, want %q", err.Error(), "timed out after 100ms") + // The reported duration is the bound that was configured, not the time the + // command happened to run for. + if want := "timed out after " + formatCommandTimeout(bound); err.Error() != want { + t.Errorf("Error() = %q, want %q", err.Error(), want) } } @@ -182,13 +200,60 @@ func TestRunCommandCombinedPipeContext_ReaderSeesEOFWhenCommandExits(t *testing. } } +// deadlineOnDemand is a context whose deadline the test fires by hand. +// +// It takes the wall clock out of a test whose subject is an ordering rather +// than a duration. Err reports context.DeadlineExceeded because that is what +// the production path reads to tell a bound expiring apart from a command +// failing, and context.WithTimeout over a parent like this one propagates that +// error to the context the command is actually started with. +type deadlineOnDemand struct { + done chan struct{} +} + +func (d *deadlineOnDemand) Deadline() (time.Time, bool) { return time.Time{}, false } +func (d *deadlineOnDemand) Done() <-chan struct{} { return d.done } +func (*deadlineOnDemand) Value(any) any { return nil } + +func (d *deadlineOnDemand) Err() error { + select { + case <-d.done: + return context.DeadlineExceeded + default: + return nil + } +} + +func (d *deadlineOnDemand) expire() { close(d.done) } + +// TestRunCommandCombinedPipeContext_Timeout asserts that output the command +// wrote before its bound expired is still readable afterwards: a killed +// command's diagnostics are most of what makes a timeout reportable. +// +// The ordering is established rather than assumed. The original test gave +// process spawn, `exec` and a pipe flush a shared 100ms budget and then +// asserted the output had arrived, so a loaded machine failed it for losing a +// race rather than for the pipe having lost anything — invisible to a clean run +// and reproducible under -count=25. Here the child announces, through a file, +// that it has finished writing; only then is the deadline fired. Nothing is +// consumed from the pipe until after the kill, so what the assertion reads is +// still what survived it. func TestRunCommandCombinedPipeContext_Timeout(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "written") + parent := &deadlineOnDemand{done: make(chan struct{})} + ce := NewCommandExecutor() - reader, wait, err := ce.runCommandCombinedPipeContext(context.Background(), 100*time.Millisecond, "sh", "-c", "echo started; exec sleep 5") + // The bound is long enough that it cannot be what fires. The deadline under + // test is the parent's, and expire() below is what fires it. + reader, wait, err := ce.runCommandCombinedPipeContext(parent, time.Hour, + "sh", "-c", `echo started; : > "$0"; exec sleep 300`, sentinel) if err != nil { t.Fatalf("runCommandCombinedPipeContext failed to start: %v", err) } + waitForFile(t, sentinel) + parent.expire() + content, err := io.ReadAll(reader) if err != nil { t.Fatalf("reading combined output failed: %v", err) @@ -207,6 +272,26 @@ func TestRunCommandCombinedPipeContext_Timeout(t *testing.T) { } } +// waitForFile blocks until a path exists. It converges rather than budgeting: a +// path that never appears means the child never ran, which is a failure worth +// the test binary's own timeout reporting, not one worth guessing a duration +// for. +func waitForFile(t *testing.T, path string) { + t.Helper() + + for { + if _, err := os.Stat(path); err == nil { + return + } + + select { + case <-t.Context().Done(): + t.Fatalf("the command never created %s, so it never wrote its output", path) + case <-time.After(time.Millisecond): + } + } +} + func TestRunCommandPipeContext_CommandFailureKeepsStderr(t *testing.T) { ce := NewCommandExecutor() stdout, wait, err := ce.runCommandPipeContext(context.Background(), 10*time.Second, "sh", "-c", "echo oops >&2; exit 2") diff --git a/src/internal/app/docker_service_ready.go b/src/internal/app/docker_service_ready.go new file mode 100644 index 00000000..2f0d98cf --- /dev/null +++ b/src/internal/app/docker_service_ready.go @@ -0,0 +1,210 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// databaseReadyTimeout bounds how long a restore waits for the database +// container to come back healthy before it starts the services that depend on +// it. A Neo4j restart after a load is tens of seconds; the bound is for a +// database that never comes back, so it fails the run with the state it saw. +const databaseReadyTimeout = 5 * time.Minute + +// dockerServiceReadyPollInterval is the pause between two readings of the +// container state. +const dockerServiceReadyPollInterval = time.Second + +// dockerServiceReadyConfirmations is how many consecutive ready readings the +// wait needs. One is not enough: right after the Community load resumes the +// suspended Neo4j process, that process finishes the shutdown it was stopped +// in, and for that instant the container still reads as running. +const dockerServiceReadyConfirmations = 2 + +// dockerContainerState is the part of `docker inspect`'s State the readiness +// wait reads. +type dockerContainerState struct { + Status string `json:"Status"` + Running bool `json:"Running"` + Health *struct { + Status string `json:"Status"` + FailingStreak int `json:"FailingStreak"` + } `json:"Health"` +} + +// dockerReadiness says whether every container of a service can satisfy a +// `depends_on: condition: service_healthy` (or `service_started` when the +// service has no healthcheck), and whether one of them has to be started for +// that to ever happen. +// +// A container whose health still reads "healthy" while its failing streak is +// above zero is not ready: Docker keeps the last status until the retries run +// out, so a Neo4j that was down for the length of a load reads "healthy" with +// a streak of failed checks behind it, and that reading is stale. +func dockerReadiness(states []dockerContainerState) (ready bool, needsStart bool, detail string) { + if len(states) == 0 { + return false, false, "no container" + } + ready = true + details := make([]string, 0, len(states)) + for _, st := range states { + switch { + case st.Status == "exited" || st.Status == "created" || st.Status == "dead": + ready = false + needsStart = true + details = append(details, st.Status) + case !st.Running: + ready = false + details = append(details, st.Status) + case st.Health == nil: + details = append(details, "running") + case st.Health.Status == "healthy" && st.Health.FailingStreak == 0: + details = append(details, "healthy") + default: + ready = false + details = append(details, fmt.Sprintf("%s (failing streak %d)", st.Health.Status, st.Health.FailingStreak)) + } + } + return ready, needsStart, strings.Join(details, ", ") +} + +// waitForDockerServiceReady polls observe until dockerReadiness reports the +// service ready on dockerServiceReadyConfirmations consecutive readings, or +// timeout passes. A container that exited and that its restart policy does not +// bring back is started once per such reading, because `docker compose start` +// of a dependent refuses an exited dependency instead of starting it. +// +// observe and start each receive the time left before the deadline and must +// not run past it: a Docker CLI or daemon that hangs would otherwise hold the +// wait past its bound, because the deadline is only checked between readings. +func waitForDockerServiceReady( + service string, + observe func(limit time.Duration) ([]dockerContainerState, error), + start func(limit time.Duration) error, + timeout, interval time.Duration, +) error { + deadline := time.Now().Add(timeout) + confirmed := 0 + last := "not read" + for { + remaining := time.Until(deadline) + if remaining <= 0 { + return fmt.Errorf("%s did not become healthy within %s (last state: %s)", service, timeout, last) + } + states, err := observe(remaining) + if err != nil { + confirmed = 0 + last = err.Error() + } else { + ready, needsStart, detail := dockerReadiness(states) + last = detail + switch { + case ready: + confirmed++ + if confirmed >= dockerServiceReadyConfirmations { + return nil + } + case needsStart: + confirmed = 0 + logrus.Infof("Starting %s, which is %s...", service, detail) + if err := start(time.Until(deadline)); err != nil { + last = fmt.Sprintf("%s; start failed: %v", detail, err) + } + default: + confirmed = 0 + } + } + if !time.Now().Before(deadline) { + return fmt.Errorf("%s did not become healthy within %s (last state: %s)", service, timeout, last) + } + time.Sleep(interval) + } +} + +// serviceContainerStates reads the State of every container of one compose +// service, stopped ones included. Every docker call it makes, together, ends +// within limit; a call cut short returns an error naming that call. +func (d *DockerBackend) serviceContainerStates(service string, limit time.Duration) ([]dockerContainerState, error) { + deadline := time.Now().Add(limit) + output, err := d.executor.runCommandContextKillGroup(context.Background(), limit, "docker", d.composeArgs("ps", "-a", "-q", service)...) + if err != nil { + return nil, fmt.Errorf("docker compose ps %s: %w", service, composeLifecycleError(output, err)) + } + ids := strings.Fields(output) + states := make([]dockerContainerState, 0, len(ids)) + for _, id := range ids { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil, fmt.Errorf("docker inspect %s: %w", id, &timeoutError{timeout: limit}) + } + out, err := d.executor.runCommandContextKillGroup(context.Background(), remaining, "docker", "inspect", "--format", "{{json .State}}", id) + if err != nil { + return nil, fmt.Errorf("docker inspect %s: %w", id, composeLifecycleError(out, err)) + } + var st dockerContainerState + if err := json.Unmarshal([]byte(out), &st); err != nil { + return nil, fmt.Errorf("failed to parse state of container %s: %w", id, err) + } + states = append(states, st) + } + return states, nil +} + +// WaitServiceReady waits until every container of service is running and, +// where it has a healthcheck, freshly healthy. +func (d *DockerBackend) WaitServiceReady(service string, timeout time.Duration) error { + return waitForDockerServiceReady( + service, + func(limit time.Duration) ([]dockerContainerState, error) { + return d.serviceContainerStates(service, limit) + }, + func(limit time.Duration) error { return d.startWithin(limit, service) }, + timeout, dockerServiceReadyPollInterval, + ) +} + +// startWithin is Start bounded by limit: a `docker compose start` still +// running when limit passes is killed and reported as timed out. +func (d *DockerBackend) startWithin(limit time.Duration, service string) error { + if limit <= 0 { + return fmt.Errorf("docker compose start %s: %w", service, &timeoutError{timeout: limit}) + } + output, err := d.executor.runCommandContextKillGroup(context.Background(), limit, "docker", d.composeArgs("start", service)...) + if err != nil { + return fmt.Errorf("docker compose start %s: %w", service, composeLifecycleError(output, err)) + } + return nil +} + +// startAppServicesAfterRestore brings infrahub-server and task-worker back at +// the end of a restore, once the database they depend on is ready. +// +// On Docker, Infrahub's compose file declares infrahub-server +// `depends_on: database: condition: service_healthy`, and `docker compose +// start` checks that condition once and refuses — "dependency failed to start: +// container ... exited" — rather than waiting through it. The Community load +// leaves Neo4j finishing a shutdown that its restart policy then undoes, so the +// start raced the database's restart. Kubernetes has no such gate: scaling the +// deployments up is accepted whatever state the database is in. +func (iops *InfrahubOps) startAppServicesAfterRestore() error { + backend, err := iops.ensureBackend() + if err != nil { + return err + } + if docker, ok := backend.(*DockerBackend); ok { + logrus.Info("Waiting for the database to be healthy...") + if err := docker.WaitServiceReady(serviceNeo4j, databaseReadyTimeout); err != nil { + return fmt.Errorf("failed to restart infrahub services: %w", err) + } + } + logrus.Info("Restarting Infrahub services...") + if err := iops.StartServices("infrahub-server", "task-worker"); err != nil { + return fmt.Errorf("failed to restart infrahub services: %w", err) + } + return nil +} diff --git a/src/internal/app/docker_service_ready_test.go b/src/internal/app/docker_service_ready_test.go new file mode 100644 index 00000000..0fa7cad7 --- /dev/null +++ b/src/internal/app/docker_service_ready_test.go @@ -0,0 +1,203 @@ +package app + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func runningWithHealth(status string, streak int) dockerContainerState { + st := dockerContainerState{Status: "running", Running: true} + st.Health = &struct { + Status string `json:"Status"` + FailingStreak int `json:"FailingStreak"` + }{Status: status, FailingStreak: streak} + return st +} + +func TestDockerReadiness(t *testing.T) { + cases := []struct { + name string + states []dockerContainerState + ready bool + needStart bool + }{ + {"no container", nil, false, false}, + {"healthy", []dockerContainerState{runningWithHealth("healthy", 0)}, true, false}, + {"stale healthy after an outage", []dockerContainerState{runningWithHealth("healthy", 4)}, false, false}, + {"starting", []dockerContainerState{runningWithHealth("starting", 0)}, false, false}, + {"unhealthy", []dockerContainerState{runningWithHealth("unhealthy", 20)}, false, false}, + {"restarting", []dockerContainerState{{Status: "restarting"}}, false, false}, + {"exited", []dockerContainerState{{Status: "exited"}}, false, true}, + {"running without healthcheck", []dockerContainerState{{Status: "running", Running: true}}, true, false}, + {"one replica not ready", []dockerContainerState{runningWithHealth("healthy", 0), {Status: "restarting"}}, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ready, needStart, _ := dockerReadiness(tc.states) + if ready != tc.ready || needStart != tc.needStart { + t.Fatalf("dockerReadiness = (%v, %v), want (%v, %v)", ready, needStart, tc.ready, tc.needStart) + } + }) + } +} + +// The sequence the Community streamed load leaves behind on Docker: the +// resumed Neo4j finishes its shutdown, the container exits, its restart policy +// (or the wait) starts it again, and its healthcheck passes some time later. +// Starting infrahub-server at any point before the end of the sequence is +// what compose refuses. +func TestWaitForDockerServiceReadyWaitsThroughARestart(t *testing.T) { + readings := [][]dockerContainerState{ + {runningWithHealth("healthy", 5)}, // stale: Neo4j was down for the load + {{Status: "exited"}}, + {{Status: "restarting"}}, + {runningWithHealth("starting", 0)}, + {runningWithHealth("healthy", 0)}, + {runningWithHealth("healthy", 0)}, + } + calls, starts := 0, 0 + observe := func(time.Duration) ([]dockerContainerState, error) { + r := readings[calls] + if calls < len(readings)-1 { + calls++ + } + return r, nil + } + start := func(time.Duration) error { starts++; return nil } + + if err := waitForDockerServiceReady("database", observe, start, time.Minute, time.Millisecond); err != nil { + t.Fatalf("waitForDockerServiceReady = %v, want nil", err) + } + if calls != len(readings)-1 { + t.Fatalf("returned after %d readings, want all %d", calls+1, len(readings)) + } + if starts != 1 { + t.Fatalf("start called %d times, want 1 (for the exited reading)", starts) + } +} + +func TestWaitForDockerServiceReadyNeedsConsecutiveReadings(t *testing.T) { + readings := [][]dockerContainerState{ + {runningWithHealth("healthy", 0)}, + {{Status: "restarting"}}, + {runningWithHealth("healthy", 0)}, + {runningWithHealth("healthy", 0)}, + } + calls := 0 + observe := func(time.Duration) ([]dockerContainerState, error) { + r := readings[calls] + calls++ + return r, nil + } + if err := waitForDockerServiceReady("database", observe, func(time.Duration) error { return nil }, time.Minute, time.Millisecond); err != nil { + t.Fatalf("waitForDockerServiceReady = %v, want nil", err) + } + if calls != 4 { + t.Fatalf("observed %d times, want 4", calls) + } +} + +func TestWaitForDockerServiceReadyTimesOutWithLastState(t *testing.T) { + observe := func(time.Duration) ([]dockerContainerState, error) { + return []dockerContainerState{runningWithHealth("unhealthy", 20)}, nil + } + err := waitForDockerServiceReady("database", observe, func(time.Duration) error { return nil }, 20*time.Millisecond, time.Millisecond) + if err == nil { + t.Fatal("waitForDockerServiceReady = nil, want a timeout error") + } + if !strings.Contains(err.Error(), "database did not become healthy") || !strings.Contains(err.Error(), "unhealthy (failing streak 20)") { + t.Fatalf("error = %q, want the service and its last state", err) + } +} + +func TestWaitForDockerServiceReadyRetriesObserveErrors(t *testing.T) { + calls := 0 + observe := func(time.Duration) ([]dockerContainerState, error) { + calls++ + if calls == 1 { + return nil, errors.New("no such container") + } + return []dockerContainerState{runningWithHealth("healthy", 0)}, nil + } + if err := waitForDockerServiceReady("database", observe, func(time.Duration) error { return nil }, time.Minute, time.Millisecond); err != nil { + t.Fatalf("waitForDockerServiceReady = %v, want nil", err) + } +} + +func TestComposeLifecycleError(t *testing.T) { + if err := composeLifecycleError("whatever", nil); err != nil { + t.Fatalf("composeLifecycleError(_, nil) = %v, want nil", err) + } + base := errors.New("exit status 1") + if err := composeLifecycleError(" ", base); err != base { + t.Fatalf("composeLifecycleError(blank, err) = %v, want err unchanged", err) + } + err := composeLifecycleError("dependency failed to start: container db exited (0)\n", base) + if !errors.Is(err, base) || err.Error() != "exit status 1: dependency failed to start: container db exited (0)" { + t.Fatalf("composeLifecycleError = %q, want the output appended to the wrapped error", err) + } +} + +// A `docker compose start` that fails must say why: the output compose +// printed is part of the error, not discarded. +func TestDockerBackendStartIncludesComposeOutput(t *testing.T) { + bin := t.TempDir() + script := "#!/bin/sh\necho 'dependency failed to start: container infrahub-database-1 exited (0)'\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(script), 0o755); err != nil { + t.Fatalf("writing fake docker = %v", err) + } + t.Setenv("PATH", bin) + + d := NewDockerBackend(&Configuration{}, NewCommandExecutor()) + for name, op := range map[string]func(...string) error{"Start": d.Start, "Stop": d.Stop} { + err := op("infrahub-server") + if err == nil || !strings.Contains(err.Error(), "container infrahub-database-1 exited (0)") { + t.Fatalf("%s error = %v, want it to carry the compose output", name, err) + } + } +} + +// A docker call that hangs must not hold the wait past its bound: each call +// gets only the time left, and the error names the call that ran out. +func TestDockerBackendWaitServiceReadyBoundsHungDockerCalls(t *testing.T) { + bin := t.TempDir() + // Not `exec sleep`: the sleep is a child that keeps the output pipe open + // after the killed shell, as the compose plugin does under the docker CLI. + script := "#!/bin/sh\nsleep 30\n" + if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(script), 0o755); err != nil { + t.Fatalf("writing fake docker = %v", err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + d := NewDockerBackend(&Configuration{}, NewCommandExecutor()) + began := time.Now() + err := d.WaitServiceReady("database", 300*time.Millisecond) + if elapsed := time.Since(began); elapsed > 10*time.Second { + t.Fatalf("WaitServiceReady returned after %s, want it bounded by its timeout", elapsed) + } + if err == nil || !strings.Contains(err.Error(), "docker compose ps database") || !strings.Contains(err.Error(), "timed out after") { + t.Fatalf("error = %v, want a timeout naming docker compose ps", err) + } +} + +func TestDockerBackendStartWithinBoundsHungStart(t *testing.T) { + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "docker"), []byte("#!/bin/sh\nsleep 30\n"), 0o755); err != nil { + t.Fatalf("writing fake docker = %v", err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + + d := NewDockerBackend(&Configuration{}, NewCommandExecutor()) + began := time.Now() + err := d.startWithin(300*time.Millisecond, "database") + if elapsed := time.Since(began); elapsed > 10*time.Second { + t.Fatalf("startWithin returned after %s, want it bounded by its limit", elapsed) + } + if err == nil || !strings.Contains(err.Error(), "docker compose start database") || !strings.Contains(err.Error(), "timed out after") { + t.Fatalf("error = %v, want a timeout naming docker compose start", err) + } +} diff --git a/src/internal/app/environment.go b/src/internal/app/environment.go index 0543878b..1d51ad23 100644 --- a/src/internal/app/environment.go +++ b/src/internal/app/environment.go @@ -13,6 +13,23 @@ var ErrCLIUnavailable = errors.New("CLI not available") type ExecOptions struct { User string Env map[string]string + + // Pod names the exact pod to run in, instead of resolving one from the + // service. It is empty for every path that has a deployment to resolve + // against, which is every path that exists today. + // + // It exists for a workload this run created and therefore already knows the + // name of. The alternative — registering that pod under the service's name + // so the ordinary resolver finds it — is what the transient + // external-database workload used to do, and Start and scaleServices + // replace the resolver's cache wholesale. A restore calls StartServices by + // design, so the execution target de-registered itself mid-run and the next + // copy resolved to nothing, against a pod that was still ready. + // + // Kubernetes-only. The Docker backend addresses a Compose service and has + // no pod to name, so it ignores this field; nothing outside the Kubernetes + // external-database path sets it. + Pod string } type EnvironmentBackend interface { @@ -30,6 +47,28 @@ type EnvironmentBackend interface { IsRunning(service string) (bool, error) } +// runtimeNamer is a backend that can name the command-line tool it shells out +// to. +// +// It is an optional capability rather than a method on EnvironmentBackend so +// that a test double is not obliged to answer it — the same idiom serviceLocator +// and separateExecer follow. Both real backends implement it, which is what +// makes FR-018's "name what is missing" answerable from the detection loop. +type runtimeNamer interface { + RuntimeCommand() string +} + +// runtimeCommandOf names the tool a backend needs, falling back to the +// backend's own name where it does not say. The fallback is a less precise +// message, never a missing one. +func runtimeCommandOf(backend EnvironmentBackend) string { + if namer, ok := backend.(runtimeNamer); ok { + return namer.RuntimeCommand() + } + + return backend.Name() +} + // Shared utility functions func nonEmptyLines(output string) []string { @@ -43,6 +82,21 @@ func nonEmptyLines(output string) []string { return lines } +// commaFields is nonEmptyLines for a comma-separated operator-supplied list: +// the entries, trimmed, with empty ones dropped so a trailing comma is not an +// entry. It exists because the flag parsers each opened with the same loop and +// a change to that rule had to land in every one of them. +func commaFields(text string) []string { + fields := []string{} + for _, field := range strings.Split(text, ",") { + if trimmed := strings.TrimSpace(field); trimmed != "" { + fields = append(fields, trimmed) + } + } + + return fields +} + func contains(values []string, target string) bool { for _, value := range values { if value == target { diff --git a/src/internal/app/environment_docker.go b/src/internal/app/environment_docker.go index 7566e740..e8265ff1 100644 --- a/src/internal/app/environment_docker.go +++ b/src/internal/app/environment_docker.go @@ -31,6 +31,10 @@ func (d *DockerBackend) Info() string { return d.project } +// RuntimeCommand names the tool this backend shells out to, so a run that found +// neither runtime can say which ones it looked for (FR-018). +func (*DockerBackend) RuntimeCommand() string { return "docker" } + func (d *DockerBackend) Detect() error { if err := d.executor.runCommandQuiet("docker", "--version"); err != nil { // If user explicitly specified Docker project, this is a hard error @@ -471,8 +475,7 @@ func (d *DockerBackend) Start(services ...string) error { } args := append([]string{"start"}, services...) cmd := d.composeArgs(args...) - _, err := d.executor.runCommand("docker", cmd...) - return err + return composeLifecycleError(d.executor.runCommand("docker", cmd...)) } func (d *DockerBackend) Stop(services ...string) error { @@ -481,7 +484,20 @@ func (d *DockerBackend) Stop(services ...string) error { } args := append([]string{"stop"}, services...) cmd := d.composeArgs(args...) - _, err := d.executor.runCommand("docker", cmd...) + return composeLifecycleError(d.executor.runCommand("docker", cmd...)) +} + +// composeLifecycleError carries what `docker compose start|stop` printed into +// the error it returns. Compose explains a refusal only in its output — "dependency +// failed to start: container ... exited (0)" — and an exit status alone left a +// failed restore reporting nothing but "exit status 1". +func composeLifecycleError(output string, err error) error { + if err == nil { + return nil + } + if output = strings.TrimSpace(output); output != "" { + return fmt.Errorf("%w: %s", err, output) + } return err } @@ -489,6 +505,14 @@ func (d *DockerBackend) IsRunning(service string) (bool, error) { cmd := d.composeArgs("ps", service) output, err := d.executor.runCommand("docker", cmd...) if err != nil { + // A service the compose project does not define is not running, which + // is what an optional service (task-manager-background-svc on an older + // stack) looks like. stopAppContainers fails the run on an error, so + // reporting this one as an error would refuse every backup and restore + // of such a stack. + if strings.Contains(output, "no such service") { + return false, nil + } return false, err } return strings.Contains(output, "Up"), nil diff --git a/src/internal/app/environment_kubernetes.go b/src/internal/app/environment_kubernetes.go index c69d8230..06d96a2e 100644 --- a/src/internal/app/environment_kubernetes.go +++ b/src/internal/app/environment_kubernetes.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "slices" "sort" "strconv" "strings" @@ -20,17 +21,52 @@ import ( // errors as collector failures instead of masking them (FIX-2). var errNoPodsMatched = errors.New("no pods matched the service") -// podRunner executes a kubectl command and returns its trimmed output. The +// podRunner executes a kubectl command and returns its trimmed stdout. The // shared pod-resolution helpers (getPodForService, GetAllPods — used by the -// backup tool) pass the unbounded executor.runCommand; the collect primitives -// pass a timeout-bounded runner so a wedged API server cannot hang the bundle -// run before the exec/copy timeout even applies (research R2, FIX-5). +// backup tool) pass the unbounded stdoutRunner; the collect primitives pass a +// timeout-bounded runner so a wedged API server cannot hang the bundle run +// before the exec/copy timeout even applies (research R2, FIX-5). Both read +// stdout alone, for the reason separatedRunner records. type podRunner func(name string, args ...string) (string, error) +// stdoutRunner is the unbounded podRunner: separatedRunner's answer with no +// bound on the call, for the backup tool's own path. The four resolvers that +// used to pass the merged executor.runCommand here parse the same JSONPath +// values the bounded ones do, so a kubectl notice corrupted them the same way; +// there is one answer to where a notice lands, and this is how the unbounded +// path gives it. +func (k *KubernetesBackend) stdoutRunner() podRunner { + return func(name string, args ...string) (string, error) { + stdout, stderr, err := k.executor.runCommandSeparate(name, args...) + + return stdout, withStderr(err, stderr) + } +} + // boundedRunner returns a podRunner that bounds every kubectl call by timeout. func (k *KubernetesBackend) boundedRunner(ctx context.Context, timeout time.Duration) podRunner { + return separatedRunner(ctx, k.executor, timeout) +} + +// separatedRunner is the one podRunner that reads the executor under a bound, +// and it answers with the command's stdout alone. +// +// Every consumer of a podRunner parses what comes back as data — a pod name +// from a JSONPath listing, a phase, the UID a credential Secret's ownerReference +// is built from — and kubectl writes its own notices to stderr on a healthy +// call: a deprecation warning, a kubeconfig complaint, a plugin's chatter. +// Merged output puts that notice in front of the value, so the UID field holds +// `"\n"`, the ownerReference matches no live pod, and the Secret +// holding the database password outlives the run (FR-023). Stdout alone is the +// value; stderr goes where a failure's reason belongs, into the error +// (withStderr), and is dropped on success. Both bounded runner constructors — +// this backend's boundedRunner and the external path's boundedExternalRunner — +// come through here, so there is one answer to where a kubectl notice lands. +func separatedRunner(ctx context.Context, executor *CommandExecutor, timeout time.Duration) podRunner { return func(name string, args ...string) (string, error) { - return k.executor.runCommandContext(ctx, timeout, name, args...) + stdout, stderr, err := executor.runCommandSeparateContext(ctx, timeout, name, args...) + + return stdout, withStderr(err, stderr) } } @@ -40,17 +76,101 @@ type KubernetesBackend struct { namespace string podCache map[string]string replicaCache map[string]int // stores original replica counts before stopping + + // workloadListings memoises the namespace-wide workload listing per kind. + // + // It survives a scale, unlike the pod caches, because of what it reads: a + // workload's name, its selector labels and its pod template's labels. A + // scale changes a replica count and none of those. Both readers — the + // resolver that decides which workload to scale and the location decision + // — ask the same question of the same two kinds, so without this a run + // locating two databases and resolving six app services pays a dozen + // identical full-namespace listings. + workloadListings map[string][]kubernetesWorkload + + // workloads memoises the workload each service resolved to. Stop, + // scaleServices and Start each resolve the same service in turn, and one + // resolution is up to ten kubectl calls; a scale changes a workload's + // replica count and nothing about its kind or name, so unlike the pod + // caches this survives one. + workloads map[string]resolvedWorkload + + // serviceLocations memoises the internal-versus-external decision per + // service. + // + // Every path that touches a database resolves through a databaseTarget, and + // each of those asks the question again — so a run that captures two + // databases and reads their metadata sweeps up to four label selectors plus + // a workload query per operation, against an answer that cannot change + // while the run lasts. A database does not move out of the deployment + // mid-backup; where its pods are is a different question, and that is what + // podCache holds and what Start and scaleServices clear. + // + // It is deliberately not cleared alongside podCache for that reason: a + // restore scales the deployment down and back up, which changes which pods + // exist and changes nothing about whether the database is one of them. + serviceLocations map[string]EndpointLocation + + // namespacePodListings memoises the namespace-wide ownership listing, keyed + // by the field selector it was issued with. + // + // It is not a second resolution cache: podCache answers "which pod is this + // service's?", and this holds the pods kubectl listed for a question that + // names no service at all — parsed once, with the transient workloads + // already excluded, and read as-is by every hit. Two call sites ask it — + // the fallback pod resolver and the location decision — once per service + // each, and the answer cannot differ between them, because the query does + // not mention the service. + // + // Both of those ask *which pods exist*, and that is the whole of what this + // memo may answer. It must never answer *what those pods are doing*: a + // phase is read precisely because it is expected to change, and a memo + // there is not a saved call but a frozen answer. The running check + // therefore goes through freshNamespacePodListing, which touches this map + // neither to read nor to write — see the doc block there for what a + // replayed phase did to the quiesce and restart waits. + // + // It is cleared wherever podCache is, and for the same reason: a scale + // changes which pods exist. That is also its limit — the stop loop clears + // it each time it actually stops something — so it saves the repeats within + // one burst of resolution, not across a mutation. + namespacePodListings map[string][]labelledPod + + // transientWorkloads are the transient external-database workloads this + // backend created, tracked from the moment their pod exists so a run's exit + // path removes every one of them with a single call however many it created + // (FR-011). A run creates two per external database — a probe and then the + // capture — and the probe is the one a failure between them would otherwise + // leak. + transientWorkloads []*TransientWorkload } func NewKubernetesBackend(config *Configuration, executor *CommandExecutor) *KubernetesBackend { return &KubernetesBackend{ - config: config, - executor: executor, - podCache: map[string]string{}, - replicaCache: map[string]int{}, + config: config, + executor: executor, + podCache: map[string]string{}, + replicaCache: map[string]int{}, + workloadListings: map[string][]kubernetesWorkload{}, + workloads: map[string]resolvedWorkload{}, + serviceLocations: map[string]EndpointLocation{}, + namespacePodListings: map[string][]labelledPod{}, } } +// dropPodCaches forgets where the deployment's pods are. +// +// Both caches describe which pods exist — podCache the one resolved per +// service, namespacePodListings the listing that answer came out of — and a +// scale is exactly what changes that. They are cleared together so a caller +// cannot invalidate one and read the other. serviceLocations is deliberately +// not among them: a restore scales the deployment down and back up, which +// changes which pods exist and nothing about whether a database is one of them. +func (k *KubernetesBackend) dropPodCaches() { + k.podCache = map[string]string{} + k.namespacePodListings = map[string][]labelledPod{} +} + func (k *KubernetesBackend) Name() string { return "kubernetes" } @@ -59,6 +179,10 @@ func (k *KubernetesBackend) Info() string { return k.namespace } +// RuntimeCommand names the tool this backend shells out to; see +// DockerBackend.RuntimeCommand. +func (*KubernetesBackend) RuntimeCommand() string { return "kubectl" } + func (k *KubernetesBackend) Detect() error { if err := k.executor.runCommandQuiet("kubectl", "version", "--client"); err != nil { // If user explicitly specified K8s namespace, this is a hard error @@ -71,8 +195,11 @@ func (k *KubernetesBackend) Detect() error { if k.config.K8sNamespace != "" { k.namespace = k.config.K8sNamespace - if _, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-l", "app.kubernetes.io/name=infrahub"); err != nil { - return fmt.Errorf("failed to verify namespace %s: %w", k.namespace, err) + if output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-l", "app.kubernetes.io/name=infrahub"); err != nil { + // kubectl's own message is the only place a refusal ("forbidden") or a + // missing namespace is stated, so it is carried rather than reduced to + // an exit status. + return fmt.Errorf("failed to verify namespace %s: listing its pods failed (%w): %s; check --k8s-namespace and that this identity can list pods there", k.namespace, err, output) } return nil } @@ -94,9 +221,41 @@ func (k *KubernetesBackend) Detect() error { } } +// targetedPod returns the pod the options name, or "" when they name none. See +// ExecOptions.Pod for why naming one exists at all. +func targetedPod(opts *ExecOptions) string { + if opts == nil { + return "" + } + + return strings.TrimSpace(opts.Pod) +} + +// execPod is the pod a command runs in: the one the options name, or the one +// the service resolves to. Naming a pod is resolution that already happened, +// not a second way of resolving — so it is read first and the resolver is not +// consulted at all. +func (k *KubernetesBackend) execPod(service string, opts *ExecOptions) (string, error) { + if pod := targetedPod(opts); pod != "" { + return pod, nil + } + + return k.getPodForService(service) +} + +// execPodContext is execPod under a bound, for the collect primitives and every +// external-database operation. +func (k *KubernetesBackend) execPodContext(ctx context.Context, timeout time.Duration, service string, opts *ExecOptions) (string, error) { + if pod := targetedPod(opts); pod != "" { + return pod, nil + } + + return k.getPodForServiceContext(ctx, timeout, service) +} + // buildExecArgs resolves the pod and constructs kubectl exec arguments. func (k *KubernetesBackend) buildExecArgs(service string, command []string, opts *ExecOptions) ([]string, error) { - pod, err := k.getPodForService(service) + pod, err := k.execPod(service, opts) if err != nil { return nil, err } @@ -131,7 +290,7 @@ func (k *KubernetesBackend) ExecStreamPipe(service string, command []string, opt } func (k *KubernetesBackend) ExecWritePipe(service string, command []string, opts *ExecOptions, stdin io.Reader) (func() error, error) { - pod, err := k.getPodForService(service) + pod, err := k.execPod(service, opts) if err != nil { return nil, err } @@ -146,11 +305,8 @@ func (k *KubernetesBackend) CopyTo(service, src, dest string) error { if err != nil { return err } - target := fmt.Sprintf("%s/%s:%s", k.namespace, pod, dest) - if _, err := k.executor.runCommand("kubectl", "cp", src, target); err != nil { - return err - } - return nil + + return k.CopyToPod(pod, src, dest) } func (k *KubernetesBackend) CopyFrom(service, src, dest string) error { @@ -158,10 +314,38 @@ func (k *KubernetesBackend) CopyFrom(service, src, dest string) error { if err != nil { return err } - source := fmt.Sprintf("%s/%s:%s", k.namespace, pod, src) - if _, err := k.executor.runCommand("kubectl", "cp", source, dest); err != nil { + + return k.CopyFromPod(pod, src, dest) +} + +// CopyToPod and CopyFromPod are CopyTo and CopyFrom against a pod the caller +// already knows the name of. They are what ExecOptions.Pod is for the exec +// primitives: a copy is the one operation that takes no options, so the +// alternative to naming the pod here would be registering it under the +// service's name — which is the registration ExecOptions.Pod exists to +// replace, and which a restore's own StartServices call silently undid. +// +// CopyTo and CopyFrom resolve a pod and then call these, so there is one place +// a `kubectl cp` argument is built either way. +func (k *KubernetesBackend) CopyToPod(pod, src, dest string) error { + if _, err := k.executor.runCommand("kubectl", "cp", src, k.podPath(pod, dest)); err != nil { return err } + + return nil +} + +// podPath is the `/:` argument every `kubectl cp` takes, +// built in one place so the four copy primitives cannot disagree about it. +func (k *KubernetesBackend) podPath(pod, path string) string { + return fmt.Sprintf("%s/%s:%s", k.namespace, pod, path) +} + +func (k *KubernetesBackend) CopyFromPod(pod, src, dest string) error { + if _, err := k.executor.runCommand("kubectl", "cp", k.podPath(pod, src), dest); err != nil { + return err + } + return nil } @@ -181,7 +365,7 @@ func (k *KubernetesBackend) Start(services ...string) error { return fmt.Errorf("failed to scale %s (%s/%s) to %d replicas: %w", service, kind, resource, replicas, err) } } - k.podCache = map[string]string{} + k.dropPodCaches() return nil } @@ -247,11 +431,8 @@ func (k *KubernetesBackend) IsRunning(service string) (bool, error) { // kubectl's stdout only, so a warning kubectl writes to stderr does not make a // successful read fail. func (k *KubernetesBackend) getReplicaCount(kind, resource string) (int, error) { - output, errOutput, err := k.executor.runCommandSeparate("kubectl", "get", kind, resource, "-n", k.namespace, "-o", "jsonpath={.spec.replicas}") + output, err := k.stdoutRunner()("kubectl", "get", kind, resource, "-n", k.namespace, "-o", "jsonpath={.spec.replicas}") if err != nil { - if line := commandErrorLine(errOutput); line != "" { - return 0, fmt.Errorf("%w: %s", err, line) - } return 0, err } count, err := strconv.Atoi(output) @@ -262,43 +443,240 @@ func (k *KubernetesBackend) getReplicaCount(kind, resource string) (int, error) } func (k *KubernetesBackend) getPodStatuses(service string) ([]string, error) { - selectors := k.podSelectors(service) - for _, selector := range selectors { - output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.status.phase}{\"\\n\"}{end}") - if err != nil { + return k.getPodStatusesWith(k.stdoutRunner(), service) +} + +// getPodStatusesWith is getPodStatuses against the supplied runner, following +// the package's injection idiom so the running check is testable without a +// cluster. +// +// IsRunning has one caller that acts destructively on the answer — +// stopAppContainers, deciding whether to stop and scale a workload to zero — so +// this is a destructive decision dressed as a query. The quiesce and restart +// waits read it too, for the transition rather than the decision (see +// freshNamespacePodListing), so its contract belongs to both: narrowing or +// widening what counts as running changes which workloads get stopped *and* +// what those two loops wait for. One namespace listing answers it, and the +// three answers it can give are drawn apart accordingly: +// +// - the pods a label selector would have matched: they declared themselves, +// and their phases are the answer (podsServiceSelectorsWouldMatch). +// - nothing a selector would have matched: the listing decides on the +// ownership evidence it carries. `redis-cache`, `othertool-cache-0` and +// `rabbitmq-message-queue` are not this deployment's, so their phases are +// not this service's status — the shipped `strings.Contains(name, service)` +// reported all three as Infrahub's `cache`, and stopAppContainers then +// stopped them. +// - the listing failed: the question was not answered. An error, never an +// empty answer, because a caller that cannot determine whether a workload +// is running must not stop it (FR-002). +// +// The four `=` queries this used to issue are gone, and with them +// the only failure mode T097 was written about. They asked the cluster for the +// very fields the listing already carries, so they are now reproduced from it — +// exactly, in selector order, on the same evidence a selector matches +// (podsServiceSelectorsWouldMatch). What T097 fixed was a *throttled selector +// call* turning a determinable status into an error, on the one topology that +// matters: podSelectors queries `=infrahub-server`, the chart labels those +// pods `infrahub/service=server`, so on a Helm deployment the listing was +// already the only thing that ever matched `infrahub-server`. There is no +// selector call left to fail, which is the strongest form that property takes +// — and correspondingly the one query's failure is now the whole of the +// undetermined status, reported as such rather than as an empty answer. +// +// Reproducing the selector matches, rather than reading the whole listing +// through the ownership rule, is what keeps this from becoming a *broader* +// running check than the one it replaces: ownership also claims a pod carrying +// a name this deployment's release prefixes claim, and IsRunning answering true +// from one of those is a stop and a scale to zero on evidence no selector ever +// produced. +// +// A listing that names no matching pod is a real "not running", which is what +// an absent optional service looks like, so it stays an empty answer. +// +// The query is fresh every time. IsRunning is what the quiesce and restart +// waits poll in a sleep loop with nothing mutating between iterations, so an +// answer replayed from a memo would make the transition they are waiting for +// unobservable however long they waited (see freshNamespacePodListing). +func (k *KubernetesBackend) getPodStatusesWith(run podRunner, service string) ([]string, error) { + // Read fresh every time, because a phase is what the callers above are + // waiting to see change (see freshNamespacePodListing). + pods, err := k.freshNamespacePodListing(run) + if err != nil { + return nil, fmt.Errorf("failed to determine whether %s is running in namespace %s: %w", service, k.namespace, err) + } + + if declared := podsServiceSelectorsWouldMatch(pods, k.podSelectors(service)); len(declared) > 0 { + return podPhases(declared), nil + } + + prefixes := releasePrefixes(declaredPodServices(pods)) + statuses := []string{} + for _, pod := range pods { + // The same rule deploymentClaimsService decides the location on, under + // the same policy: this is only ever asked about the app services (see + // appServicesStoppedForBackup), for which an unanchored prefixed name + // still claims, because the cost of narrowing is a workload left + // running through an offline capture. + if deploymentClaimsResource(prefixes, pod.Name, pod.Service, service) { + statuses = append(statuses, pod.Phase) + continue } - statuses := nonEmptyLines(output) - if len(statuses) > 0 { - return statuses, nil - } + logUnclaimedResource("pod", pod.Name, pod.Service, service, prefixes) } - // Fallback to all pods search - output, err := k.executor.runCommand("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\";\"}{.status.phase}{\"\\n\"}{end}") + + return statuses, nil +} + +// namespacePodListing answers "which pods exist?" from the namespace-wide +// ownership listing, replaying the answer for a field selector it has already +// been asked about. +// +// Two call sites ask it — the fallback pod resolver (claimedPodNamesWith) and +// the location decision (deploymentClaimsService) — identically, once per +// service they were asked about. The query names no service, so its answer +// cannot differ between them, and both questions are settled for as long as the +// pods are: which pod is a service's, and whether the deployment holds one at +// all. That is what makes the memo sound, and it is the only kind of question +// it may be asked. The running check is not one of them and must not become +// one — see freshNamespacePodListing. +// +// The memo is per backend and dropped by every scale, so a caller that is about +// to change which pods exist never reads an answer from before it did. A listing +// that fails is not memoised: it says the cluster could not answer, not that the +// namespace is empty (FR-002). +func (k *KubernetesBackend) namespacePodListing(run podRunner, fieldSelector string) ([]labelledPod, error) { + if pods, cached := k.namespacePodListings[fieldSelector]; cached { + return pods, nil + } + + listed, err := k.issueNamespacePodListing(run, fieldSelector) if err != nil { return nil, err } - statuses := []string{} - for _, line := range strings.Split(output, "\n") { - if line == "" { - continue + pods := parseLabelledPods(listed) + k.namespacePodListings[fieldSelector] = pods + + return pods, nil +} + +// freshNamespacePodListing answers "what are those pods doing?" — the same +// query as namespacePodListing, never memoised. +// +// It is a separate function rather than a parameter because the difference is +// in the question, not in the caller. A phase is read because it is expected to +// change, and confirmAppContainersQuiescedWithin and +// reportAppContainersRunningWithin poll IsRunning in a sleep loop with nothing +// mutating between iterations — they are waiting for the answer to move. +// Served from the memo, the first poll's bytes were replayed until the window +// ran out, so the transition those loops exist to detect could not be observed: +// a restore whose pods terminated normally aborted after the destructive step, +// blaming "a pod that will not terminate", and a restore that in fact worked +// reported every service as having failed to come back. +// +// Invalidating the memo from those loops would be the wrong fix twice over: it +// is one caller deciding another's freshness, and it would also discard the +// resolution answers the run still needs. Nothing here reads or writes +// namespacePodListings, so the next poll loop someone writes cannot reintroduce +// the bug by forgetting to. +// +// It takes no field selector because the running check narrows by none: a +// terminal pod is part of the answer to what the service is doing. +func (k *KubernetesBackend) freshNamespacePodListing(run podRunner) ([]labelledPod, error) { + listed, err := k.issueNamespacePodListing(run, "") + if err != nil { + return nil, err + } + + return parseLabelledPods(listed), nil +} + +// issueNamespacePodListing issues the namespace-wide ownership listing. +// +// It is the one place the query is built and the one place its failure is +// worded, which is what keeps the memoised and the fresh reading of it the same +// listing — and the reason neither of them can memoise an error: a failure says +// the cluster could not answer, not that the namespace is empty (FR-002). +func (k *KubernetesBackend) issueNamespacePodListing(run podRunner, fieldSelector string) (string, error) { + listed, err := run("kubectl", k.podListingArgsFor(podOwnershipJSONPath, fieldSelector, "")...) + if err != nil { + return "", fmt.Errorf("failed to list pods in namespace %s: %w", k.namespace, err) + } + + return listed, nil +} + +// claimedPodNamesWith lists the namespace's pods with the ownership evidence +// their labels carry, and returns — in listing order — the ones this deployment +// claims for the service. +// +// It is the fallback both pod resolvers use, and it exists because they used to +// answer "whose pod is this?" with a bare name match while the location +// decision answered the same question through kubernetes_ownership.go. So a +// co-hosted `othertool-task-manager-…` carrying no matching label was refused +// by the location rule and accepted here — and the pods these resolvers exec +// `env` in are where ensureNeo4jDiscovery and ensurePostgresDiscovery read the +// endpoint a restore later writes into. Resolution was the permissive one of +// the two, which is the wrong way round. +// +// What an unanchored prefixed name means here is unanchoredNamePolicyFor's +// answer, not this function's, and that is the second half of the same fix. It +// used to be the constant unanchoredNameClaims whatever the service was, so a +// namespace that declared nothing resolved `postgres-database-0` as the +// deployment's `database` while the location decision refused the very same pod +// — and the pod this resolver returns is where ensureNeo4jDiscovery and +// ensurePostgresDiscovery read the endpoint a restore later writes into. For the +// app services the permissive reading stays, and it has to: a namespace whose +// pods carry none of the four service labels — a kustomize deployment, a +// hand-rolled manifest, an older chart — would otherwise answer "no pods" for +// every service, taking a deployment that works today off the path it works on. +// +// terminal says which pods are out of scope and is the client-side half of the +// field selector passed beside it: resolution excludes both terminal phases +// because it is looking for somewhere to exec, enumeration excludes only +// Succeeded because a crashed replica is much of what a bundle is collected for. +func (k *KubernetesBackend) claimedPodNamesWith(run podRunner, service, fieldSelector string, terminal func(string) bool) ([]string, error) { + listed, err := k.namespacePodListing(run, fieldSelector) + if err != nil { + return nil, err + } + + pods := filterLabelledPods(listed, terminal) + prefixes := releasePrefixes(declaredPodServices(pods)) + + // Declarations first, then claimed names: a pod that says what it is + // outranks one that merely carries a name this deployment claims, whatever + // order the listing happened to arrive in. The singular resolver takes the + // first of these, so the order is the answer. + names := []string{} + for _, pod := range pods { + if declaresService(pod.Service, service) { + names = append(names, pod.Name) } - parts := strings.Split(line, ";") - if len(parts) != 2 { + } + for _, pod := range pods { + if declaresService(pod.Service, service) { continue } - if strings.Contains(parts[0], service) { - statuses = append(statuses, parts[1]) + if deploymentClaimsResource(prefixes, pod.Name, pod.Service, service) { + names = append(names, pod.Name) + + continue } + logUnclaimedResource("pod", pod.Name, pod.Service, service, prefixes) } - return statuses, nil + + return names, nil } // getPodForService resolves a single pod for a service using the unbounded -// executor. It is the shared helper used by the backup tool; its behavior is -// unchanged. +// executor. It is the shared helper used by the backup tool. Like its plural +// sibling GetAllPods it returns errNoPodsMatched only when kubectl calls +// succeeded but nothing matched, and the wrapped kubectl error when the +// cluster/API itself is unreachable (FIX-2). func (k *KubernetesBackend) getPodForService(service string) (string, error) { - return k.getPodForServiceWith(k.executor.runCommand, service) + return k.getPodForServiceWith(k.stdoutRunner(), service) } // getPodForServiceContext is the timeout-bounded pod resolver the collect @@ -311,96 +689,437 @@ func (k *KubernetesBackend) getPodForServiceContext(ctx context.Context, timeout // getPodForServiceWith resolves a single pod for a service via the supplied // runner, caching the result. HA clusters resolve to the primary pod. +// +// Only pods something can still run in are candidates: this resolves an exec +// target, and a completed or failed pod is not one — a finished backup-CronJob +// pod named `-database-backup-28…` is not a database (see +// livePodFieldSelector). +// +// A successful-but-empty match returns errNoPodsMatched; a kubectl failure +// returns the wrapped kubectl error, exactly as getAllPodsWith draws the same +// distinction (FIX-2). "Successful" means the question was actually answered: +// if every label selector query failed, the empty match is not a match at all +// and a real error is returned even when the fallback listing itself succeeded +// (FR-002). func (k *KubernetesBackend) getPodForServiceWith(run podRunner, service string) (string, error) { if pod, ok := k.podCache[service]; ok && pod != "" { return pod, nil } + // A selector that matched several pods is read for the primary (HA + // clusters like CloudNativePG); the ownership fallback takes the first name + // it claimed, as it always has. + pickLive := func(pods []labelledPod) []string { + names := livePodNames(pods) + if len(names) > 1 { + if primary := k.findPrimaryPod(run, names); primary != "" { + return []string{primary} + } + } + + return names + } + + pods, err := k.podsForServiceWith(run, service, livePodFieldSelector, pickLive, podPhaseIsTerminal) + if err != nil { + return "", err + } + + k.podCache[service] = pods[0] + + return pods[0], nil +} + +// podsForServiceWith is the resolution the singular and plural pod resolvers +// share: the label-selector tier, then the ownership fallback, then the FR-002 +// discrimination between "nothing matched" and "nothing was answered". It is +// one function so that discrimination is spelled once. +// +// fieldSelector is the phase filter the caller resolves under, pick reduces a +// selector's listing to the names the caller accepts (and is where the singular +// resolver chooses a primary), and terminal is the phase test the fallback +// drops pods on. A non-empty answer is returned as soon as a tier gives one. +// +// The fallback is the namespace read for what each pod declares itself to be +// as well as what it is called. The Helm chart labels some pods with a +// different service value than the canonical name (e.g. infrahub-server pods +// carry infrahub/service=server), so label selectors alone would miss them. A +// failure there means the cluster/API is unreachable, not that the service is +// absent, so it is wrapped rather than reported as an empty match. +func (k *KubernetesBackend) podsForServiceWith(run podRunner, service, fieldSelector string, pick func([]labelledPod) []string, terminal func(string) bool) ([]string, error) { selectors := k.podSelectors(service) + selectorErrs := []error{} for _, selector := range selectors { - output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + output, err := run("kubectl", k.podListingArgs(fieldSelector, selector)...) if err != nil { + selectorErrs = append(selectorErrs, fmt.Errorf("selector %s: %w", selector, err)) + continue } - pods := nonEmptyLines(output) - if len(pods) > 0 { - // If multiple pods found, try to find the primary (for HA clusters like CloudNativePG) - if len(pods) > 1 { - if primary := k.findPrimaryPod(run, pods); primary != "" { - k.podCache[service] = primary - return primary, nil - } - } - k.podCache[service] = pods[0] - return pods[0], nil + if pods := pick(parseLabelledPods(output)); len(pods) > 0 { + return pods, nil } } - output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + names, err := k.claimedPodNamesWith(run, service, fieldSelector, terminal) if err != nil { - return "", err + return nil, err } - for _, name := range nonEmptyLines(output) { - if strings.Contains(name, service) { - k.podCache[service] = name - return name, nil - } + if len(names) > 0 { + return names, nil + } + + // Every label selector query failed, so nothing here established that the + // service has no pod: the sentinel would tell the caller "not deployed" on + // the strength of queries that were never answered (FR-002). + if len(selectors) > 0 && len(selectorErrs) == len(selectors) { + return nil, fmt.Errorf("failed to list pods for service %s in namespace %s: every label selector query failed: %w", service, k.namespace, errors.Join(selectorErrs...)) } - return "", fmt.Errorf("no pods found for service %s in namespace %s", service, k.namespace) + return nil, fmt.Errorf("no pods found for service %s in namespace %s: %w", service, k.namespace, errNoPodsMatched) } -// GetAllPods returns all pod names for a given service using the unbounded -// executor (shared helper used by the backup tool). It returns errNoPodsMatched -// only when kubectl calls succeeded but nothing matched, and the wrapped -// kubectl error when the cluster/API itself is unreachable (FIX-2). -func (k *KubernetesBackend) GetAllPods(service string) ([]string, error) { - return k.getAllPodsWith(k.executor.runCommand, service) +// errLocationUndetermined is the FR-002 refusal, as a value callers and tests +// can identify rather than a message they have to match: the cluster did not +// answer where the database lives, which is not the same fact as the database +// being absent. +var errLocationUndetermined = errors.New("the deployment could not be queried for it, so where it lives was never established") + +// undeterminedLocation is what an operator reads when the cluster could not +// answer where a database lives. +// +// The failure-message contract requires this one to name the query that failed +// and the permission that would be missing, and — explicitly — *not* to say the +// database is absent. Those two readings have opposite remedies: an operator +// told "no database container here" configures an external database they do not +// have, while what they actually need is a Role or a corrected namespace. That +// conflation is the defect this feature exists to remove, so the refusal states +// which of the two it is rather than leaving the wording to imply either. +// +// The permission is named conditionally because a denial is only one of the +// causes: a throttled or unreachable API server produces the same undetermined +// answer, and asserting an RBAC cause for it would send the operator to fix +// something that is not broken. +func (k *KubernetesBackend) undeterminedLocation(service string, cause error) error { + return fmt.Errorf( + "failed to determine whether %s runs in namespace %s: %w: %w. "+ + "This is the query failing, not evidence that the database is absent — the run stops here rather than treating %s as a database outside the deployment (FR-002). "+ + "If the query was refused, the identity this tool runs as needs get and list on pods, deployments and statefulsets in namespace %s; "+ + "otherwise check that --k8s-namespace names the right namespace and that the API server is reachable", + service, k.namespace, errLocationUndetermined, cause, service, k.namespace) } -// getAllPodsContext is the timeout-bounded pod enumerator the collect primitives -// use (research R2, FIX-5). -func (k *KubernetesBackend) getAllPodsContext(ctx context.Context, timeout time.Duration, service string) ([]string, error) { - return k.getAllPodsWith(k.boundedRunner(ctx, timeout), service) +// locateService answers whether a service runs inside the deployment, which is +// the whole of the internal-versus-external decision (FR-001, FR-002). External +// is returned only when every label selector answered and none matched; every +// other outcome — a wrong namespace, RBAC that hides pods, an unreachable API +// server — is an error, so no failure to query the cluster can be reinterpreted +// as "the database must be somewhere else" and send the run looking for it on +// the network. +// +// The query is time-bounded (FR-025). This is the first step of every external +// path, so a cluster that stops answering here would hang the run before any +// endpoint had even been resolved. +func (k *KubernetesBackend) locateService(service string) (EndpointLocation, error) { + if location, ok := k.serviceLocations[service]; ok { + return location, nil + } + + bound := externalDBControlBound(k.config) + + location, err := k.locateServiceWith(k.externalDBRunner(bound, externalDBTarget("the "+service+" service", k.namespace)), service) + if err != nil { + // A failure is not memoised. It says the cluster could not answer, not + // where the database is, and remembering it would turn one bad moment + // into a decision the rest of the run cannot revisit — while the + // caller's own retry is what this would be taking away. + return "", err + } + + k.serviceLocations[service] = location + + return location, nil } -// getAllPodsWith enumerates every pod backing a service via the supplied -// runner. A successful-but-empty match returns errNoPodsMatched; a failed -// fallback kubectl call returns the wrapped kubectl error so cluster failures -// are never mistaken for an undeployed service (FIX-2). -func (k *KubernetesBackend) getAllPodsWith(run podRunner, service string) ([]string, error) { +// locateServiceWith is locateService against the supplied runner, following the +// package's existing injection idiom so the decision is testable without a +// cluster. +// +// The evidence is deliberately asymmetric, because the two wrong answers cost +// different things: +// +// - A wrong "internal" routes a database that lives outside the deployment +// onto the in-deployment path, and runs `neo4j-admin` or `pg_dump` — and on +// the restore side a load — inside whatever resource matched. +// - A wrong "external" takes a deployment that works today off the path it +// works on. That is not a harmless conservatism even now that external +// capture exists: the run builds a transient workload, probes over Bolt for +// an address the deployment describes for its own in-cluster service, and +// fails somewhere along the way — and on the restore side it refuses +// outright, so an operator whose backups worked yesterday cannot restore. +// +// So evidence of presence is read from every source the deployment offers, and +// only a namespace in which *nothing* claims the service reads as external. +// Absence of a label is not evidence that the database is absent — three +// unrelated situations made it look like one: +// +// - Label evidence alone. A pod labelled by kustomize, by a hand-rolled +// manifest or by a different chart version matches none of the four +// selectors, and an externally-managed CloudNativePG cluster — pods +// `-1`, labelled `cnpg.io/cluster` — matches none of them either, +// while findPrimaryPod exists precisely to support that topology. +// - Live pods only. livePodFieldSelector excludes Failed, whose justification +// is a finished backup-CronJob pod — a reason to exclude Succeeded, not +// Failed, from a question about where a database lives. A pod evicted +// mid-recreate, or one on a drained node, is not evidence of absence, and a +// StatefulSet scaled to zero has no pods to read at all. +// - No floor on the control bound, which made every query here expire; see +// resolveExternalDBTimeout. +// +// The order is: a label selector query, then the namespace's own resources read +// for what they declare and for whether this deployment claims them (see +// kubernetes_ownership.go, which answers "does this deployment claim it?" +// rather than "does this name resemble the service?"). A name carrying a +// release prefix counts only against a prefix some resource *declared* — which +// is what keeps `postgres-database-0` from an unrelated StatefulSet from +// putting a genuinely external database on the in-deployment path — while a +// name that is the service's own with nothing before it counts on its own, +// which is what the CloudNativePG shape above needs and what this used to +// refuse. Refusing it made this decision strictly narrower than the pod +// resolver's own fallback: the resolver claimed `task-manager-db-1` and this +// called the same database external, so a restore the resolver could have run +// was refused outright. +// +// An unanswered selector query is never read as absence: if no selector matched +// and any of them failed, the location was not established and the caller gets +// an error (FR-002). +func (k *KubernetesBackend) locateServiceWith(run podRunner, service string) (EndpointLocation, error) { selectors := k.podSelectors(service) + selectorErrs := []error{} for _, selector := range selectors { - output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-l", selector, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + // completedPodFieldSelector, not livePodFieldSelector: this asks where + // the database lives, not which pod to exec in, and a pod that failed + // is still where it lives. + output, err := run("kubectl", k.podListingArgs(completedPodFieldSelector, selector)...) if err != nil { + selectorErrs = append(selectorErrs, fmt.Errorf("selector %s: %w", selector, err)) + continue } - pods := nonEmptyLines(output) - if len(pods) > 0 { - return pods, nil + candidates := parseLabelledPods(output) + if pods := enumerablePodNames(candidates); len(pods) > 0 { + logrus.Debugf("Service %s runs in namespace %s: pod %s matches %s", service, k.namespace, pods[0], selector) + k.seedPodCache(service, candidates) + + return EndpointLocationInternal, nil } } - // Fallback: substring match on pod names, mirroring getPodForService. The - // Helm chart labels some pods with a different service value than the - // canonical name (e.g. infrahub-server pods carry infrahub/service=server), - // so label selectors alone would miss them. A failure here means the - // cluster/API is unreachable, not that the service is absent. - output, err := run("kubectl", "get", "pods", "-n", k.namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + if len(selectorErrs) > 0 { + return "", k.undeterminedLocation(service, errors.Join(selectorErrs...)) + } + + claimed, err := k.deploymentClaimsService(run, service) if err != nil { - return nil, fmt.Errorf("failed to list pods in namespace %s: %w", k.namespace, err) + return "", k.undeterminedLocation(service, err) + } + if claimed { + return EndpointLocationInternal, nil } - pods := []string{} - for _, name := range nonEmptyLines(output) { - if strings.Contains(name, service) { - pods = append(pods, name) + + return EndpointLocationExternal, nil +} + +// seedPodCache hands the resolver what the location decision already read, so +// the same `kubectl get pods` is not issued twice within a second of itself. +// +// It seeds only when exactly one live pod matched, and that restraint is the +// whole of its safety: +// +// - The location decision lists with completedPodFieldSelector, because it +// asks where a database lives and a pod that failed is still where it +// lives. The resolver lists with livePodFieldSelector, because it resolves +// an exec target. Seeding the pod this decision found would put a +// terminated pod in front of every exec that followed, so only the live +// subset is eligible. +// - Above one live pod the resolver does something this cannot: it asks which +// is the primary, which is how an HA CloudNativePG cluster resolves to the +// member that can be read. Seeding there would quietly replace that choice +// with whichever pod the listing happened to put first. +// +// So the case it covers is the single-replica database — the one this feature +// meets on every deployment it runs against, and the one paying the duplicated +// listing (FR-015). +func (k *KubernetesBackend) seedPodCache(service string, candidates []labelledPod) { + if _, cached := k.podCache[service]; cached { + return + } + + if live := livePodNames(candidates); len(live) == 1 { + k.podCache[service] = live[0] + } +} + +// deploymentClaimsService is the evidence the label selectors cannot carry: +// whether anything in the namespace declares itself as the service, or carries +// a name this deployment claims. +// +// Workloads are read as well as pods, and that is what answers the shapes with +// no pod to look at — a StatefulSet scaled to zero, a pod evicted between two +// listings. They are read only once no pod declares the service, which is +// exactly when they decide the answer: from there, a workload listing that +// fails leaves "is a scaled-down database here?" unanswered, and reading that +// as "external" is the reinterpretation FR-002 forbids — permission denied is +// named there as a failure to query, not as evidence. So a failure on either +// listing is reported. +func (k *KubernetesBackend) deploymentClaimsService(run podRunner, service string) (bool, error) { + listed, err := k.namespacePodListing(run, completedPodFieldSelector) + if err != nil { + return false, err + } + + pods := runnablePods(listed) + for _, pod := range pods { + if declaresService(pod.Service, service) { + logrus.Debugf("Service %s runs in namespace %s: pod %s declares it (%s)", service, k.namespace, pod.Name, pod.Service) + + return true, nil } } - if len(pods) > 0 { - return pods, nil + + workloads, err := k.listAllWorkloadsWith(run) + if err != nil { + return false, err } + for _, workload := range workloads { + if declaresService(declaredServiceIn(workload.SelectorLabels, workload.TemplateLabels), service) { + logrus.Debugf("Service %s runs in namespace %s: workload %s declares it", service, k.namespace, workload.Name) - return nil, fmt.Errorf("no pods found for service %s in namespace %s: %w", service, k.namespace, errNoPodsMatched) + return true, nil + } + } + + // The release the deployment named itself with, read from whatever declared + // itself — a pod, or a workload whose pods are gone. Without one there is + // nothing to anchor a *prefixed* name to, and a bare resemblance is + // evidence about a name: `postgres-database-0` from an unrelated StatefulSet + // reads exactly like the deployment's own database. Nothing at all is what + // external is — which is the same rule, and now the same policy, that + // getPodStatusesWith and the pod resolver decide on (see + // unanchoredNamePolicyFor). + prefixes := slices.Concat( + releasePrefixes(declaredPodServices(pods)), + releasePrefixes(declaredWorkloadServices(workloads)), + ) + + for _, pod := range pods { + if deploymentClaimsResource(prefixes, pod.Name, pod.Service, service) { + logrus.Debugf("Service %s runs in namespace %s: pod %s carries a name this deployment claims (release prefixes %v)", service, k.namespace, pod.Name, prefixes) + + return true, nil + } + } + for _, workload := range workloads { + declared := declaredServiceIn(workload.SelectorLabels, workload.TemplateLabels) + if deploymentClaimsResource(prefixes, workload.Name, declared, service) { + logrus.Debugf("Service %s runs in namespace %s: workload %s carries a name this deployment claims (release prefixes %v)", service, k.namespace, workload.Name, prefixes) + + return true, nil + } + } + + return false, nil +} + +// listAllWorkloadsWith lists every Deployment and StatefulSet in the namespace. +// A kind it could not read is an error, not an empty kind: see +// deploymentClaimsService for why (FR-002). +func (k *KubernetesBackend) listAllWorkloadsWith(run podRunner) ([]kubernetesWorkload, error) { + workloads := []kubernetesWorkload{} + for _, kind := range workloadKinds { + listed, err := k.listWorkloadsWith(run, kind) + if err != nil { + return nil, fmt.Errorf("failed to list %s in namespace %s: %w", kind, k.namespace, err) + } + workloads = append(workloads, listed...) + } + + return workloads, nil +} + +// podListingArgs builds a pod listing that reports each pod's phase, with the +// supplied field selector and label selector applied. +func (k *KubernetesBackend) podListingArgs(fieldSelector, labelSelector string) []string { + return k.podListingArgsFor(podNamePhaseJSONPath, fieldSelector, labelSelector) +} + +// podListingArgsFor is podListingArgs for a listing that asks for something +// other than name-and-phase — the ownership listing, which carries each pod's +// declared service alongside them. +// +// Every pod listing this package issues about a service is built here, and +// every one of them excludes the transient external-database workload (FR-028). +// That is what the exclusion being a *choke point* means: a listing cannot be +// added that forgets it, because the arguments cannot be built without it. +// Before this it was applied at one enumerator of four — getAllPodsWith — while +// the singular resolver, the running check and the location decision each +// listed the namespace with no exclusion at all. +// +// The label selector is a string rather than raw arguments for the same reason: +// the exclusion has to be joined to whatever the caller selects on, and a +// caller that passed `-l` itself could pass it twice. +func (k *KubernetesBackend) podListingArgsFor(output, fieldSelector, labelSelector string) []string { + args := []string{"get", "pods", "-n", k.namespace, "-l", excludeTransientWorkloads(labelSelector)} + if fieldSelector != "" { + args = append(args, "--field-selector="+fieldSelector) + } + + return append(args, "-o", output) +} + +// GetAllPods returns all pod names for a given service using the unbounded +// executor (shared helper used by the backup tool). It returns errNoPodsMatched +// only when kubectl calls succeeded but nothing matched, and the wrapped +// kubectl error when the cluster/API itself is unreachable (FIX-2). +func (k *KubernetesBackend) GetAllPods(service string) ([]string, error) { + return k.getAllPodsWith(k.stdoutRunner(), service) +} + +// getAllPodsContext is the timeout-bounded pod enumerator the collect primitives +// use (research R2, FIX-5). +func (k *KubernetesBackend) getAllPodsContext(ctx context.Context, timeout time.Duration, service string) ([]string, error) { + return k.getAllPodsWith(k.boundedRunner(ctx, timeout), service) +} + +// getAllPodsWith enumerates every pod backing a service via the supplied +// runner. A successful-but-empty match returns errNoPodsMatched; a failed +// kubectl call returns the wrapped kubectl error so cluster failures are never +// mistaken for an undeployed service (FIX-2) — including the case where every +// label selector query failed and only the fallback listing answered, which +// establishes nothing about the service. +// +// Enumeration keeps failed pods and drops only pods that ran to completion: +// this is what per-replica log and diagnostic collection walks, and a crashed +// replica is much of what a bundle is collected for (see +// completedPodFieldSelector). +// +// Like every other pod listing in this package, the listings here exclude the +// transient external-database workload, which is what keeps it from being +// published as a replica of the service it stands in for (FR-028). The +// choke point for that exclusion is podListingArgsFor, where the arguments are +// built — not this function, which used to apply it while the singular +// resolver, the running check and the location decision applied none. +// +// The exclusion is stated twice on purpose. The label selector is the mechanism +// that holds whatever the pod is called; the name filter is what still holds if +// a future listing is issued without going through podListingArgsFor. Neither +// is redundant, and neither on its own is the whole guarantee: the transient +// pod's name also deliberately contains no service name as a substring (see +// transientObjectPrefix), so today the fallback below would not match it even +// unfiltered — but that is a naming convention, not an invariant, and FR-028 +// must not rest on it. +func (k *KubernetesBackend) getAllPodsWith(run podRunner, service string) ([]string, error) { + return k.podsForServiceWith(run, service, completedPodFieldSelector, enumerablePodNames, podPhaseIsCompleted) } // KubernetesBackend implements the collect-side primitives (spec @@ -421,7 +1140,7 @@ var ( // kubectl exec arguments, so the pod-resolution kubectl call is bounded too // (FIX-5) rather than only the exec that follows. func (k *KubernetesBackend) buildExecArgsContext(ctx context.Context, timeout time.Duration, service string, command []string, opts *ExecOptions) ([]string, error) { - pod, err := k.getPodForServiceContext(ctx, timeout, service) + pod, err := k.execPodContext(ctx, timeout, service, opts) if err != nil { return nil, err } @@ -471,10 +1190,35 @@ func (k *KubernetesBackend) CopyFromContext(ctx context.Context, timeout time.Du if err != nil { return err } - source := fmt.Sprintf("%s/%s:%s", k.namespace, pod, src) - if _, err := k.executor.runCommandContext(ctx, timeout, "kubectl", "cp", source, dest); err != nil { + + return k.CopyFromPodContext(ctx, timeout, pod, src, dest) +} + +// CopyFromPodContext is CopyFromContext against a pod the caller already knows +// the name of: CopyFromPod's bounded sibling, and the one an operation against +// an external database uses, where every kubectl call has to be bounded +// (FR-025). +func (k *KubernetesBackend) CopyFromPodContext(ctx context.Context, timeout time.Duration, pod, src, dest string) error { + if _, err := k.executor.runCommandContext(ctx, timeout, "kubectl", "cp", k.podPath(pod, src), dest); err != nil { + return err + } + + return nil +} + +// CopyToPodContext is CopyToPod's bounded sibling, and the direction the +// restore path needs: the PostgreSQL dump has to reach the transient workload +// before the client in it can read it back into an external server. +// +// It is bounded for the reason the copy *out* is, only more so. This transfer +// happens with Infrahub scaled to zero, so an unbounded `kubectl cp` onto a +// wedged API server leaves the deployment quiesced with nothing reporting why +// (FR-025). +func (k *KubernetesBackend) CopyToPodContext(ctx context.Context, timeout time.Duration, pod, src, dest string) error { + if _, err := k.executor.runCommandContext(ctx, timeout, "kubectl", "cp", src, k.podPath(pod, dest)); err != nil { return err } + return nil } diff --git a/src/internal/app/environment_kubernetes_replicas_test.go b/src/internal/app/environment_kubernetes_replicas_test.go index 062f7f8d..81563456 100644 --- a/src/internal/app/environment_kubernetes_replicas_test.go +++ b/src/internal/app/environment_kubernetes_replicas_test.go @@ -76,12 +76,6 @@ func (f *fakeKubectl) scaleCalls() []string { return calls } -func newTestKubernetesBackend() *KubernetesBackend { - k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) - k.namespace = "infrahub" - return k -} - func assertScaleCalls(t *testing.T, got []string, want ...string) { t.Helper() if strings.Join(got, " ") != strings.Join(want, " ") { @@ -187,7 +181,7 @@ func TestRecordReplicaCounts_FailsBeforeAnyChange(t *testing.T) { k := newTestKubernetesBackend() iops := &InfrahubOps{config: &Configuration{}, backend: k, executor: k.executor} - err := iops.recordReplicaCounts(appServices...) + err := iops.recordReplicaCounts(appServicesStoppedForBackup...) if err == nil { t.Fatal("recordReplicaCounts succeeded, want an error because a replica count cannot be read") } diff --git a/src/internal/app/environment_kubernetes_test.go b/src/internal/app/environment_kubernetes_test.go index e17f8d3b..82e322a0 100644 --- a/src/internal/app/environment_kubernetes_test.go +++ b/src/internal/app/environment_kubernetes_test.go @@ -3,9 +3,12 @@ package app import ( "errors" "fmt" + "os" + "path/filepath" "slices" "strings" "testing" + "time" ) func TestSplitHelmChartLabel(t *testing.T) { @@ -142,6 +145,1240 @@ func TestGetAllPodsWith_Branches(t *testing.T) { }) } +// newTestKubernetesBackend is a backend with an empty pod cache, so each case +// below resolves from its own stub runner rather than from a previous case's +// cached answer. +func newTestKubernetesBackend() *KubernetesBackend { + k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + k.namespace = "infrahub" + + return k +} + +// TestGetPodForServiceWith_Branches is the singular resolver's half of the +// distinction GetAllPods has always drawn (FIX-2). Every caller on the backup +// path resolves through getPodForServiceWith, and the internal-versus-external +// decision reads its error: a genuinely empty match means the database is +// elsewhere, while a kubectl/RBAC/API failure means the question was not +// answered at all. +func TestGetPodForServiceWith_Branches(t *testing.T) { + t.Run("a matching label selector returns the pod", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + return "infrahub-database-0\n", nil + } + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, "database") + if err != nil { + t.Fatalf("getPodForServiceWith failed: %v", err) + } + if pod != "infrahub-database-0" { + t.Errorf("pod = %q, want infrahub-database-0", pod) + } + }) + + t.Run("kubectl succeeds but nothing matches returns errNoPodsMatched", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { return "", nil } + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, "database") + if pod != "" { + t.Errorf("pod = %q, want no pod", pod) + } + if !errors.Is(err, errNoPodsMatched) { + t.Errorf("err = %v, want errNoPodsMatched so the empty match is distinguishable", err) + } + }) + + t.Run("a pod-list permission failure surfaces a real error, not the sentinel", func(t *testing.T) { + // What RBAC that cannot list pods in the namespace actually looks like: + // every kubectl call fails, including the substring fallback. + run := func(name string, args ...string) (string, error) { + return "", fmt.Errorf(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods" in API group "" in the namespace "infrahub"`) + } + _, err := newTestKubernetesBackend().getPodForServiceWith(run, "database") + if err == nil { + t.Fatal("getPodForServiceWith succeeded despite being unable to list pods, want error") + } + if errors.Is(err, errNoPodsMatched) { + t.Errorf("a permission failure was masked as errNoPodsMatched: %v", err) + } + if !strings.Contains(err.Error(), "failed to list pods") { + t.Errorf("err = %v, want the wrapped kubectl failure", err) + } + if !strings.Contains(err.Error(), "is forbidden") { + t.Errorf("err = %v, want it to keep the reason kubectl gave", err) + } + }) +} + +// TestLocateServiceSeedsTheResolver is the FR-015 cost the location gate would +// otherwise add to every Kubernetes run, all-internal ones included: the +// decision issues the pod listing, learns the name, throws it away, and the +// resolver issues the same listing a moment later. +func TestLocateServiceSeedsTheResolver(t *testing.T) { + t.Run("a single live pod is handed to the resolver", func(t *testing.T) { + listings := 0 + run := func(_ string, _ ...string) (string, error) { + listings++ + + return "infrahub-database-0;Running\n", nil + } + + backend := newTestKubernetesBackend() + if _, err := backend.locateServiceWith(run, serviceNeo4j); err != nil { + t.Fatalf("locateServiceWith() error = %v", err) + } + + after := listings + pod, err := backend.getPodForServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("getPodForServiceWith() error = %v", err) + } + if pod != "infrahub-database-0" { + t.Errorf("resolved pod = %q, want the one the location decision already read", pod) + } + if listings != after { + t.Errorf("the resolver issued %d further listings, want none: the decision had already read the answer", listings-after) + } + }) + + t.Run("a terminated pod is never seeded", func(t *testing.T) { + // The location decision counts a Failed pod as evidence the database + // lives here; the resolver must not exec into one. Seeding what the + // decision matched would put a terminated pod in front of every exec + // that followed. + run := func(_ string, _ ...string) (string, error) { return "infrahub-database-0;Failed\n", nil } + + backend := newTestKubernetesBackend() + if location, err := backend.locateServiceWith(run, serviceNeo4j); err != nil || location != EndpointLocationInternal { + t.Fatalf("locateServiceWith() = %q, %v; want the failed pod to still read as internal", location, err) + } + if pod, cached := backend.podCache[serviceNeo4j]; cached { + t.Errorf("seeded the resolver with %q, which is in a terminal phase", pod) + } + }) + + t.Run("an HA cluster is left to the primary lookup", func(t *testing.T) { + // Above one live pod the resolver asks which member is the primary. + // Seeding here would replace that with whichever pod the listing + // happened to put first. + run := func(_ string, _ ...string) (string, error) { + return "pg-cluster-1;Running\npg-cluster-2;Running\n", nil + } + + backend := newTestKubernetesBackend() + if _, err := backend.locateServiceWith(run, serviceTaskManagerDB); err != nil { + t.Fatalf("locateServiceWith() error = %v", err) + } + if pod, cached := backend.podCache[serviceTaskManagerDB]; cached { + t.Errorf("seeded the resolver with %q, bypassing the primary lookup an HA cluster needs", pod) + } + }) +} + +// TestLocateServiceMemoisesItsAnswer covers the other half of the same cost. +// Every path that touches a database resolves through a databaseTarget and each +// asks the question again, against an answer that cannot change while the run +// lasts. +func TestLocateServiceMemoisesItsAnswer(t *testing.T) { + backend := newTestKubernetesBackend() + backend.serviceLocations[serviceNeo4j] = EndpointLocationExternal + + location, err := backend.locateService(serviceNeo4j) + if err != nil { + t.Fatalf("locateService() error = %v", err) + } + if location != EndpointLocationExternal { + t.Errorf("location = %q, want the memoised %q", location, EndpointLocationExternal) + } + + // A failure must not be memoised: it says the cluster could not answer, not + // where the database is, and remembering it would take away the retry the + // caller still has. + failing := newTestKubernetesBackend() + if _, err := failing.locateServiceWith(func(string, ...string) (string, error) { + return "", fmt.Errorf("the server was unable to return a response") + }, serviceNeo4j); err == nil { + t.Fatal("locateServiceWith() = nil error, want the listing failure") + } + if _, cached := failing.serviceLocations[serviceNeo4j]; cached { + t.Error("a listing failure was memoised as a location") + } +} + +// TestLocateServiceWith is FR-002 at the point where it can go wrong most +// dangerously. External mode is engaged only on a positively empty match; a +// namespace the run cannot list pods in must fail, because reinterpreting that +// as "the database must be external" would send the run looking for a database +// on the network instead of stopping. +func TestLocateServiceWith(t *testing.T) { + t.Run("a resolved pod means the database is internal", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + return "infrahub-database-0\n", nil + } + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q", location, EndpointLocationInternal) + } + }) + + t.Run("an empty match means the database is external", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + if isWorkloadListing(args) { + return emptyWorkloadListing, nil + } + + return "", nil + } + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationExternal { + t.Errorf("location = %q, want %q", location, EndpointLocationExternal) + } + }) + + t.Run("a pod-list permission failure is an error and not external", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + return "", fmt.Errorf(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods" in API group "" in the namespace "infrahub"`) + } + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err == nil { + t.Fatal("locateServiceWith succeeded despite being unable to list pods, want error") + } + if location == EndpointLocationExternal { + t.Error("a permission failure engaged external mode, which would send the run onto the network looking for a database that is in the namespace it could not read") + } + if location != "" { + t.Errorf("location = %q, want the zero value alongside an error", location) + } + if !strings.Contains(err.Error(), serviceNeo4j) { + t.Errorf("err = %v, want it to name the %q service", err, serviceNeo4j) + } + if !strings.Contains(err.Error(), "is forbidden") { + t.Errorf("err = %v, want it to keep the reason kubectl gave", err) + } + }) + + t.Run("an unreachable API server is an error and not external", func(t *testing.T) { + run := func(name string, args ...string) (string, error) { + return "", fmt.Errorf("The connection to the server 10.0.0.1:6443 was refused") + } + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceTaskManagerDB) + if err == nil { + t.Fatal("locateServiceWith succeeded despite an unreachable API server, want error") + } + if location == EndpointLocationExternal { + t.Errorf("an unreachable control plane engaged external mode: %v", err) + } + }) +} + +// TestGetAllPodsExcludesTheTransientWorkload is FR-028 asserted at the +// resolver: registering the transient external-database pod so that execution +// resolves to it must not make it discoverable as a replica of the service it +// stands in for. GetAllPods and the bounded getAllPodsContext are one-line +// delegations to getAllPodsWith, and ServiceReplicas enumerates through +// getAllPodsContext, so the exclusion is asserted here once for all three. +// +// Both mechanisms are covered, because each protects against a different future +// change: the label selector holds whatever the pod is named, and the name +// filter holds if a label change ever collided with a selector key. +func TestGetAllPodsExcludesTheTransientWorkload(t *testing.T) { + transientPod := transientObjectPrefix + "-capture-a1b2c3d4" + + t.Run("a transient pod a selector matched is not returned as a replica", func(t *testing.T) { + // The cluster is pretended to ignore the exclusion, which is what a + // future label collision would look like: every listing returns the + // transient pod alongside the deployment's own. + k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + k.namespace = "infrahub" + run := func(_ string, _ ...string) (string, error) { + return transientPod + "\ndatabase-0\n", nil + } + + pods, err := k.getAllPodsWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("getAllPodsWith failed: %v", err) + } + if !contains(pods, "database-0") { + t.Errorf("pods = %v, want the deployment's own database pod", pods) + } + if contains(pods, transientPod) { + t.Errorf("pods = %v, want the transient workload %s excluded (FR-028)", pods, transientPod) + } + }) + + t.Run("a namespace holding only the transient pod reports no replicas", func(t *testing.T) { + // The external-database case: the service has no pod of its own, and the + // transient stand-in must not be mistaken for one — a replica count of + // one here would put a backup client into troubleshooting bundles. + k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + k.namespace = "infrahub" + run := func(_ string, _ ...string) (string, error) { + return transientPod + "\n", nil + } + + pods, err := k.getAllPodsWith(run, serviceNeo4j) + if !errors.Is(err, errNoPodsMatched) { + t.Errorf("getAllPodsWith() = %v, %v, want errNoPodsMatched", pods, err) + } + if len(pods) != 0 { + t.Errorf("pods = %v, want none", pods) + } + }) + + t.Run("every pod listing carries the label exclusion", func(t *testing.T) { + k := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + k.namespace = "infrahub" + listings := [][]string{} + run := func(_ string, args ...string) (string, error) { + listings = append(listings, args) + + return "", nil + } + + if _, err := k.getAllPodsWith(run, serviceNeo4j); !errors.Is(err, errNoPodsMatched) { + t.Fatalf("getAllPodsWith() error = %v, want errNoPodsMatched", err) + } + if len(listings) == 0 { + t.Fatal("no kubectl listings were issued") + } + for _, args := range listings { + selector := "" + for i, arg := range args { + if arg == "-l" && i+1 < len(args) { + selector = args[i+1] + } + } + if !strings.Contains(selector, transientWorkloadExclusion) { + t.Errorf("listing %v selected on %q, want it to exclude %s server-side", args, selector, transientLabelMarker) + } + } + }) +} + +// podListing renders a kubectl pod listing the way podNamePhaseJSONPath does, +// so a stub runner answers in the shape the resolvers parse. +func podListing(pods ...labelledPod) string { + lines := make([]string, 0, len(pods)) + for _, pod := range pods { + // Rendered through ownershipLine so a labelledPod handed to a fake + // cluster round-trips through the real parser. It used to render + // `;` alone, which silently dropped Service and would + // silently drop Transient — a fake that cannot express the field the + // filter reads is a fake that cannot test the filter. + labels := map[string]string{} + if pod.Transient { + labels[transientLabelMarker] = "true" + } + if pod.Service != "" { + labels[serviceLabelKeys[0]] = pod.Service + } + lines = append(lines, ownershipLine(pod.Name, pod.Phase, labels)) + } + + return strings.Join(lines, "\n") + "\n" +} + +// selectorOf returns the value of the -l argument in a recorded kubectl call. +func selectorOf(args []string) string { + for i, arg := range args { + if arg == "-l" && i+1 < len(args) { + return args[i+1] + } + } + + return "" +} + +// serviceSelectorOf returns the service-label clause of a recorded kubectl +// call's -l argument, with the transient-workload exclusion every pod listing +// now carries removed (see podListingArgsFor). A namespace listing selects on +// the exclusion alone and so has no service clause. +func serviceSelectorOf(args []string) string { + selector := selectorOf(args) + if selector == transientWorkloadExclusion { + return "" + } + + return strings.TrimSuffix(selector, ","+transientWorkloadExclusion) +} + +// isServiceSelectorQuery reports whether a recorded kubectl call selected on a +// service label rather than listing the namespace. `-l` is now present on both, +// so its presence no longer tells them apart; what the call selects on does. +func isServiceSelectorQuery(args []string) bool { + return serviceSelectorOf(args) != "" +} + +// emptyWorkloadListing is `kubectl get -o json` for a namespace that +// holds none of that kind. +const emptyWorkloadListing = `{"items":[]}` + +// isWorkloadListing reports whether a recorded kubectl call lists a workload +// kind rather than pods. +func isWorkloadListing(args []string) bool { + return slices.Contains(workloadKinds, resourceOf(args)) +} + +// labelBlindCluster is the namespace shape every false positive shares: no pod +// declares itself as the service, and the full-namespace listing holds a pod +// whose *name* names it. Label-selected listings answer empty; the unselected +// listing answers with the supplied pods. +func labelBlindCluster(pods ...labelledPod) (podRunner, *int) { + fallbackCalls := 0 + run := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", nil + } + if isWorkloadListing(args) { + // The namespace holds no workloads. A failed workload listing is + // an unanswered location question (FR-002), so the fake answers + // it rather than leaving it to fail. + return emptyWorkloadListing, nil + } + fallbackCalls++ + + return podListing(pods...), nil + } + + return run, &fallbackCalls +} + +// TestLocateServiceWithRejectsPodsThatOnlyNameTheService is finding 1 at the +// decision that matters. Each of these pods names a database service without +// being one, and the tool execs `neo4j-admin`, `pg_dump` — and on restore, a +// load — inside whatever pod the location decision accepted. So a namespace +// holding one of them must still report a genuinely external database as +// external. +func TestLocateServiceWithRejectsPodsThatOnlyNameTheService(t *testing.T) { + tests := []struct { + name string + service string + pod labelledPod + }{ + { + name: "an unrelated StatefulSet's pod", + service: serviceNeo4j, + pod: labelledPod{Name: "postgres-database-0", Phase: "Running"}, + }, + { + name: "a completed backup Job's pod", + service: serviceNeo4j, + pod: labelledPod{Name: "infrahub-database-backup-28123456-x9v2t", Phase: "Succeeded"}, + }, + { + name: "a metrics sidecar", + service: serviceTaskManagerDB, + pod: labelledPod{Name: "task-manager-db-exporter-6b8f7c9d4-lm2xq", Phase: "Running"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run, _ := labelBlindCluster(tt.pod) + + location, err := newTestKubernetesBackend().locateServiceWith(run, tt.service) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationExternal { + t.Errorf("location = %q for a namespace whose only %s-shaped pod is %s, want %q: the run would capture (and on restore write) inside a pod that belongs to something else", + location, tt.service, tt.pod.Name, EndpointLocationExternal) + } + }) + } +} + +// TestGetPodForServiceWithRejectsFinishedPods is the "at minimum" half of +// finding 1 at the resolver an exec target comes from: a pod nothing is running +// in is not somewhere to run a database utility, whatever it is called. +func TestGetPodForServiceWithRejectsFinishedPods(t *testing.T) { + finished := []labelledPod{ + {Name: "infrahub-database-backup-28123456-x9v2t", Phase: "Succeeded"}, + {Name: "infrahub-database-migrate-lm2xq", Phase: "Failed"}, + } + + t.Run("a finished pod is not resolved as the service", func(t *testing.T) { + run, _ := labelBlindCluster(finished...) + + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, serviceNeo4j) + if !errors.Is(err, errNoPodsMatched) { + t.Errorf("getPodForServiceWith() = %q, %v, want errNoPodsMatched", pod, err) + } + if pod != "" { + t.Errorf("pod = %q, want no pod: nothing runs in a %s pod", pod, "Succeeded/Failed") + } + }) + + t.Run("a live pod alongside finished ones still resolves", func(t *testing.T) { + // The namespace declares itself, so `infrahub` is a release prefix this + // deployment established and the database pod is claimed on its name. + // A namespace declaring nothing at all is a different question, and one + // the location decision already answers "external" — the resolver now + // gives the same answer rather than a second one (see + // unanchoredNamePolicyFor). + listing := ownershipListing( + ownershipLine("infrahub-server-7d9f4-abcde", "Running", infrahubServerLabels), + ownershipLine(finished[0].Name, finished[0].Phase, nil), + ownershipLine(finished[1].Name, finished[1].Phase, nil), + ownershipLine("infrahub-database-0", "Running", nil), + ) + run := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", nil + } + + return listing, nil + } + + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("getPodForServiceWith failed: %v", err) + } + if pod != "infrahub-database-0" { + t.Errorf("pod = %q, want infrahub-database-0", pod) + } + }) + + t.Run("every pod listing asks the cluster for live pods only", func(t *testing.T) { + listings := [][]string{} + run := func(_ string, args ...string) (string, error) { + listings = append(listings, args) + + return "", nil + } + + if _, err := newTestKubernetesBackend().getPodForServiceWith(run, serviceNeo4j); !errors.Is(err, errNoPodsMatched) { + t.Fatalf("getPodForServiceWith() error = %v, want errNoPodsMatched", err) + } + if len(listings) == 0 { + t.Fatal("no kubectl listings were issued") + } + for _, args := range listings { + if !contains(args, "--field-selector="+livePodFieldSelector) { + t.Errorf("listing %v does not filter finished pods server-side, want --field-selector=%s", args, livePodFieldSelector) + } + } + }) +} + +// TestNameFallbackStillResolvesTheHelmLabelledPod is the constraint the +// tightening had to preserve: the chart labels some pods with a different +// service value than the canonical name (infrahub-server pods carry +// infrahub/service=server), and both resolvers exist in their current shape +// because of it. infrahub-collect's per-replica log collection enumerates +// through getAllPodsWith, so both are asserted. +func TestNameFallbackStillResolvesTheHelmLabelledPod(t *testing.T) { + helmPod := labelledPod{Name: "infrahub-server-7d9f4b6c8-abcde", Phase: "Running"} + + t.Run("the singular resolver", func(t *testing.T) { + run, fallbackCalls := labelBlindCluster(helmPod) + + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, "infrahub-server") + if err != nil { + t.Fatalf("getPodForServiceWith failed: %v", err) + } + if pod != helmPod.Name { + t.Errorf("pod = %q, want %q resolved by name after the labels missed it", pod, helmPod.Name) + } + if *fallbackCalls == 0 { + t.Error("the name fallback was never reached, so this case did not exercise it") + } + }) + + t.Run("the plural resolver", func(t *testing.T) { + run, _ := labelBlindCluster(helmPod, labelledPod{Name: "infrahub-server-7d9f4b6c8-fghij", Phase: "Running"}) + + pods, err := newTestKubernetesBackend().getAllPodsWith(run, "infrahub-server") + if err != nil { + t.Fatalf("getAllPodsWith failed: %v", err) + } + if len(pods) != 2 || pods[0] != helmPod.Name { + t.Errorf("pods = %v, want both infrahub-server pods resolved by name", pods) + } + }) + + t.Run("a name that merely contains the letters does not match", func(t *testing.T) { + run, _ := labelBlindCluster(labelledPod{Name: "mydatabasetool-0", Phase: "Running"}) + + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, serviceNeo4j) + if !errors.Is(err, errNoPodsMatched) { + t.Errorf("getPodForServiceWith() = %q, %v, want errNoPodsMatched", pod, err) + } + }) +} + +// TestEverySelectorFailingIsNeverAnEmptyMatch is finding 3. Multiple selectors +// exist because charts label differently, so one failing must not abort the +// search — but if every one of them failed, nothing was established, and the +// sentinel would tell the caller "not deployed" (and locateService "external") +// on the strength of queries that were never answered. The shape that reaches +// it: a shrunken --external-db-timeout under which the four selector calls +// expire while the plain listing answers. +func TestEverySelectorFailingIsNeverAnEmptyMatch(t *testing.T) { + // Label-selected listings fail; the unselected fallback listing succeeds + // and holds nothing that names the service. + selectorsExpire := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", fmt.Errorf("context deadline exceeded") + } + + return podListing(labelledPod{Name: "infrahub-server-7d9f4b6c8-abcde", Phase: "Running"}), nil + } + + t.Run("the singular resolver reports the failure", func(t *testing.T) { + _, err := newTestKubernetesBackend().getPodForServiceWith(selectorsExpire, serviceNeo4j) + if err == nil { + t.Fatal("getPodForServiceWith succeeded despite every selector query failing, want error") + } + if errors.Is(err, errNoPodsMatched) { + t.Errorf("unanswered selector queries were reported as an empty match: %v", err) + } + if !strings.Contains(err.Error(), "context deadline exceeded") { + t.Errorf("err = %v, want it to keep the reason the queries failed", err) + } + }) + + t.Run("the plural resolver reports the failure", func(t *testing.T) { + pods, err := newTestKubernetesBackend().getAllPodsWith(selectorsExpire, serviceNeo4j) + if err == nil { + t.Fatalf("getAllPodsWith() = %v, nil, want an error", pods) + } + if errors.Is(err, errNoPodsMatched) { + t.Errorf("unanswered selector queries were reported as an empty match: %v", err) + } + }) + + t.Run("the location decision refuses to call it external", func(t *testing.T) { + location, err := newTestKubernetesBackend().locateServiceWith(selectorsExpire, serviceNeo4j) + if err == nil { + t.Fatal("locateServiceWith succeeded despite every selector query failing, want error") + } + if location == EndpointLocationExternal { + t.Error("a loaded API server turned into an external database, which is FR-002 exactly") + } + if location != "" { + t.Errorf("location = %q, want the zero value alongside an error", location) + } + }) + + t.Run("one selector failing still resolves through the others", func(t *testing.T) { + // The first selector fails; a later one matches. Charts label + // differently, so this must not abort the search. + run := func(_ string, args ...string) (string, error) { + switch serviceSelectorOf(args) { + case "app.kubernetes.io/component=" + serviceNeo4j: + return "", fmt.Errorf("etcdserver: request timed out") + case "infrahub/service=" + serviceNeo4j: + return podListing(labelledPod{Name: "infrahub-database-0", Phase: "Running"}), nil + } + + return "", nil + } + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q: a pod carrying the chart's own label was found", location, EndpointLocationInternal) + } + }) +} + +// resourceOf returns the resource kind a recorded kubectl call asked for. +func resourceOf(args []string) string { + for i, arg := range args { + if arg == "get" && i+1 < len(args) { + return args[i+1] + } + } + + return "" +} + +// labelBlindNamespace answers the three listings the location decision issues: +// label-selected pod queries, which never match; the whole-namespace ownership +// listing, which carries what each pod declares as well as what it is called; +// and the Deployment/StatefulSet listings. It is labelBlindCluster with the +// ownership and workload evidence a namespace actually offers. +// +// pods is an ownershipListing; workloadsByKind maps "deployment"/"statefulset" +// to a workloadListing. A kind with no entry lists nothing. +func labelBlindNamespace(t *testing.T, pods string, workloadsByKind map[string]string) podRunner { + t.Helper() + empty := workloadListing(t, nil) + + return func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", nil + } + if resourceOf(args) == "pods" { + return pods, nil + } + if listing, ok := workloadsByKind[resourceOf(args)]; ok { + return listing, nil + } + + return empty, nil + } +} + +// serverPod is the one resource every Infrahub deployment has and labels: it is +// what establishes the release name the rest of the deployment's names are +// built from. +var serverPod = ownershipLine("infrahub-server-7d9f4b6c8-abcde", "Running", map[string]string{"infrahub/service": "server"}) + +// TestLocateServiceWithReadsOwnershipNotLabelsAlone is the other half of +// finding 1, and the half that had been traded away. Deciding on label evidence +// alone declared a working deployment external and hard-refused the run, for +// three unrelated reasons — and external capture is unavailable in this build, +// so that refusal is the whole backup. Absence of a label is not evidence that +// the database is absent. +func TestLocateServiceWithReadsOwnershipNotLabelsAlone(t *testing.T) { + t.Run("a database the deployment claims by name is internal", func(t *testing.T) { + // Kustomize, a hand-rolled manifest or a different chart version: + // nothing on the database's own pod matches a selector, but the + // namespace says whose deployment this is. + run := labelBlindNamespace(t, ownershipListing( + serverPod, + ownershipLine("infrahub-database-0", "Running", nil), + ownershipLine("infrahub-task-manager-db-0", "Running", nil), + ), nil) + + for _, service := range []string{serviceNeo4j, serviceTaskManagerDB} { + location, err := newTestKubernetesBackend().locateServiceWith(run, service) + if err != nil { + t.Fatalf("locateServiceWith(%s) failed: %v", service, err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q for %s, want %q: the deployment's own release names the pod, and refusing here refuses a backup that works today", + location, service, EndpointLocationInternal) + } + } + }) + + t.Run("a CloudNativePG cluster the deployment claims is internal", func(t *testing.T) { + // The topology findPrimaryPod exists to support: pods named + // -1, labelled cnpg.io/cluster, which is none of the four + // keys a selector queries. + run := labelBlindNamespace(t, ownershipListing( + serverPod, + ownershipLine("infrahub-task-manager-db-1", "Running", map[string]string{"cnpg.io/cluster": "infrahub-task-manager-db"}), + ownershipLine("infrahub-task-manager-db-2", "Running", map[string]string{"cnpg.io/cluster": "infrahub-task-manager-db"}), + ), nil) + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceTaskManagerDB) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q", location, EndpointLocationInternal) + } + }) + + t.Run("a failed pod is not evidence of absence", func(t *testing.T) { + // A pod evicted mid-recreate, or one on a drained node. Where the + // database lives is not the same question as which pod to exec in. + run := labelBlindNamespace(t, ownershipListing( + serverPod, + ownershipLine("infrahub-database-0", "Failed", nil), + ), nil) + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q: a pod that failed is still where the database lives", location, EndpointLocationInternal) + } + }) + + t.Run("a statefulset scaled to zero is internal", func(t *testing.T) { + // No pod to read at all, which is what makes the workload evidence + // necessary rather than merely corroborating. + run := labelBlindNamespace(t, ownershipListing(serverPod), map[string]string{ + "statefulset": workloadListing(t, map[string]map[string]string{ + "infrahub-database": {"infrahub/service": serviceNeo4j}, + }), + }) + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q: the StatefulSet declares the service, it just has no pods right now", location, EndpointLocationInternal) + } + }) + + t.Run("an unlabelled statefulset the deployment claims by name is internal", func(t *testing.T) { + run := labelBlindNamespace(t, ownershipListing(serverPod), map[string]string{ + "statefulset": workloadListing(t, map[string]map[string]string{"infrahub-database": nil}), + }) + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q", location, EndpointLocationInternal) + } + }) + + t.Run("a workload listing that fails is an error, not external", func(t *testing.T) { + // No pod claims the service, so the workloads are what decide whether + // a scaled-down database is still here. Without them that is + // unanswered, and FR-002 names permission denied as a failure to + // query, never as evidence that the database is external. + run := func(_ string, args ...string) (string, error) { + switch { + case isServiceSelectorQuery(args): + return "", nil + case resourceOf(args) == "pods": + return "", nil + default: + return "", fmt.Errorf(`deployments.apps is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "deployments"`) + } + } + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err == nil { + t.Fatalf("locateServiceWith = %q with no error despite the workload listing being forbidden, want error", location) + } + if !strings.Contains(err.Error(), "forbidden") { + t.Errorf("error = %v, want the workload listing's failure carried through", err) + } + if location != "" { + t.Errorf("location = %q, want the zero value alongside an error", location) + } + }) + + t.Run("a pod listing that fails is still an error", func(t *testing.T) { + // The pod listing is the answer, not an addition to it, so FR-002 + // applies to it in full. + run := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", nil + } + + return "", fmt.Errorf("The connection to the server 10.0.0.1:6443 was refused") + } + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err == nil { + t.Fatal("locateServiceWith succeeded despite being unable to list the namespace's pods, want error") + } + if location != "" { + t.Errorf("location = %q, want the zero value alongside an error", location) + } + }) +} + +// TestLocateServiceWithStillRefusesWhatTheDeploymentDoesNotClaim is the +// constraint the reintroduced evidence had to preserve. A name-shaped match +// counts only against a release prefix something declared, so a namespace that +// says whose deployment it is still rejects the resources that merely name a +// database service. +func TestLocateServiceWithStillRefusesWhatTheDeploymentDoesNotClaim(t *testing.T) { + tests := []struct { + name string + service string + pod string + }{ + { + name: "an unrelated StatefulSet's pod", + service: serviceNeo4j, + pod: ownershipLine("postgres-database-0", "Running", nil), + }, + { + name: "a completed backup Job's pod", + service: serviceNeo4j, + pod: ownershipLine("infrahub-database-backup-28123456-x9v2t", "Succeeded", nil), + }, + { + name: "a metrics sidecar", + service: serviceTaskManagerDB, + pod: ownershipLine("task-manager-db-exporter-6b8f7c9d4-lm2xq", "Running", nil), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := labelBlindNamespace(t, ownershipListing(serverPod, tt.pod), nil) + + location, err := newTestKubernetesBackend().locateServiceWith(run, tt.service) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationExternal { + t.Errorf("location = %q for a namespace whose only %s-shaped resource is %q, want %q: the run would capture (and on restore write) inside something that belongs elsewhere", + location, tt.service, tt.pod, EndpointLocationExternal) + } + }) + } + + t.Run("with nothing declaring itself, a name alone claims nothing", func(t *testing.T) { + // No server pod, so no release prefix: the same shape as before this + // evidence was reintroduced. + run := labelBlindNamespace(t, ownershipListing(ownershipLine("infrahub-database-0", "Running", nil)), nil) + + location, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if location != EndpointLocationExternal { + t.Errorf("location = %q, want %q", location, EndpointLocationExternal) + } + }) +} + +// TestLocateServiceWithAsksForPodsThatFailedToo pins the listing +// filter. Excluding Failed is right for resolving an exec target and wrong for +// asking where a database lives, and the two share podListingArgs. +func TestLocateServiceWithAsksForPodsThatFailedToo(t *testing.T) { + listings := [][]string{} + run := func(_ string, args ...string) (string, error) { + listings = append(listings, args) + if isWorkloadListing(args) { + return emptyWorkloadListing, nil + } + + return "", nil + } + + if _, err := newTestKubernetesBackend().locateServiceWith(run, serviceNeo4j); err != nil { + t.Fatalf("locateServiceWith failed: %v", err) + } + if len(listings) == 0 { + t.Fatal("no kubectl listings were issued") + } + for _, args := range listings { + if resourceOf(args) != "pods" { + continue + } + if contains(args, "--field-selector="+livePodFieldSelector) { + t.Errorf("listing %v excludes Failed pods, which is a reason to exclude Succeeded and not Failed from a question about where a database lives", args) + } + if !contains(args, "--field-selector="+completedPodFieldSelector) { + t.Errorf("listing %v does not drop completed pods server-side, want --field-selector=%s", args, completedPodFieldSelector) + } + } +} + +// TestEveryPodListingExcludesTheTransientWorkload is FR-028 stated as the +// structural claim T082 makes: the exclusion is applied where the arguments are +// built, so it holds for every enumerator rather than for the one that +// remembered it. Before this, getAllPodsWith excluded the transient workload +// while getPodStatusesWith, getPodForServiceWith and locateServiceWith listed +// the namespace with no exclusion at all. +func TestEveryPodListingExcludesTheTransientWorkload(t *testing.T) { + enumerators := map[string]func(*KubernetesBackend, podRunner){ + "getPodForServiceWith": func(k *KubernetesBackend, run podRunner) { + _, _ = k.getPodForServiceWith(run, serviceNeo4j) + }, + "getAllPodsWith": func(k *KubernetesBackend, run podRunner) { + _, _ = k.getAllPodsWith(run, serviceNeo4j) + }, + "getPodStatusesWith": func(k *KubernetesBackend, run podRunner) { + _, _ = k.getPodStatusesWith(run, "cache") + }, + "locateServiceWith": func(k *KubernetesBackend, run podRunner) { + _, _ = k.locateServiceWith(run, serviceNeo4j) + }, + } + + for name, enumerate := range enumerators { + t.Run(name+" excludes it from every pod listing", func(t *testing.T) { + listings := [][]string{} + run := func(_ string, args ...string) (string, error) { + listings = append(listings, args) + + return "", nil + } + + enumerate(newTestKubernetesBackend(), run) + + podListings := 0 + for _, args := range listings { + if resourceOf(args) != "pods" { + continue + } + podListings++ + if !strings.Contains(selectorOf(args), transientWorkloadExclusion) { + t.Errorf("listing %v does not exclude the transient workload, want %s in its selector", args, transientWorkloadExclusion) + } + } + if podListings == 0 { + t.Fatal("no pod listings were issued, so the exclusion was not exercised") + } + }) + } +} + +// TestResolverDropsATransientPodALabelSelectorReturned is the client-side half +// of the same exclusion, at the one place it can actually bite: a pod a label +// selector returned is taken without a name match, so a future label change +// that collided with one of the selector keys would hand the transient workload +// straight back as the service's exec target. +func TestResolverDropsATransientPodALabelSelectorReturned(t *testing.T) { + transient := "infrahub-backup-xdb-capture-a1b2c3d4" + run := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return podListing( + labelledPod{Name: transient, Phase: "Running"}, + labelledPod{Name: "infrahub-database-0", Phase: "Running"}, + ), nil + } + + return "", nil + } + + pod, err := newTestKubernetesBackend().getPodForServiceWith(run, serviceNeo4j) + if err != nil { + t.Fatalf("getPodForServiceWith failed: %v", err) + } + if pod == transient { + t.Fatalf("pod = %q, want the deployment's own pod: this run's transient workload is not a replica of %s (FR-028)", pod, serviceNeo4j) + } + if pod != "infrahub-database-0" { + t.Errorf("pod = %q, want infrahub-database-0", pod) + } +} + +// TestResolutionIssuesOneNamespaceListing pins T119's listing reduction to the +// questions the memo is sound for: where a service lives, and which pod is its. +// Both are settled for as long as the pods are, and the namespace-wide ownership +// query names no service at all — so six services asking it is one query, which +// is what it used to be six of. +// +// It is asserted through locateServiceWith rather than through the running check +// it used to be asserted through. That reading had the memo answering a *phase*, +// which is the one question it must not answer, and the loops that poll a phase +// could then never see it change (TestQuiesceConfirmationObservesPodsTerminating). +// The four label selectors per service stay either way: they name the service, so +// they are six different questions. +func TestResolutionIssuesOneNamespaceListing(t *testing.T) { + backend := newTestKubernetesBackend() + + selectorQueries := 0 + listings := 0 + run := func(_ string, args ...string) (string, error) { + if resourceOf(args) != "pods" { + // The workload listings the location decision adds; not a pod + // listing and not what this memo holds. + return emptyWorkloadListing, nil + } + if isServiceSelectorQuery(args) { + selectorQueries++ + + return "", nil + } + listings++ + + return podListing(labelledPod{Name: "infrahub-infrahub-server-7d9f4b6c8-abcde", Phase: "Running"}), nil + } + + for _, service := range appServicesStoppedForBackup { + if _, err := backend.locateServiceWith(run, service); err != nil { + t.Fatalf("locateServiceWith(%s) = %v, want the location determined", service, err) + } + } + + if listings != 1 { + t.Errorf("namespace listings = %d, want 1 for %d location queries (it was one per service)", listings, len(appServicesStoppedForBackup)) + } + + wantSelectors := len(appServicesStoppedForBackup) * len(serviceLabelKeys) + if selectorQueries != wantSelectors { + t.Errorf("selector queries = %d, want %d: one per service per service-label key", selectorQueries, wantSelectors) + } +} + +// TestRunningCheckReadsAFreshNamespaceListing is the other half of the same +// split, at the level the bug lived: the running check must read the cluster +// every time it is asked. +// +// The listing is memoised for resolution, and the running check fell through to +// the very same helper — so the first answer was replayed for the rest of the +// run however long the caller waited. The scripted cluster below reports the pod +// Running once and gone afterwards, which is what a terminating pod looks like, +// and the second reading has to see it. +func TestRunningCheckReadsAFreshNamespaceListing(t *testing.T) { + backend := newTestKubernetesBackend() + + listings := 0 + run := func(_ string, args ...string) (string, error) { + if isServiceSelectorQuery(args) { + return "", nil + } + listings++ + if listings == 1 { + return podListing(labelledPod{Name: "infrahub-cache-0", Phase: "Running", Service: "cache"}), nil + } + + return "", nil + } + + first, err := backend.getPodStatusesWith(run, "cache") + if err != nil { + t.Fatalf("getPodStatusesWith() = %v, want the status determined", err) + } + if !slices.Contains(first, "Running") { + t.Fatalf("first reading = %v, want it to hold Running: the scripted cluster reported the pod up", first) + } + + second, err := backend.getPodStatusesWith(run, "cache") + if err != nil { + t.Fatalf("getPodStatusesWith() = %v on the second reading, want the status determined", err) + } + if slices.Contains(second, "Running") { + t.Errorf("second reading = %v, want the pod gone: the memoised listing was replayed, so no wait can observe a pod terminating", second) + } + + if listings != 2 { + t.Errorf("namespace listings = %d, want 2: the running check reads the cluster each time it is asked", listings) + } + + if len(backend.namespacePodListings) != 0 { + t.Errorf("namespacePodListings = %v, want it untouched: the running check must not write the memo either, or the next resolution reads a phase it happened to catch", + backend.namespacePodListings) + } +} + +// TestQuiesceConfirmationDoesNotReplayAnEarlierPollsAnswer is the production +// loop over the property above, and the reason it is worth a stubbed `kubectl` +// rather than a backend double: the double every other test in this file uses +// answers IsRunning from a map, so it never reaches a cache and cannot fail +// this. +// +// The order in RestoreBackup is wipeTransientData → stopAppContainers → +// confirmAppContainersQuiesced. `kubectl scale` returns as soon as the replica +// count is recorded, so a poll legitimately sees Running while the pod is still +// terminating. With the listing memoised, that answer was replayed for the rest +// of the run: the wait burned its whole window and the restore aborted with +// "still running 5m after it was stopped" — naming a pod that had terminated +// normally, with the data already wiped and the deployment at zero. +// +// The two waits below are the two halves of that, in the order the restore meets +// them, and nothing between them invalidates a cache because nothing between +// them scales anything. The first is what makes the second mean something: a +// wait that gave up because the pod was Running is a wait that observed the pod +// Running, so the second cannot pass on a cluster it never saw up. Neither turns +// on timing — a poll interval decides how many queries a window spends, not what +// any of them answers. +func TestQuiesceConfirmationDoesNotReplayAnEarlierPollsAnswer(t *testing.T) { + // The pod exists for as long as this file does, and only the test writes + // it. A stub that recorded its own calls would be asserting on whether a + // subprocess may write to the test's temporary directory, which is a + // property of the machine rather than of the wait. + pod := filepath.Join(t.TempDir(), "cache-pod-running") + if err := os.WriteFile(pod, []byte("Running\n"), 0o644); err != nil { + t.Fatalf("declaring the cache pod = %v, want nil", err) + } + + bin := t.TempDir() + writeTerminatingPodKubectl(t, filepath.Join(bin, "kubectl"), pod) + t.Setenv("PATH", bin) + + backend := NewKubernetesBackend(&Configuration{}, NewCommandExecutor()) + backend.namespace = "infrahub" + iops := &InfrahubOps{config: &Configuration{}, executor: NewCommandExecutor(), backend: backend} + + err := iops.confirmAppContainersQuiescedWithin([]string{"cache"}, 100*time.Millisecond, 20*time.Millisecond) + if err == nil { + t.Fatal("confirmAppContainersQuiescedWithin() = nil with the cache pod still Running, want the wait to refuse: a restore must not write over a database something may still be writing to") + } + if !strings.Contains(err.Error(), "still running") { + t.Fatalf("err = %v, want the wait's own refusal, so what follows is measured against a pod this run saw Running", err) + } + + if err := os.Remove(pod); err != nil { + t.Fatalf("terminating the cache pod = %v, want nil", err) + } + + if err := iops.confirmAppContainersQuiescedWithin([]string{"cache"}, 2*time.Second, 20*time.Millisecond); err != nil { + t.Fatalf("confirmAppContainersQuiescedWithin() = %v after the pod terminated, want nil: the wait was replaying an earlier poll's listing, so no amount of waiting could observe the pod going", err) + } +} + +// writeTerminatingPodKubectl writes a `kubectl` that reports one Running cache +// pod for as long as `pod` exists, and an empty namespace once it is gone. +// +// The two listing shapes are told apart by the output they ask for: only the +// ownership jsonpath reads `.metadata.labels`, while the label-selector queries +// ask for name and phase alone (see podListingArgsFor). Answering the selectors +// with nothing is what a Helm deployment does — the chart labels those pods with +// a service value the selectors do not carry — and it is what sends the running +// check to the namespace listing this test is about. +func writeTerminatingPodKubectl(t *testing.T, path, pod string) { + t.Helper() + + script := fmt.Sprintf(`#!/bin/sh +case "$*" in +*labels*) + if [ -f %q ]; then + printf 'infrahub-cache-0;Running;;cache\n' + fi + ;; +esac +exit 0 +`, pod) + + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("writing the kubectl stub = %v, want nil", err) + } +} + +// TestNamespacePodListingIsDroppedByAScale is the memo's safety property: a +// caller that has just changed which pods exist must not read the listing from +// before it did. dropPodCaches is what Start and scaleServices call, and it is +// the only thing either of them does about a cache. +func TestNamespacePodListingIsDroppedByAScale(t *testing.T) { + backend := newTestKubernetesBackend() + backend.namespacePodListings[completedPodFieldSelector] = []labelledPod{{Name: "infrahub-cache-0", Phase: "Running"}} + backend.podCache["cache"] = "infrahub-cache-0" + + backend.dropPodCaches() + + if len(backend.namespacePodListings) != 0 { + t.Errorf("namespacePodListings = %v after a scale, want it dropped alongside podCache", backend.namespacePodListings) + } + if len(backend.podCache) != 0 { + t.Errorf("podCache = %v after a scale, want the two cleared together", backend.podCache) + } + if len(backend.serviceLocations) != 0 { + t.Error("scaling cleared serviceLocations; a scale changes which pods exist, not whether a database is one of them") + } +} + +// TestNamespacePodListingDoesNotMemoiseAFailure is FR-002 at the memo: a +// listing that failed says the cluster could not answer, not that the namespace +// holds no pods, so the next caller must be able to ask again. +func TestNamespacePodListingDoesNotMemoiseAFailure(t *testing.T) { + backend := newTestKubernetesBackend() + + calls := 0 + run := func(_ string, _ ...string) (string, error) { + calls++ + + return "", fmt.Errorf("the server was unable to return a response") + } + + for i := range 2 { + if _, err := backend.namespacePodListing(run, ""); err == nil { + t.Fatalf("namespacePodListing() call %d = nil, want the kubectl failure reported", i+1) + } + } + + if calls != 2 { + t.Errorf("kubectl calls = %d, want 2: a failure is not an answer to memoise", calls) + } +} + // fakePod is one pod of a fakeNamespace: its name and labels. type fakePod struct { name string @@ -174,6 +1411,10 @@ func (ns fakeNamespace) run(name string, args ...string) (string, error) { selector = args[i+1] } } + // Every pod listing excludes the tool's transient workloads (see + // excludeTransientWorkloads). None of the pods here is one, so the + // exclusion is removed before the rest of the selector is matched. + selector = strings.TrimPrefix(strings.TrimSuffix(selector, ","+transientWorkloadExclusion), transientWorkloadExclusion) names := []string{} for _, pod := range ns { if selector == "" || selectorMatchesLabels(selector, pod.labels) { diff --git a/src/internal/app/external_backup_gate.go b/src/internal/app/external_backup_gate.go new file mode 100644 index 00000000..f637318d --- /dev/null +++ b/src/internal/app/external_backup_gate.go @@ -0,0 +1,258 @@ +package app + +import ( + "errors" + "fmt" + + "github.com/sirupsen/logrus" +) + +// Where a database lives has to be settled before a backup or restore run +// touches the deployment at all, because every step after that point assumes a +// container to read the database in — or, on the restore side, to write it in. +// +// Without this gate the sequence for an external database is: the edition probe +// execs `cypher-shell` into a container that is not there, the failure is read +// as Community (NewNeo4jEditionInfo's documented fallback), the run announces a +// ten-second abort window, and then scales infrahub-server, task-worker, +// task-manager, task-manager-background-svc, cache and message-queue to zero +// for an offline capture it cannot perform. That is an outage caused by a +// misdiagnosis, followed by a failure — and constitution Principle II +// (NON-NEGOTIABLE) allows a backup no destructive effect on the running +// instance beyond the documented stop/start of containers. +// +// The restore path is the same sequence with the destruction moved earlier and +// made unrecoverable. Without a gate, an external Neo4j restore misdiagnoses the +// edition, wipes transient data, scales six services to zero, half-restores +// PostgreSQL, and then runs `neo4j-admin database load +// --overwrite-destination=true` inside whatever pod the name fallback resolved. +// The deployment is down, its task-manager database has been partly overwritten, +// and the database the operator meant to restore was never touched. +// +// So the location is established first, and a database that is not in the +// deployment stops the run here, with nothing stopped and nothing misreported. + +// Both arms of this gate are now implemented, so neither refuses on the +// capability. What each does instead is resolve every database's location and +// then, for the external ones, prepare them — which is the same "settle it +// before anything is stopped" property the refusals had, with the answer being +// a prepared source or a prepared restore rather than an error. +// +// What the restore arm still refuses is an unauthorised one. Restoring into a +// database the deployment does not manage requires the authorisation FR-009 +// defines, which a capture does not, and that check is a property of the target +// rather than a step: see databaseTarget.reachable. + +// errExternalRestoreUnauthorised is FR-009's refusal: the authorisation to +// write to a database the deployment does not manage was not supplied. +// +// It is a property of the target rather than a step in each restore path, so +// that it is exercised on every external restore whatever entry point reached +// it — which is what stopped it from being forgotten by the path that made such +// a restore possible, the way all three restore entry points once shipped with +// no reachability check at all. +var errExternalRestoreUnauthorised = errors.New("restoring into a database that lives outside the deployment requires an explicit authorisation, which was not supplied") + +// databaseServicesForBackup are the databases a run will read or write, in the +// order they are gated. Neo4j is always touched; the task manager's PostgreSQL +// only when the run includes it, so `--exclude-task-manager` is not made to +// fail on a database it never touches. +func databaseServicesForBackup(includeTaskManager bool) []string { + services := []string{serviceNeo4j} + if includeTaskManager { + services = append(services, serviceTaskManagerDB) + } + + return services +} + +// resolveDatabaseTargets resolves every database an operation will touch, in +// the order they are gated, and is the only way a *run* obtains targets. The +// constructor it calls is directly callable within the package, so what makes +// a target evidence rather than a convention is the witness that constructor +// sets (see databaseTarget.gated). +// +// It is one function rather than one per operation because the thing that was +// wrong was never the content of the two gates — it was that there were two, +// and that adding an entry point meant remembering to call the right one. It is +// also the seam: this is where the resolved locations are handed to the external +// capture and restore paths, and where FR-009's authorisation is checked before +// a restore reaches one. That check happens inside databaseTargetWith — before +// the edition branch, and before a single container has been stopped. +// +// The targets are returned, and every caller reads them. They were discarded — +// `if _, err := databaseTargetWith(…)` — which passes a "is it called?" check +// and leaves T083's design with the shape and none of the substance: the +// location the target carried was established here and then asked for again by +// what came next. +func resolveDatabaseTargets(queries deploymentQueries, operation databaseOperation, includeTaskManager bool, auth ExternalRestoreAuth) (databaseTargets, error) { + return resolveDatabaseTargetsFor(queries, operation, databaseServicesForBackup(includeTaskManager), auth) +} + +// resolveDatabaseTargetsFor is resolveDatabaseTargets for an explicit list of +// databases, for the one restore that may write the task-manager database +// without Neo4j: a single Plakar postgres snapshot. +func resolveDatabaseTargetsFor(queries deploymentQueries, operation databaseOperation, services []string, auth ExternalRestoreAuth) (databaseTargets, error) { + targets := make(databaseTargets, 0, len(services)) + for _, service := range services { + target, err := databaseTargetWith(queries, operation, service, auth) + if err != nil { + return nil, err + } + targets = append(targets, target) + } + + return targets, nil +} + +// prepareDatabaseCapture establishes where every database this run will read +// lives. It runs before the edition probe, before the abort window, and before +// anything is stopped, and it is the only place the run decides that a database +// is reachable the way the in-deployment paths expect. +func (iops *InfrahubOps) prepareDatabaseCapture(includeTaskManager bool) error { + queries, err := iops.deploymentQueries() + if err != nil { + return err + } + + return iops.prepareDatabaseCaptureWith(queries, includeTaskManager) +} + +// prepareDatabaseCaptureWith is prepareDatabaseCapture against the supplied +// queries, following resolveEndpointWith's injection idiom so the gate is +// testable without a cluster. +func (iops *InfrahubOps) prepareDatabaseCaptureWith(queries deploymentQueries, includeTaskManager bool) error { + // The authorisation is passed on the capture path too, and read on neither: + // reachable consults it only for a restore. Passing it uniformly is what + // keeps the constructor's signature from implying that a capture has an + // authorisation question of its own — it does not (FR-009). + targets, err := resolveDatabaseTargets(queries, databaseCapture, includeTaskManager, iops.externalRestoreAuth()) + if err != nil { + return err + } + + // Every location is now established, so what remains is to learn what a + // capture of each external database needs before the run can safely go on: + // its edition, its version, its size and its members. This is the last point + // at which those answers are free — after it, the edition decision may stop + // six Infrahub services for a cold capture. + // + // The targets are what carries the locations across, rather than the service + // names plus a second reading of where each one lives. + return iops.prepareExternalSources(targets) +} + +// externalMisdiagnosisHint is the tail every "this database is external" +// refusal carries: the two things that would make an in-deployment database +// look like an external one. +// +// It is shared rather than repeated because it belongs on all of them equally. +// A misdiagnosed namespace is the likeliest reason an operator is reading any +// of these messages, and a refusal that omits it sends them to configure an +// external database they do not have. +func externalMisdiagnosisHint(service string) string { + return fmt.Sprintf( + "If it does run in this namespace, the run is looking in the wrong place rather than at an external database: "+ + "check --k8s-namespace, and that its pods carry one of the labels services are resolved by "+ + "(app.kubernetes.io/component, app, component or infrahub/service, set to %q)", service) +} + +// prepareDatabaseRestore establishes where every database this restore will +// write lives. It is prepareDatabaseCapture's sibling, and it exists separately +// because a restore has three entry points rather than two and a different +// thing to say when it refuses. +// +// Its placement in each of them is load-bearing: it must run before +// stopAppContainers, because ensureNeo4jDiscovery and ensurePostgresDiscovery +// exec `env` in infrahub-server and task-manager, which that call has just +// scaled to zero — and before wipeTransientData, which is destructive on its +// own. +func (iops *InfrahubOps) prepareDatabaseRestore(includeTaskManager bool) error { + queries, err := iops.deploymentQueries() + if err != nil { + return err + } + + return iops.prepareDatabaseRestoreWith(queries, includeTaskManager) +} + +// prepareDatabaseRestoreWith is prepareDatabaseRestore against the supplied +// queries, following prepareDatabaseCaptureWith's injection idiom so the gate is +// testable without a cluster. +func (iops *InfrahubOps) prepareDatabaseRestoreWith(queries deploymentQueries, includeTaskManager bool) error { + return iops.prepareDatabaseRestoreOfWith(queries, databaseServicesForBackup(includeTaskManager)) +} + +// prepareDatabaseRestoreOf is prepareDatabaseRestore for exactly the databases +// a restore writes. A single Plakar postgres snapshot writes the task-manager +// database and never Neo4j, so it must not be refused, asked for an +// authorisation, or given a workload for an external Neo4j it never touches. +func (iops *InfrahubOps) prepareDatabaseRestoreOf(services []string) error { + queries, err := iops.deploymentQueries() + if err != nil { + return err + } + + return iops.prepareDatabaseRestoreOfWith(queries, services) +} + +// prepareDatabaseRestoreOfWith is prepareDatabaseRestoreOf against the supplied +// queries. +func (iops *InfrahubOps) prepareDatabaseRestoreOfWith(queries deploymentQueries, services []string) error { + targets, err := resolveDatabaseTargetsFor(queries, databaseRestore, services, iops.externalRestoreAuth()) + if err != nil { + return err + } + + logrus.Debugf("Restore will write to %d database(s): %v", len(targets), targets.services()) + + // Every location is established and every external one carries FR-009's + // authorisation, so what remains is to prepare what a restore into each + // external database needs before the run can safely go on: the workload it + // runs in, the trust anchor that workload's client reads, and the server's + // edition and version. This is the last point at which those are free — + // after it, the deployment's own components are scaled to zero and the + // anchor is no longer readable out of them. + // + // The targets are what carries the locations across, rather than the + // service names plus a second reading of where each one lives. + return iops.prepareExternalRestores(targets) +} + +// externalRestoreAuth reads FR-009's authorisation off the configuration, +// tolerating a Configuration that was never built — an absent authorisation is +// the safe reading of "nothing said so", and the one every other channel +// resolves to when it says nothing. +func (iops *InfrahubOps) externalRestoreAuth() ExternalRestoreAuth { + if iops == nil || iops.config == nil { + return ExternalRestoreAuth{} + } + + return iops.config.ExternalDB.Restore +} + +// externalRestoreUnauthorised is what an operator who has not authorised the +// restore reads. It names the two channels FR-010 requires — the flag for an +// attended run, the environment variable for an unattended one — and states +// that nothing was stopped and no data was changed, which on this path is the +// fact that matters most. +// +// It says outright that --force does not confer it, because that is the flag an +// operator reaching a refused restore reaches for: --force overrides a version +// mismatch and an incomplete backup group, and an operator who has learnt that +// it means "I know what I am doing" will try it here. Enabling scheduled +// restore does not confer it either, for the same reason FR-009 names both — a +// capability that arrives as a side effect of an unrelated setting is one +// nobody decided to grant. +// +// It also names the two things that would make an in-deployment database look +// external, because a misdiagnosed namespace is the likeliest reason an +// operator is reading this at all, and authorising a restore into a database +// they do not actually have is the wrong remedy. +func externalRestoreUnauthorised(service string) error { + return fmt.Errorf( + "the %s database does not run in this deployment, and %w; no Infrahub service was stopped and no data was changed. "+ + "Supply --%s, or set %s, to authorise it — deliberately distinct from --force, which does not confer it, and from enabling scheduled restore, which does not either. %s", + service, errExternalRestoreUnauthorised, AllowExternalRestoreFlag, AllowExternalRestoreEnvVar, + externalMisdiagnosisHint(service)) +} diff --git a/src/internal/app/external_backup_gate_test.go b/src/internal/app/external_backup_gate_test.go new file mode 100644 index 00000000..ef3905ee --- /dev/null +++ b/src/internal/app/external_backup_gate_test.go @@ -0,0 +1,755 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// gatedBackend is a deployment whose databases can be placed inside or outside +// it, and which records every service it was asked to stop or start and every +// command it was asked to run. +// +// The recording is the point of the test below: what makes the pre-gate +// behaviour a Principle II violation is not that the run failed, it is that it +// scaled Infrahub to zero on the way to failing. +type gatedBackend struct { + bareBackend + + // locations answers the location query per service; a service absent from + // the map is inside the deployment. + locations map[string]EndpointLocation + // locateErr fails the location query, standing in for a cluster that did + // not answer. + locateErr error + // cypherErr fails every cypher-shell call, standing in for a database + // that did not answer a query. + cypherErr error + + stopped []string + started []string + execs []string +} + +func newGatedBackend() *gatedBackend { + return &gatedBackend{ + bareBackend: bareBackend{name: "kubernetes"}, + locations: map[string]EndpointLocation{}, + } +} + +func (b *gatedBackend) locateService(service string) (EndpointLocation, error) { + if b.locateErr != nil { + return "", b.locateErr + } + if location, ok := b.locations[service]; ok { + return location, nil + } + + return EndpointLocationInternal, nil +} + +func (b *gatedBackend) Exec(service string, command []string, opts *ExecOptions) (string, error) { + b.execs = append(b.execs, strings.Join(command, " ")) + + if len(command) > 0 && command[0] == "cypher-shell" { + if b.cypherErr != nil { + return "", b.cypherErr + } + // The edition probe. Community is what an absent container's failed + // probe defaults to, and it is the branch that stops Infrahub — so a + // gate that fired late would be visible here as stopped services. + return "community\n", nil + } + + return "", nil +} + +func (b *gatedBackend) Stop(services ...string) error { + b.stopped = append(b.stopped, services...) + + return nil +} + +func (b *gatedBackend) Start(services ...string) error { + b.started = append(b.started, services...) + + return nil +} + +// unansweredStatusBackend is a gatedBackend whose running check never answers. +type unansweredStatusBackend struct { + *gatedBackend +} + +func (b *unansweredStatusBackend) IsRunning(service string) (bool, error) { + return false, errors.New("status query not answered") +} + +var ( + _ EnvironmentBackend = (*gatedBackend)(nil) + _ serviceLocator = (*gatedBackend)(nil) +) + +// newGatedOps is a create run against the supplied deployment, with a backup +// directory of its own. +func newGatedOps(t *testing.T, backend EnvironmentBackend, storage BackendType) *InfrahubOps { + t.Helper() + + cfg := createRetentionConfig(t.TempDir(), RetentionConfig{}) + cfg.Backend = storage + cfg.Plakar = &PlakarConfig{RepoPath: t.TempDir()} + + return &InfrahubOps{config: cfg, backend: backend, executor: NewCommandExecutor()} +} + +// TestCreateBackupRefusesAnExternalDatabaseBeforeStoppingAnything is the +// regression this gate exists for. An external Neo4j has no container, so the +// edition probe fails and is read as Community; the run then announced a +// ten-second abort window and scaled infrahub-server, task-worker, +// task-manager, task-manager-background-svc, cache and message-queue to zero +// before the capture failed anyway — an outage caused by a misdiagnosis, which +// constitution Principle II forbids. +// +// Both storage backends are covered because CreateBackup and +// CreatePlakarBackup each carried the sequence. +func TestCreateBackupRefusesAnExternalDatabaseBeforeStoppingAnything(t *testing.T) { + for _, storage := range []BackendType{BackendTarball, BackendPlakar} { + t.Run(string(storage), func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, storage) + + started := time.Now() + err := iops.CreateBackup(true, "all", false, false, true, 0, false, false, "") + elapsed := time.Since(started) + + // This deployment cannot host a transient workload — the double is + // not the real Kubernetes backend — so the run still stops. What + // changed with the capture path is only *where*: it is no longer the + // capability that refuses, it is the workload that cannot be built. + if err == nil { + t.Fatal("CreateBackup() = nil, want the run stopped where it cannot host a transient workload") + } + if !strings.Contains(err.Error(), "transient workload") { + t.Errorf("err = %v, want it to name the workload the external capture needs", err) + } + + // Principle II: nothing was stopped on the way to the refusal. + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none: the run took the deployment down before refusing", backend.stopped) + } + if len(backend.started) > 0 { + t.Errorf("services started = %v, want none: nothing was stopped, so nothing needed restarting", backend.started) + } + + // The edition branch was never reached, so no absent container was + // read as Community and no abort window was announced. + for _, call := range backend.execs { + if strings.HasPrefix(call, "cypher-shell") { + t.Errorf("the edition probe ran against an absent container (%q), want the location settled first", call) + } + } + if elapsed > 5*time.Second { + t.Errorf("the run took %v, want the refusal before the Community branch's ten-second abort window", elapsed) + } + }) + } +} + +// TestPrepareDatabaseCaptureWith covers the gate's own decisions against the +// injected location query, the way resolveEndpointWith is driven. +func TestPrepareDatabaseCaptureWith(t *testing.T) { + locate := func(locations map[string]EndpointLocation, err error) deploymentQueries { + return deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + if err != nil { + return "", err + } + if location, ok := locations[service]; ok { + return location, nil + } + + return EndpointLocationInternal, nil + }, + discover: func(string) error { return nil }, + } + } + + t.Run("all-internal databases pass the gate untouched", func(t *testing.T) { + iops := NewInfrahubOps() + if err := iops.prepareDatabaseCaptureWith(locate(nil, nil), true); err != nil { + t.Errorf("prepareDatabaseCaptureWith() = %v, want nil: an in-deployment backup must be unchanged (FR-015)", err) + } + }) + + t.Run("an external task manager is prepared rather than refused", func(t *testing.T) { + queries := locate(map[string]EndpointLocation{serviceTaskManagerDB: EndpointLocationExternal}, nil) + iops := newGatedOps(t, newGatedBackend(), BackendTarball) + + // The gate now hands an external database to the capture path instead of + // refusing it. Against this double the preparation cannot get as far as a + // workload, which is what the failure says — and that is the assertion + // that matters: the run reached the capture path. + err := iops.prepareDatabaseCaptureWith(queries, true) + if err == nil { + t.Fatal("prepareDatabaseCaptureWith() = nil, want the preparation reported") + } + if !strings.Contains(err.Error(), "transient workload") { + t.Errorf("err = %v, want it to name the workload the external capture needs", err) + } + }) + + t.Run("a database the run excludes is not gated", func(t *testing.T) { + // --exclude-task-manager: the run never reads that database, so where + // it lives cannot stop the run. + queries := locate(map[string]EndpointLocation{serviceTaskManagerDB: EndpointLocationExternal}, nil) + + if err := NewInfrahubOps().prepareDatabaseCaptureWith(queries, false); err != nil { + t.Errorf("prepareDatabaseCaptureWith() = %v, want nil for a database the run excludes", err) + } + }) + + t.Run("a location that could not be established is an error, not a refusal", func(t *testing.T) { + queries := locate(nil, fmt.Errorf(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods"`)) + + err := NewInfrahubOps().prepareDatabaseCaptureWith(queries, true) + if err == nil { + t.Fatal("prepareDatabaseCaptureWith() = nil, want the unanswered location query reported") + } + if !strings.Contains(err.Error(), "is forbidden") { + t.Errorf("err = %v, want it to keep the reason kubectl gave", err) + } + }) +} + +// TestLocateServiceInFallsBackToInternal pins the Docker Compose half of +// FR-015: a backend that cannot answer the location question is not asked to, +// and its deployment stays on exactly the path it is on today. +func TestLocateServiceInFallsBackToInternal(t *testing.T) { + location, err := locateServiceIn(&bareBackend{name: "docker"})(serviceNeo4j) + if err != nil { + t.Fatalf("locateServiceIn failed: %v", err) + } + if location != EndpointLocationInternal { + t.Errorf("location = %q, want %q for a backend with no location query", location, EndpointLocationInternal) + } +} + +// TestRestoreRefusesAnExternalDatabaseBeforeTouchingTheDeployment is the same +// regression on the path where it costs the most. An external Neo4j restore +// misdiagnosed the edition, wiped transient data, scaled six services to zero, +// half-restored PostgreSQL, and then ran `neo4j-admin database load +// --overwrite-destination=true` inside whatever pod the name fallback resolved +// — leaving the deployment down and its task-manager database partly +// overwritten, having never touched the database the operator meant to restore. +// +// All three entry points are covered because each reaches the destructive +// sequence on its own: RestoreBackup carries it, RestorePlakarBackup is not +// reached through RestoreBackup's gate but through the delegation at its first +// line, and RestoreLatestBackup selects an archive and hands it on. +func TestRestoreRefusesAnExternalDatabaseBeforeTouchingTheDeployment(t *testing.T) { + // A gzip stream with nothing restorable in it. The archive has to get past + // the format sniff, and then never be extracted: the refusal comes before + // anything is read out of it. + // A real archive, because the gate reads which databases the restore + // writes from the archive's own metadata. + writeArchive := func(t *testing.T, iops *InfrahubOps) string { + t.Helper() + + return writeRestoreArchive(t, iops) + } + + assertRefused := func(t *testing.T, backend *gatedBackend, err error) { + t.Helper() + + if err == nil { + t.Fatal("the restore = nil, want the external database refused") + } + // The refusal is FR-009's, because none of these runs supplied the + // authorisation. That is now the whole of what an external restore is + // refused on — the capability itself landed — so this is the gate that + // must fire before anything is touched. + if !errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("err = %v, want it to wrap errExternalRestoreUnauthorised", err) + } + for _, want := range []string{serviceNeo4j, "does not run in this deployment", "no data was changed", "--k8s-namespace", AllowExternalRestoreFlag, AllowExternalRestoreEnvVar} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want the message to mention %q", err, want) + } + } + + // Principle II, and the reason the gate sits where it does: the + // discovery its successor needs reads `env` out of infrahub-server and + // task-manager, which stopAppContainers has scaled to zero. + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none: the restore took the deployment down before refusing", backend.stopped) + } + if len(backend.started) > 0 { + t.Errorf("services started = %v, want none", backend.started) + } + // `env` is DetectEnvironment reading the deployment's own credentials, + // which is read-only and happens on every restore — and is exactly why + // the gate cannot be moved any later: those reads need infrahub-server + // and task-manager up. Nothing else may have run. + for _, call := range backend.execs { + if strings.HasPrefix(call, "env") { + continue + } + t.Errorf("command %q ran against the deployment, want the location settled first", call) + } + } + + t.Run("restore", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + + assertRefused(t, backend, iops.RestoreBackup(writeArchive(t, iops), false, false, 0, "", true, false)) + }) + + t.Run("restore --latest", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + writeArchive(t, iops) + + assertRefused(t, backend, iops.RestoreLatestBackup(false, false, false, 0, "", true, false)) + }) + + t.Run("restore on the plakar backend", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendPlakar) + commitGroupSnapshot(t, iops, ComponentNeo4j) + + // Through RestoreBackup, which delegates on its first line: the gate it + // carries is never reached on this backend, so RestorePlakarBackup + // needs its own. + assertRefused(t, backend, iops.RestoreBackup("ignored-on-this-backend", false, false, 0, "", true, false)) + }) + + t.Run("an external task manager is refused too", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + + err := iops.RestoreBackup(writeRestoreArchiveWith(t, iops, true), false, false, 0, "", true, false) + if !errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("RestoreBackup() = %v, want the refusal", err) + } + if !strings.Contains(err.Error(), serviceTaskManagerDB) { + t.Errorf("err = %v, want it to name %q", err, serviceTaskManagerDB) + } + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none", backend.stopped) + } + }) + + // The archive decides which databases a restore writes. One without the + // task-manager database writes only Neo4j, so an external PostgreSQL is + // not this run's concern: refusing it for want of an authorisation, or + // creating a workload for it, would fail a Neo4j-only restore over a + // database it never touches. + t.Run("an external task manager is not gated when the archive omits it", func(t *testing.T) { + backend := &unansweredStatusBackend{gatedBackend: newGatedBackend()} + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + + // The status query fails, so a run that gets past the gate stops at + // the first step after it that asks the deployment anything, rather + // than waiting out the quiesce window. + err := iops.RestoreBackup(writeRestoreArchive(t, iops), false, false, 0, "", true, false) + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("RestoreBackup() = %v, want no authorisation asked for a database the archive does not carry", err) + } + if err == nil || !strings.Contains(err.Error(), "cannot determine whether") { + t.Fatalf("RestoreBackup() = %v, want the run past the gate and stopped at the running check", err) + } + }) + + // T046's other half: --force is not the authorisation. It is the flag an + // operator reaching a refused restore reaches for, since it overrides a + // version mismatch and an incomplete backup group, and every restore above + // passes it — so if it conferred the capability, none of them would have + // refused. + t.Run("--force does not authorise an external restore", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + + forced := iops.RestoreBackup(writeArchive(t, iops), false, false, 0, "", true, false) + if !errors.Is(forced, errExternalRestoreUnauthorised) { + t.Fatalf("RestoreBackup(force=true) = %v, want the authorisation still required", forced) + } + }) + + // With the authorisation the run gets past the gate. It still stops — + // this deployment double cannot host a transient workload — but it stops + // where the *capture* path stops for the same reason, which is the + // evidence that the authorisation is what the gate was refusing on rather + // than the capability. + t.Run("the authorisation gets the run past the gate", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + iops.config.ExternalDB.Restore = ExternalRestoreAuth{Source: ExternalRestoreAuthFlag} + + err := iops.RestoreBackup(writeArchive(t, iops), false, false, 0, "", true, false) + if err == nil { + t.Fatal("RestoreBackup() = nil, want the run stopped where it cannot host a transient workload") + } + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("err = %v, want the authorisation to have been accepted", err) + } + if !strings.Contains(err.Error(), "transient workload") { + t.Errorf("err = %v, want it to name the workload the external restore needs", err) + } + + // Principle II holds either way: an authorised restore that cannot be + // performed must not take the deployment down on its way to saying so. + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none", backend.stopped) + } + }) +} + +// commitGroupSnapshot commits one component snapshot of a single backup group +// into the run's Plakar repository. A group holding only Neo4j is a complete +// one, because it declares no component list to be measured against. +func commitGroupSnapshot(t *testing.T, iops *InfrahubOps, component string) { + t.Helper() + + if iops.config.Plakar.CacheDir == "" { + iops.config.Plakar.CacheDir = t.TempDir() + } + + pathname, body := "/neo4j.dump", []byte("neo4j dump bytes") + if component == ComponentPostgres { + pathname, body = "/"+prefectDumpFilename, []byte("prefect dump bytes") + } + + commitPlakarSnapshot(t, iops, []string{ + TagBackupID + "=20260805_120000", + TagComponent + "=" + component, + TagNeo4jEdition + "=" + neo4jEditionCommunity, + }, pathname, body) +} + +// TestEveryRestorePathGatesTheTaskManagerOnWhatItRestores is e200950's fix +// carried to the two restore paths it did not reach. Each one gated the +// task-manager database on --exclude-taskmanager alone, before the backup it +// restores was read, so a backup without that database was refused, or given +// a transient workload, for an external PostgreSQL the restore never writes. +// Each now gates it on what the backup carries, and still before anything in +// the deployment is stopped. +func TestEveryRestorePathGatesTheTaskManagerOnWhatItRestores(t *testing.T) { + // assertPastTheGate is a run that was not refused on the task manager and + // stopped at the first step after the gate that asks the deployment + // anything: the running check, which unansweredStatusBackend fails. + assertPastTheGate := func(t *testing.T, err error) { + t.Helper() + + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("restore = %v, want no authorisation asked for a database the backup does not carry", err) + } + if err == nil || !strings.Contains(err.Error(), "cannot determine whether") { + t.Fatalf("restore = %v, want the run past the gate and stopped at the running check", err) + } + } + + assertRefusedUntouched := func(t *testing.T, backend *gatedBackend, err error) { + t.Helper() + + if !errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("restore = %v, want the external task manager refused", err) + } + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none", backend.stopped) + } + } + + t.Run("plakar group without postgres is not gated on an external task manager", func(t *testing.T) { + backend := &unansweredStatusBackend{gatedBackend: newGatedBackend()} + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendPlakar) + commitGroupSnapshot(t, iops, ComponentNeo4j) + + assertPastTheGate(t, iops.RestorePlakarBackup(false, false, 0, true, false)) + }) + + t.Run("plakar group with postgres is gated on an external task manager", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendPlakar) + commitGroupSnapshot(t, iops, ComponentNeo4j) + commitGroupSnapshot(t, iops, ComponentPostgres) + + assertRefusedUntouched(t, backend, iops.RestorePlakarBackup(false, false, 0, true, false)) + }) + + t.Run("plakar neo4j snapshot is not gated on an external task manager", func(t *testing.T) { + backend := &unansweredStatusBackend{gatedBackend: newGatedBackend()} + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendPlakar) + iops.config.Plakar.CacheDir = t.TempDir() + kctx, repo := newPlakarSnapshot(t, iops, + []string{TagComponent + "=" + ComponentNeo4j, TagNeo4jEdition + "=" + neo4jEditionCommunity}, + "/neo4j.dump", []byte("neo4j dump bytes")) + + assertPastTheGate(t, iops.restoreSingleSnapshot(kctx, repo, "", false, false, false)) + }) + + t.Run("plakar postgres snapshot is gated on an external task manager", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendPlakar) + iops.config.Plakar.CacheDir = t.TempDir() + kctx, repo := newPlakarSnapshot(t, iops, + []string{TagComponent + "=" + ComponentPostgres}, + "/"+prefectDumpFilename, []byte("prefect dump bytes")) + + assertRefusedUntouched(t, backend, iops.restoreSingleSnapshot(kctx, repo, "", false, false, false)) + }) + + // The S3 leg of restore --latest cannot read the archive before it is + // downloaded, so it gates only Neo4j there and leaves the task manager to + // RestoreBackup, which reads the archive first. + newest := backupRef{Name: backupNameAt(retentionNow)} + + t.Run("restore --latest --s3 downloads despite an external task manager", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceTaskManagerDB] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + client := newFakeS3RestoreClient("prod", newest) + + restored := 0 + err := iops.restoreLatestFromS3(context.Background(), client, "", func(string) error { + restored++ + return nil + }) + if err != nil { + t.Fatalf("restoreLatestFromS3() = %v, want nil", err) + } + if len(client.keys) != 1 || restored != 1 { + t.Errorf("downloads = %v, restores = %d, want the archive downloaded and handed to the restore", client.keys, restored) + } + }) + + t.Run("restore --latest --s3 refuses an external Neo4j before downloading", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + client := newFakeS3RestoreClient("prod", newest) + + err := iops.restoreLatestFromS3(context.Background(), client, "", func(string) error { + t.Fatal("the restore ran, want the run refused before the download") + return nil + }) + assertRefusedUntouched(t, backend, err) + if len(client.keys) > 0 { + t.Errorf("downloads = %v, want none", client.keys) + } + }) +} + +// TestPrepareDatabaseRestoreWith covers the restore gate's own decisions, the +// way TestPrepareDatabaseCaptureWith covers the capture gate's. +func TestPrepareDatabaseRestoreWith(t *testing.T) { + locate := func(locations map[string]EndpointLocation, err error) deploymentQueries { + return deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + if err != nil { + return "", err + } + if location, ok := locations[service]; ok { + return location, nil + } + + return EndpointLocationInternal, nil + }, + discover: func(string) error { return nil }, + } + } + + t.Run("all-internal databases pass the gate untouched", func(t *testing.T) { + if err := NewInfrahubOps().prepareDatabaseRestoreWith(locate(nil, nil), true); err != nil { + t.Errorf("prepareDatabaseRestoreWith() = %v, want nil: an in-deployment restore must be unchanged (FR-015)", err) + } + }) + + t.Run("a database the run excludes is not gated", func(t *testing.T) { + queries := locate(map[string]EndpointLocation{serviceTaskManagerDB: EndpointLocationExternal}, nil) + + if err := NewInfrahubOps().prepareDatabaseRestoreWith(queries, false); err != nil { + t.Errorf("prepareDatabaseRestoreWith() = %v, want nil for a database the run excludes", err) + } + }) + + t.Run("a location that could not be established is an error, not a refusal", func(t *testing.T) { + queries := locate(nil, fmt.Errorf(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods"`)) + + err := NewInfrahubOps().prepareDatabaseRestoreWith(queries, true) + if err == nil { + t.Fatal("prepareDatabaseRestoreWith() = nil, want the unanswered location query reported") + } + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Errorf("err = %v, want the cluster failure rather than a claim about where the database lives (FR-002)", err) + } + if !strings.Contains(err.Error(), "cannot start the restore") { + t.Errorf("err = %v, want it to say the restore never started", err) + } + }) + + t.Run("an unauthorised external restore refuses and a capture does not", func(t *testing.T) { + // They are different capabilities: an external restore additionally + // needs FR-009's authorisation, and a capture must not be made to ask + // for one it has no business requiring. + queries := locate(map[string]EndpointLocation{serviceNeo4j: EndpointLocationExternal}, nil) + iops := newGatedOps(t, newGatedBackend(), BackendTarball) + + restoreErr := iops.prepareDatabaseRestoreWith(queries, true) + if !errors.Is(restoreErr, errExternalRestoreUnauthorised) { + t.Errorf("restore = %v, want the authorisation required", restoreErr) + } + + captureErr := iops.prepareDatabaseCaptureWith(queries, true) + if errors.Is(captureErr, errExternalRestoreUnauthorised) { + t.Errorf("the capture failure %v asks for the restore authorisation", captureErr) + } + }) + + t.Run("an authorised external restore is prepared rather than refused", func(t *testing.T) { + queries := locate(map[string]EndpointLocation{serviceNeo4j: EndpointLocationExternal}, nil) + iops := newGatedOps(t, newGatedBackend(), BackendTarball) + iops.config.ExternalDB.Restore = ExternalRestoreAuth{Source: ExternalRestoreAuthConfig} + + err := iops.prepareDatabaseRestoreWith(queries, true) + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("prepareDatabaseRestoreWith() = %v, want the authorisation accepted", err) + } + // It fails on the workload this deployment double cannot host, which is + // the preparation having been attempted rather than refused. + if err == nil || !strings.Contains(err.Error(), "transient workload") { + t.Errorf("prepareDatabaseRestoreWith() = %v, want the failure to name the transient workload", err) + } + }) +} + +// TestDatabaseTargetWithIsTheOnlyWayToAReachableTarget is T083's structural +// claim. Reachability was a step each entry point performed, which is why all +// three restore entry points shipped without one while both capture entry +// points had one. It is now a property of the target: databaseTargetWith is the +// only constructor, and it hands back a target only where the in-deployment +// paths can operate on the database it names. +func TestDatabaseTargetWithIsTheOnlyWayToAReachableTarget(t *testing.T) { + queries := func(location EndpointLocation, err error) deploymentQueries { + return deploymentQueries{ + locate: func(string) (EndpointLocation, error) { return location, err }, + discover: func(string) error { return nil }, + } + } + + authorised := ExternalRestoreAuth{Source: ExternalRestoreAuthFlag} + + t.Run("an internal database yields the target every existing path uses", func(t *testing.T) { + for _, operation := range []databaseOperation{databaseCapture, databaseRestore} { + // Deliberately unauthorised: FR-009's authorisation is about + // databases the deployment does not manage, and requiring it for + // an in-deployment restore would break every existing one (FR-015). + target, err := databaseTargetWith(queries(EndpointLocationInternal, nil), operation, serviceNeo4j, ExternalRestoreAuth{}) + if err != nil { + t.Fatalf("databaseTargetWith(%s) = %v, want a target: an in-deployment run must be unchanged (FR-015)", operation, err) + } + if target.Service != serviceNeo4j || target.Location != EndpointLocationInternal || target.Operation != operation { + t.Errorf("target = %+v, want %s/%s/%s", target, serviceNeo4j, EndpointLocationInternal, operation) + } + if err := target.reachable(); err != nil { + t.Errorf("reachable() = %v on a target that was handed out, want nil", err) + } + } + }) + + t.Run("an external database yields a target for either operation", func(t *testing.T) { + // Both paths exist now, so an external database is reachable for each + // of them — a capture unconditionally, a restore on the authorisation. + for _, operation := range []databaseOperation{databaseCapture, databaseRestore} { + target, err := databaseTargetWith(queries(EndpointLocationExternal, nil), operation, serviceNeo4j, authorised) + if err != nil { + t.Fatalf("databaseTargetWith(%s) = %v, want a target: the external %s path exists", operation, err, operation) + } + if target.Location != EndpointLocationExternal || target.Operation != operation { + t.Errorf("target = %+v, want %s/%s", target, EndpointLocationExternal, operation) + } + if err := target.reachable(); err != nil { + t.Errorf("reachable() = %v on a target that was handed out, want nil", err) + } + } + }) + + t.Run("a restore refusal says no data was changed", func(t *testing.T) { + _, restoreErr := databaseTargetWith(queries(EndpointLocationExternal, nil), databaseRestore, serviceNeo4j, ExternalRestoreAuth{}) + + if !strings.Contains(restoreErr.Error(), "no data was changed") { + t.Errorf("restore refusal = %v, want it to say no data was changed: on that path it is the fact that matters most", restoreErr) + } + }) + + // FR-009: the authorisation is a gate on the target, not a note in a + // message. An external restore without it refuses differently from one with + // it, which is what makes --allow-external-restore a flag that does + // something rather than a flag that is recorded. + t.Run("an external restore without the authorisation refuses on the authorisation", func(t *testing.T) { + _, err := databaseTargetWith(queries(EndpointLocationExternal, nil), databaseRestore, serviceNeo4j, ExternalRestoreAuth{}) + if !errors.Is(err, errExternalRestoreUnauthorised) { + t.Fatalf("err = %v, want %v", err, errExternalRestoreUnauthorised) + } + for _, want := range []string{AllowExternalRestoreFlag, AllowExternalRestoreEnvVar, "no data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to name %q", err, want) + } + } + // The two flags an operator would otherwise expect to cover this are + // named as not covering it, because both of them read as "I accept the + // risk" elsewhere in this tool (FR-009). + for _, want := range []string{"--force", "scheduled restore"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to say that %q does not confer the authorisation", err, want) + } + } + + // With the authorisation the run gets past the gate, and the target is + // handed out — which is what makes --allow-external-restore a flag that + // does something rather than one that is recorded. + target, authorisedErr := databaseTargetWith(queries(EndpointLocationExternal, nil), databaseRestore, serviceNeo4j, authorised) + if authorisedErr != nil { + t.Errorf("databaseTargetWith(restore) = %v, want the authorisation accepted", authorisedErr) + } + if target.Restore != authorised { + t.Errorf("target.Restore = %+v, want the authorisation that was supplied", target.Restore) + } + }) + + t.Run("a location that could not be established names the run, not the database", func(t *testing.T) { + failed := fmt.Errorf(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot list resource "pods"`) + + for operation, want := range map[databaseOperation]string{ + databaseCapture: "cannot start the backup", + databaseRestore: "cannot start the restore", + } { + _, err := databaseTargetWith(queries("", failed), operation, serviceNeo4j, authorised) + if err == nil { + t.Fatalf("databaseTargetWith(%s) = nil, want the unanswered location query reported (FR-002)", operation) + } + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to say %q", err, want) + } + if errors.Is(err, errExternalRestoreUnauthorised) { + t.Errorf("err = %v, want the cluster failure rather than a claim about where the database lives", err) + } + } + }) +} diff --git a/src/internal/app/external_ca.go b/src/internal/app/external_ca.go new file mode 100644 index 00000000..8cb76484 --- /dev/null +++ b/src/internal/app/external_ca.go @@ -0,0 +1,327 @@ +package app + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/sirupsen/logrus" +) + +// The deployment records the certificate authority its own database connections +// trust in INFRAHUB_DB_TLS_CA_FILE, and discovery reads that setting out of the +// infrahub-server container. It was read and then used for one thing: appending +// `,ca=…` to the endpoint report. Nothing carried it to the client that opens +// the connection, so a deployment whose database certificate is issued by a +// private CA — the ordinary managed-database case — failed at the first probe on +// an unknown issuer, and the only way past it was --external-db-insecure-tls, +// which drops verification altogether rather than making it succeed. +// +// The file cannot be mounted. Its path names a location inside the +// *deployment's* own container: a pod's volumes address the cluster's storage +// rather than another pod's filesystem, and the operator's host does not have +// the file either. So the run carries the bytes across itself, in two steps: +// +// - read the PEM out of the container whose environment named it, with the +// same bounded exec discovery already uses to read that environment; +// - install it where the client will look. The clients here are JVM programs — +// `cypher-shell`, which every Bolt probe statement runs through — and a JVM +// does not read a PEM. It reads a keystore, so the PEM is imported into one +// inside the workload with the `keytool` that ships beside the JVM, and +// JAVA_OPTS points the client at it. +// +// PostgreSQL is the same problem with a different client and a different +// setting, and it was the worse half: `PGSSLMODE=verify-full` was set with no +// `PGSSLROOTCERT` beside it, so libpq looked for `~/.postgresql/root.crt` in a +// transient workload that has no such file and refused the connection before +// reaching the server. The secure default was not merely unhelpful, it was +// unusable, and the only escape — --external-db-insecure-tls — gives up +// verification entirely. Two things fix it: +// +// - the deployment's own anchor, where it names one. The task-manager +// connection URL's `sslrootcert` is the PostgreSQL counterpart of +// INFRAHUB_DB_TLS_CA_FILE, read out of the task-manager container the same +// way and written into the workload as a PEM — which is what libpq reads, +// so there is no keystore step on this arm. +// - `sslrootcert=system` where it names none, which points libpq at the +// image's own trust store. That is what makes verify-full a mode a publicly +// issued certificate can actually satisfy, and it is the same place the +// Neo4j arm leaves its client when there is nothing to install. +// +// What this covers is exactly what the settings describe. INFRAHUB_DB_TLS_* are +// the settings of Infrahub's own driver connection to Neo4j, so they are the +// trust anchor for the Bolt connections this run opens and for nothing else: +// Neo4j's backup protocol authenticates through the server's own backup SSL +// policy rather than through this file. +// +// Every step degrades rather than fails. A CA that cannot be read, written or +// imported leaves the client on the image's default trust store — which is +// where it is today — so the worst outcome of a workload without `keytool`, or +// a JVM launcher that ignores JAVA_OPTS, is the failure the operator already +// gets, never a run that worked yesterday and stops working. + +const ( + // externalCAPEMPath and externalCATrustStorePath sit on the scratch volume + // rather than in the image's own filesystem, which is the one directory the + // transient workload is guaranteed to be able to write to. + externalCAPEMPath = transientScratchPath + "/infrahub-external-ca.pem" + externalCATrustStorePath = transientScratchPath + "/infrahub-external-ca.p12" + + // externalPostgresCAPEMPath is the same file for the PostgreSQL arm, whose + // client reads the PEM directly and needs no keystore. It is named + // separately from externalCAPEMPath because the two hold different + // deployments' anchors — INFRAHUB_DB_TLS_CA_FILE and the task-manager + // connection's sslrootcert are not the same setting and need not be the + // same certificate. + externalPostgresCAPEMPath = transientScratchPath + "/infrahub-external-postgres-ca.pem" + + // externalCATrustStoreAlias names the single entry the store holds. + externalCATrustStoreAlias = "infrahub-external-ca" + + // postgresSystemTrustStore is libpq's own name for "verify against the + // image's trust store" (PostgreSQL 16 and later; the infrahub-backup image + // pins the PostgreSQL 18 client). Naming it explicitly is what makes + // verify-full satisfiable: left unset, libpq looks for + // ~/.postgresql/root.crt and fails on its absence rather than falling back + // to anything. + postgresSystemTrustStore = "system" + + // externalCATrustStorePassword protects a keystore whose entire contents is + // one public certificate the deployment already publishes to every client + // it has. It is not a credential, and FR-014 — which keeps passwords off + // command lines — is about the credentials that authenticate to the + // database. `keytool` requires a store password whether or not there is a + // secret to protect, so this is a fixed, documented literal rather than + // something a caller could mistake for one. + externalCATrustStorePassword = "infrahub-ops" +) + +// pemCertificateMarker is the one thing a file has to contain to be the CA the +// setting claims it is. Checking for it is what turns "the path was wrong, or +// the key was mounted instead" into a message naming the file, rather than an +// import failure from inside the workload. +const pemCertificateMarker = "-----BEGIN CERTIFICATE-----" + +// externalCATrustEnv is the environment that points a JVM client at the store +// installed below. JAVA_OPTS is the documented way to pass JVM flags to the +// Neo4j command-line tooling, and reaches the process as one `env VAR=value` +// argument (see prepareCommand) rather than through a shell, so the spaces in +// it need no quoting. +// +// The store replaces the JVM's default trust anchors rather than adding to +// them, and that is the correct reading of the setting: INFRAHUB_DB_TLS_CA_FILE +// is the anchor Infrahub's own driver trusts for this database, so a store +// holding exactly it gives this run the same trust the deployment has. +func externalCATrustEnv() map[string]string { + return map[string]string{ + "JAVA_OPTS": strings.Join([]string{ + "-Djavax.net.ssl.trustStore=" + externalCATrustStorePath, + "-Djavax.net.ssl.trustStoreType=PKCS12", + "-Djavax.net.ssl.trustStorePassword=" + externalCATrustStorePassword, + }, " "), + } +} + +// writeFileCommand materialises content at path inside a workload without a +// stdin pipe: the bytes travel base64-encoded in the argument vector, which is +// what lets the write go through the same bounded exec every other call on this +// path uses (FR-025) instead of an unbounded write pipe. +func writeFileCommand(path, content string) []string { + encoded := base64.StdEncoding.EncodeToString([]byte(content)) + + return []string{"sh", "-c", fmt.Sprintf("printf %%s %s | base64 -d > %s", shellQuote(encoded), shellQuote(path))} +} + +// importCACommand imports the PEM written above into the keystore the JVM will +// read. -noprompt is required because the exec has no terminal to answer the +// "trust this certificate?" question on. +func importCACommand() []string { + return []string{ + "keytool", "-importcert", "-noprompt", + "-alias", externalCATrustStoreAlias, + "-file", externalCAPEMPath, + "-keystore", externalCATrustStorePath, + "-storetype", "PKCS12", + "-storepass", externalCATrustStorePassword, + } +} + +// readDeploymentCAFile reads the CA certificate the deployment configured out of +// the container whose environment named it — infrahub-server for Neo4j, the +// same container ensureNeo4jDiscovery reads the address settings from, and +// task-manager for PostgreSQL, whose connection URL is where its +// `sslrootcert` came from. The path means nothing outside the container that +// stated it, so the container is part of the read rather than a default. +// +// It is bounded on the control bound rather than the probe bound: it is a call +// to the deployment, not to the database, which is the same distinction +// ensureNeo4jDiscovery draws for the exec that produced the path. +// The certificate is remembered against the container and path it came from, +// because the component it is read out of does not stay up for the whole run. +// A capture workload for PostgreSQL is created after an offline Neo4j capture +// has already scaled task-manager to zero, and this read is part of building +// that workload's client environment — so re-reading it there fails, the +// install falls back to the system trust store, and pg_dump fails verify-full +// against a privately signed server. The gate probes every external database +// while the deployment is still up, which is what puts the entry here in time. +func (iops *InfrahubOps) readDeploymentCAFile(container, path string) (string, error) { + key := container + ":" + path + if certificate, read := iops.deploymentCAFiles[key]; read { + return certificate, nil + } + + output, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "reading the database CA certificate", + "the "+container+" container", container, + []string{"cat", path}, nil, + ) + if err != nil { + return "", err + } + + if !strings.Contains(output, pemCertificateMarker) { + return "", fmt.Errorf("%s holds no PEM certificate", path) + } + + if iops.deploymentCAFiles == nil { + iops.deploymentCAFiles = map[string]string{} + } + iops.deploymentCAFiles[key] = output + + return output, nil +} + +// installExternalCATrust puts the deployment's own CA in front of the probe's +// JVM clients and returns the environment that points them at it. +// +// A nil map is the answer wherever there is nothing to install, and it is not a +// failure: a deployment that configured no CA, or a run that was told not to +// verify at all, has nothing to trust differently. It is also the answer when a +// step fails, for the reason at the top of this file — the client then verifies +// against the image's default anchors, which is what it does today. +// It is also installed once per workload rather than once per caller. The +// restore path asks for the same pod's trust twice — the probe that reads the +// server's version runs Bolt statements, and so does the seed that follows it — +// and `keytool -importcert` refuses an alias the store already holds. Repeating +// the install would therefore fail on the second call and return nil, dropping +// a client that had a perfectly good trust store back onto the image's default +// anchors. So the answer is remembered against the pod that carries it. +func (iops *InfrahubOps) installExternalCATrust(pod string, tls externalTLSDecision) map[string]string { + caFile := iops.config.ExternalDB.Neo4jTLS.CAFile + if caFile == "" || !tls.verify() { + return nil + } + + if env, installed := iops.externalCATrust[pod]; installed { + return env + } + + env := iops.installExternalCATrustOnce(pod, caFile, tls) + + if iops.externalCATrust == nil { + iops.externalCATrust = map[string]map[string]string{} + } + iops.externalCATrust[pod] = env + + return env +} + +// installExternalCATrustOnce is the install itself, performed exactly once per +// workload by the memo above. Every failure arm returns nil, which the memo +// records: a CA that could not be placed in this pod will not be placeable by a +// later caller either, and retrying it would repeat the warning without +// changing the outcome. +func (iops *InfrahubOps) installExternalCATrustOnce(pod, caFile string, tls externalTLSDecision) map[string]string { + if !iops.placeDeploymentCA(pod, "infrahub-server", caFile, externalCAPEMPath, tls) { + return nil + } + + if _, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "installing the database CA certificate", + transientWorkloadTarget(tls.Service), tls.Service, importCACommand(), &ExecOptions{Pod: pod}, + ); err != nil { + logrus.Warnf("Could not import the deployment's certificate authority into a trust store the transient %s workload's clients read, so the server's certificate is verified against the workload's default trust store instead: %v", tls.Service, err) + + return nil + } + + logrus.Infof("Verifying the external %s server's certificate against the authority the deployment records at %s", tls.Service, caFile) + + return externalCATrustEnv() +} + +// placeDeploymentCA reads the certificate authority the deployment records at +// caFile out of one of its own containers and writes it into the transient +// workload at destPath, reporting whether it is there. Each failure is a +// warning naming what the client verifies against instead — the workload's +// default trust store — because, for the reason at the top of this file, the +// answer to an unplaceable CA is the weakest verifying configuration and never +// a non-verifying one. The callers add what their client then needs: a keytool +// import for the JVM, the path for libpq. +func (iops *InfrahubOps) placeDeploymentCA(pod, container, caFile, destPath string, tls externalTLSDecision) bool { + certificate, err := iops.readDeploymentCAFile(container, caFile) + if err != nil { + logrus.Warnf("Could not read the certificate authority the deployment records at %s from the %s container, so the external %s server's certificate is verified against the transient workload's default trust store instead: %v", caFile, container, tls.Service, err) + + return false + } + + if _, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "installing the database CA certificate", + transientWorkloadTarget(tls.Service), tls.Service, + writeFileCommand(destPath, certificate), &ExecOptions{Pod: pod}, + ); err != nil { + logrus.Warnf("Could not place the deployment's certificate authority in the transient %s workload, so the server's certificate is verified against the workload's default trust store instead: %v", tls.Service, err) + + return false + } + + return true +} + +// postgresClientEnv is the environment a PostgreSQL client in a transient +// workload needs beyond its credentials: the TLS mode, and the trust material +// that mode needs to be satisfiable. +// +// The credentials are deliberately not here. PGUSER and PGPASSWORD reach the +// container from the secret the transient workload owns; ExecOptions.Env +// becomes an `env KEY=VALUE` prefix on the exec command line (see +// prepareCommand), which is exactly the exposure FR-014 forbids. What is left +// is TLS configuration, which is a setting rather than a secret — and +// PGSSLROOTCERT names a path or the literal `system`, never a certificate. +func (iops *InfrahubOps) postgresClientEnv(pod string, tls externalTLSDecision) map[string]string { + env := map[string]string{"PGSSLMODE": tls.postgresSSLMode()} + if !tls.verify() { + // require: encrypted, nothing checked. There is nothing for a trust + // anchor to be used for, so naming one would only mislead a log reader. + return env + } + + env["PGSSLROOTCERT"] = iops.installExternalPostgresTrust(pod, tls) + + return env +} + +// installExternalPostgresTrust puts the deployment's own certificate authority +// in front of the workload's libpq clients and returns what PGSSLROOTCERT +// should be: the path it wrote, or postgresSystemTrustStore. +// +// The system store is the answer wherever there is nothing to install and +// wherever a step fails, for the reason at the top of this file: it is the +// weakest verifying configuration rather than a non-verifying one, so the worst +// outcome of an unreadable CA is the "unknown issuer" the operator can act on, +// never a silently unverified connection. +func (iops *InfrahubOps) installExternalPostgresTrust(pod string, tls externalTLSDecision) string { + caFile := iops.config.ExternalDB.PostgresTLS.CAFile + if caFile == "" { + return postgresSystemTrustStore + } + + if !iops.placeDeploymentCA(pod, "task-manager", caFile, externalPostgresCAPEMPath, tls) { + return postgresSystemTrustStore + } + + logrus.Infof("Verifying the external %s server's certificate against the authority the deployment records at %s", tls.Service, caFile) + + return externalPostgresCAPEMPath +} diff --git a/src/internal/app/external_ca_test.go b/src/internal/app/external_ca_test.go new file mode 100644 index 00000000..18926476 --- /dev/null +++ b/src/internal/app/external_ca_test.go @@ -0,0 +1,152 @@ +package app + +import ( + "strings" + "testing" +) + +func newCAOps(backend EnvironmentBackend, caFile string) *InfrahubOps { + cfg := &Configuration{} + cfg.ExternalDB.Neo4jTLS.CAFile = caFile + + return &InfrahubOps{config: cfg, backend: backend, executor: NewCommandExecutor()} +} + +const testCAPEM = "-----BEGIN CERTIFICATE-----\nMIIBtest\n-----END CERTIFICATE-----\n" + +// TestInstallExternalCATrustCarriesTheDeploymentsCA is F5. The CA the +// deployment records in INFRAHUB_DB_TLS_CA_FILE was read into configuration and +// used only to append ",ca=…" to a log line, so a private-CA deployment failed +// at the first probe on an unknown issuer and --external-db-insecure-tls was +// the only way past. The file lives inside the deployment's own container and +// cannot be mounted, so the run carries the bytes across itself. +func TestInstallExternalCATrustCarriesTheDeploymentsCA(t *testing.T) { + verifying := externalTLSDecision{Service: serviceNeo4j, Encrypt: true} + + t.Run("the certificate is read, written and imported", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{"cat": {testCAPEM}}} + env := newCAOps(backend, "/etc/ssl/infrahub/ca.pem").installExternalCATrust("probe-pod", verifying) + + if env == nil { + t.Fatal("installExternalCATrust() = nil, want the JVM trust-store environment") + } + opts := env["JAVA_OPTS"] + for _, want := range []string{ + "-Djavax.net.ssl.trustStore=" + externalCATrustStorePath, + "-Djavax.net.ssl.trustStoreType=PKCS12", + } { + if !strings.Contains(opts, want) { + t.Errorf("JAVA_OPTS = %q, want it to carry %q", opts, want) + } + } + + if got := backend.argvFor("cat"); len(got) != 1 || got[0][1] != "/etc/ssl/infrahub/ca.pem" { + t.Errorf("read commands = %v, want the CA read from the path the deployment named", got) + } + if got := backend.argvFor("keytool"); len(got) != 1 { + t.Fatalf("keytool commands = %v, want exactly one import", got) + } + imported := strings.Join(backend.argvFor("keytool")[0], " ") + if !strings.Contains(imported, externalCAPEMPath) || !strings.Contains(imported, externalCATrustStorePath) { + t.Errorf("keytool argv = %q, want the PEM imported into the store JAVA_OPTS names", imported) + } + if got := backend.argvFor("sh"); len(got) != 1 || !strings.Contains(strings.Join(got[0], " "), externalCAPEMPath) { + t.Errorf("write commands = %v, want the PEM materialised in the workload", got) + } + }) + + t.Run("a deployment that records no CA installs nothing", func(t *testing.T) { + backend := &scriptedProbeBackend{} + if env := newCAOps(backend, "").installExternalCATrust("probe-pod", verifying); env != nil { + t.Errorf("installExternalCATrust() = %v, want nil: there is nothing to trust differently", env) + } + if len(backend.calls) != 0 { + t.Errorf("calls = %v, want none: a deployment with no CA must issue no command for one (FR-015)", backend.calls) + } + }) + + t.Run("an opted-out run installs nothing", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{"cat": {testCAPEM}}} + optedOut := externalTLSDecision{Service: serviceNeo4j, Encrypt: true, OptedOut: true} + if env := newCAOps(backend, "/etc/ssl/infrahub/ca.pem").installExternalCATrust("probe-pod", optedOut); env != nil { + t.Errorf("installExternalCATrust() = %v, want nil: nothing is verified, so nothing needs an anchor", env) + } + }) + + t.Run("a file that is not a certificate degrades rather than failing the run", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{"cat": {"not a certificate"}}} + if env := newCAOps(backend, "/etc/ssl/infrahub/ca.pem").installExternalCATrust("probe-pod", verifying); env != nil { + t.Errorf("installExternalCATrust() = %v, want nil so the client falls back to the image's trust store", env) + } + if got := backend.argvFor("keytool"); len(got) != 0 { + t.Errorf("keytool ran on a file that holds no certificate: %v", got) + } + }) + + t.Run("a workload without keytool degrades rather than failing the run", func(t *testing.T) { + backend := &scriptedProbeBackend{ + answers: map[string][]string{"cat": {testCAPEM}}, + fails: map[string]error{"keytool": errTestCommandMissing}, + } + if env := newCAOps(backend, "/etc/ssl/infrahub/ca.pem").installExternalCATrust("probe-pod", verifying); env != nil { + t.Errorf("installExternalCATrust() = %v, want nil: a missing tool must not fail a run that works today", env) + } + }) +} + +// TestDeploymentCAReadSurvivesTheScaleDown pins the ordering the capture path +// got wrong. A transient workload is created per role, and the PostgreSQL +// capture workload is created *after* an offline Neo4j capture has scaled +// task-manager to zero — so the CA read that builds that workload's client +// environment ran against a component that was no longer there. The install +// then degraded to the system trust store and pg_dump failed verify-full +// against a privately signed server, blaming the certificate rather than the +// container. +// +// The gate probes every external database while the deployment is still up, so +// the fix is that the read is remembered: a later workload uses what the gate +// read rather than asking a component that has since been stopped. +func TestDeploymentCAReadSurvivesTheScaleDown(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{"cat": {testCAPEM}}} + iops := newCAOps(backend, "") + iops.config.ExternalDB.PostgresTLS.CAFile = "/etc/ssl/prefect/ca.pem" + + verifying := externalTLSDecision{Service: serviceTaskManagerDB, Encrypt: true} + + // The gate's probe, with task-manager still up. + if got := iops.installExternalPostgresTrust("probe-pod", verifying); got != externalPostgresCAPEMPath { + t.Fatalf("gate install = %q, want %q: the probe reads the CA while the deployment is up", got, externalPostgresCAPEMPath) + } + + // stopAppContainers has since scaled task-manager to zero, so an exec into + // it fails. The capture workload is created now and needs the same anchor. + backend.fails = map[string]error{"cat": errTestCommandMissing} + + if got := iops.installExternalPostgresTrust("capture-pod", verifying); got != externalPostgresCAPEMPath { + t.Errorf("capture install = %q, want %q: falling back to %q makes pg_dump fail verify-full against a privately signed server", + got, externalPostgresCAPEMPath, postgresSystemTrustStore) + } + + if got := backend.argvFor("cat"); len(got) != 1 { + t.Errorf("CA reads = %d, want exactly one: the read is remembered rather than reissued per workload", len(got)) + } + if got := backend.argvFor("sh"); len(got) != 2 { + t.Errorf("PEM writes = %d, want one per workload: each pod needs the bytes in its own scratch volume", len(got)) + } +} + +// TestWriteFileCommandRoundTrips pins the encoding the PEM travels in. The +// bytes go through the argument vector so the write uses the same bounded exec +// as everything else on this path (FR-025) rather than an unbounded write pipe. +func TestWriteFileCommandRoundTrips(t *testing.T) { + command := writeFileCommand("/scratch/ca.pem", testCAPEM) + if len(command) != 3 || command[0] != "sh" || command[1] != "-c" { + t.Fatalf("writeFileCommand = %v, want an `sh -c` invocation", command) + } + if strings.Contains(command[2], "BEGIN CERTIFICATE") { + t.Errorf("script = %q, want the payload encoded rather than pasted into the shell text", command[2]) + } + if !strings.Contains(command[2], "base64 -d > '/scratch/ca.pem'") && !strings.Contains(command[2], "base64 -d > /scratch/ca.pem") { + t.Errorf("script = %q, want the decoded bytes redirected to the named path", command[2]) + } +} diff --git a/src/internal/app/external_capture.go b/src/internal/app/external_capture.go new file mode 100644 index 00000000..3d3727b0 --- /dev/null +++ b/src/internal/app/external_capture.go @@ -0,0 +1,1010 @@ +package app + +import ( + "fmt" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// A capture from a database outside the deployment has to settle four things +// before its first byte moves, and none of them can be settled from here: the +// server's version, so the FR-006 gate can compare it against the utility that +// will read it; its edition, because the Community mechanism cannot work +// remotely at all (FR-008); the store size, so the capture's scratch space can +// be sized to the database rather than to a guess (FR-022); and each member's +// role, so the run records what it actually observed about its source (FR-005). +// +// All four are read by a short probe workload, because the only client that can +// ask is one running inside the namespace, and a pod's storage sizing is fixed +// when the pod is created — so the pod that answers "how big is the store" can +// never be the pod sized from the answer. That ordering is why chunk 2d built +// two roles rather than one, and this file is where the two are sequenced. +// +// The probe runs at the gate, before the edition probe and before anything is +// stopped, and the capture workload is created later, around the capture itself. +// Splitting them that way is not tidiness. The edition of an external database +// is not readable by exec'ing a container that is not there, so a run that +// probed later would abort in requireDeterminedEdition before it ever reached a +// capture path; and a capture pod created at the gate would spend the run's +// wait-for-running-tasks window burning the activeDeadlineSeconds that FR-025 +// sizes for the capture. +// +// Everything here is reached only for a database this run has positively +// established as external. A deployment whose databases are all internal returns +// from prepareExternalSources before a run identifier is minted, a workload is +// created, a stray is reaped, or a setting of this feature's is consulted +// (FR-015). + +// probedFacts is what the probe workload read. +type probedFacts struct { + // ServerVersion and UtilityVersion are the two halves of the FR-006 + // comparison, both as their source worded them. + ServerVersion string + UtilityVersion string + + // Edition is the server's own edition, lower-cased. It is empty for + // PostgreSQL, which has no such distinction, and it is what an external + // Neo4j deployment's capture-method decision is made from — the in-container + // `cypher-shell` probe every internal run uses cannot answer for a database + // with no container (FR-008). + Edition string + + // StoreSizeBytes is the size the server reported, or zero when it reported + // none. Zero is not a licence to guess a scratch size: it is what makes + // resolveScratchSize fail naming --external-db-scratch-size (FR-022). + StoreSizeBytes int64 + + // Members are the cluster members hosting the database and their roles, as + // observed at capture time (FR-005). + Members []observedMember + + // RolesObserved records that this run asked the server for its members' + // roles at all, which is a different fact from what the answer was. + // + // The two have to stay apart because roleUnknown is a real recorded value: + // a member the server named but gave no role for, or a role query that + // failed, both record unknown, and an operator reading that knows the run + // looked. PostgreSQL is never asked — it has no such notion here — so its + // roles are *absent* rather than unknown, which is the distinction + // contracts/artifact-and-metadata.md draws for `source_roles`. Recording + // unknown for a database whose roles were never sought would claim an + // observation the run never made. + RolesObserved bool + + // Answered is the member that responded to the probe. It is carried because + // it is the one endpoint this run has evidence is reachable, and the + // PostgreSQL dump is taken from it: indexing Hosts[0] instead, as an earlier + // version of this path did, discards that evidence and dumps from a member + // the probe may have just walked past. + // + // It is deliberately *not* used for Neo4j's capture, which is given the whole + // ordered list and reports no such thing (FR-005). + Answered HostPort +} + +// externalSource is what a run established about one external database before +// anything was stopped: where it is, what this run decided about its +// certificate, and what the probe read. +type externalSource struct { + Endpoint *DatabaseEndpoint + TLS externalTLSDecision + Facts probedFacts +} + +// captureTargets is the endpoint list the capture will use, in try order. +func (s *externalSource) captureTargets() []HostPort { + return s.Endpoint.captureTargets() +} + +// observedRoles maps each endpoint the run supplied to the role observed for it +// at capture time (FR-005). Nothing in it says which endpoint served the +// capture, because with several endpoints supplied that is not exposed by any +// documented interface. +func (s *externalSource) observedRoles() map[string]string { + return captureRoles(s.captureTargets(), s.Facts.Members) +} + +// externalCapture is an externalSource with a workload to run the capture in. +type externalCapture struct { + *externalSource + + // Pod is the transient workload's pod, named on every exec and every copy + // this capture makes. It is named rather than registered: publishing it + // under the service's name is what the reverted version of this path did, + // and the resolver's cache is replaced wholesale by Start and scaleServices + // (see ExecOptions.Pod). + Pod string + + // Bound is the timeout on the capture operation itself (FR-025). It is the + // operator's --external-db-timeout rather than the probe's much shorter + // bound, because a capture of a large store can legitimately take hours. + Bound time.Duration + + // ClientEnv is the environment every client command in this pod needs + // beyond its credentials — the TLS mode and the trust anchor for it. It is + // resolved once, when the workload is created, because resolving it writes + // a certificate into that pod: recomputing it per command would repeat the + // write, and computing it without the pod is what left `verify-full` set + // with no anchor to verify against. + ClientEnv map[string]string + + // release gives back the capture workload. Every path that obtains an + // externalCapture must call it, including the failing ones (FR-011). The + // run's own deferred ReleaseTransientWorkloads is the backstop, not the + // plan. + release func() +} + +// Release gives back the capture workload. It tolerates a nil capture, which is +// what an internal database resolves to, so a caller can defer it +// unconditionally rather than guarding the defer. +func (c *externalCapture) Release() { + if c == nil || c.release == nil { + return + } + + c.release() +} + +// execOptions names the capture's workload as the target of one command, with +// any additional environment the client needs beyond its credentials. +// +// The credentials are never here. ExecOptions.Env becomes an `env KEY=VALUE` +// prefix on the exec command line (see prepareCommand), which is exactly the +// exposure FR-014 forbids; the clients read their credentials from the secret +// the transient workload owns. +func (c *externalCapture) execOptions(env map[string]string) *ExecOptions { + return &ExecOptions{Pod: c.Pod, Env: env} +} + +// --------------------------------------------------------------------------- +// Preparing the sources (the gate) +// --------------------------------------------------------------------------- + +// prepareExternalSources probes every external database this run will read and +// records what it established, so that the capture paths and the edition +// decision both read one answer rather than asking again. +// +// It returns before doing anything at all when no database is external, which is +// what keeps FR-015 structural rather than promised. +// +// It takes the targets the gate resolved rather than a list of service names, +// because where each database lives is exactly what a target already carries. +// Taking the names meant asking the location question a second time, in a +// second place, with its own wrapping of the failure — the shape this phase +// exists to remove, and here the copy was the one that was not the gate. +func (iops *InfrahubOps) prepareExternalSources(targets databaseTargets) error { + external := targets.external() + if len(external) == 0 { + return nil + } + + ops, err := iops.externalCaptureOps(external.services()...) + if err != nil { + return err + } + + for _, target := range external { + if err := iops.prepareExternalSourceWith(ops, target); err != nil { + return err + } + } + + return nil +} + +// prepareExternalSourceWith probes one external database. +// +// The order is the requirement. The probe runs first because it is the only +// thing that can answer the questions; the edition refusal and the version gate +// run on what it read, before a capture workload exists at all, because FR-008 +// and FR-006 both require their answer before any data is read and before any +// Infrahub service is stopped for a capture that cannot happen. +// +// It takes the target rather than a service name so that the one thing it may +// only be called for — a database the gate established lives outside the +// deployment — is carried by its argument rather than checked by its caller. +// That is only sound while a target cannot be built without passing the gate, +// which is what the first line below requires and what databaseTarget.gated +// makes checkable. +func (iops *InfrahubOps) prepareExternalSourceWith(ops externalCaptureOps, target databaseTarget) error { + service := target.Service + + endpoint, tls, err := iops.resolveExternalTarget(target, "capture") + if err != nil { + return err + } + + pod, releaseProbe, err := ops.startProbe(service, len(endpoint.Hosts)) + if err != nil { + return err + } + + facts, err := ops.probe(pod, endpoint, tls) + + // The probe has answered, or failed to; either way it is given back before + // anything else happens, so the probe and the capture workload never hold + // scratch space in the namespace at the same time. + releaseProbe() + + if err != nil { + return fmt.Errorf("failed to read what the external %s capture needs from %s: %w", service, endpoint.endpointTarget(), err) + } + + // FR-008 before FR-006: an edition whose capture mechanism cannot work + // remotely is refused whatever the versions say, and refusing on the version + // first would send the operator to change an image that was never the + // problem. + if service == serviceNeo4j { + if err := refuseExternalCommunityCapture(endpoint, facts.Edition); err != nil { + return err + } + } + + // FR-006, at the only point where it can hold: the utility has been read + // from the image that will run the capture, the server has been read over + // the wire, and no data has moved and no service has been stopped. + if err := checkVersionCompatibility(service, facts.UtilityVersion, facts.ServerVersion); err != nil { + return err + } + + iops.recordExternalSource(service, &externalSource{Endpoint: endpoint, TLS: tls, Facts: facts}) + + return nil +} + +// recordExternalSource stores what the probe established for one database. +func (iops *InfrahubOps) recordExternalSource(service string, source *externalSource) { + if iops.externalSources == nil { + iops.externalSources = map[string]*externalSource{} + } + + iops.externalSources[service] = source +} + +// externalSourceFor reports what this run established about one database, and +// nil for a database that runs inside the deployment. +// +// A nil return is the internal case and is not an error: every existing path +// reaches an internal database through its own container, and the capture paths +// branch on this rather than on a location they would have to re-resolve. +func (iops *InfrahubOps) externalSourceFor(service string) *externalSource { + return iops.externalSources[service] +} + +// externalDatabaseFor is what every path that branches on where a database +// lives dispatches on: what this run established about a database outside the +// deployment, whichever direction the run is going, and nil for one that lives +// inside it. +// +// It reads both of the run's records because the question does not depend on +// direction. A capture records a source at its gate; a restore records a +// workload whose source is the same probe of the same server; and the edition +// decision, the cluster check and the redaction refusal ask only what that +// probe read. The edition decision used to read the capture's record alone, so +// a restore against an external Neo4j found nothing there and the run stopped +// in requireDeterminedEdition against a database whose edition its own gate had +// already established — the feature's restore path was unreachable (T148). No +// capture-only counterpart is left to pick instead: a path that needs a capture +// *workload* builds it in externalCaptureOps from the source this returns, and +// a path that needs the restore workload holds what externalRestoreFor returns, +// which is a different type. +// +// Those paths used to branch on externalSourceFor alone, which asks "did +// preparation record something?" rather than "where does this database live?". +// The two coincide only for an entry point that ran a gate. Any other one +// leaves both records empty, a nil then reads as internal, and the run execs +// `neo4j-admin` or `pg_dump` inside a container the database does not live in — +// silently, because nothing on that path asks the question again. So the +// location is the authority here: a nil against a database located *external* +// is neither branch, and it is an error naming what was not done rather than +// the in-deployment path taken by default. +// +// It issues no query a deployment did not already make. The gate memoises every +// location it establishes (see locateService), and on Docker Compose the answer +// is internal by construction — so a run that went through the gate reads a +// memo, and an internal deployment's path is unchanged (FR-015). +func (iops *InfrahubOps) externalDatabaseFor(service string) (*externalSource, error) { + if source := iops.externalSourceFor(service); source != nil { + return source, nil + } + if restore := iops.externalRestores[service]; restore != nil { + return restore.externalSource, nil + } + + return nil, iops.refuseUnpreparedExternal(service, + "nothing was prepared to reach it, so there is no container to run its client in; "+ + "this run reached a database path without the gate that establishes what an external database needs") +} + +// refuseUnpreparedExternal is the answer for a database nothing was prepared +// for: nil where the database runs in the deployment, and a refusal where it +// does not — because a run reaching an external database without its +// preparation has skipped the gate. notPrepared names, for the refusal, what +// this run did not do. +// +// It is shared by the capture and restore readers so the location question is +// asked one way, and it issues no query a deployment did not already make: the +// gate memoises every location it establishes (see locateService), and on +// Docker Compose the answer is internal by construction (FR-015). +func (iops *InfrahubOps) refuseUnpreparedExternal(service, notPrepared string) error { + queries, err := iops.deploymentQueries() + if err != nil { + return err + } + + location, err := queries.locate(service) + if err != nil { + return fmt.Errorf("cannot determine whether the %s database runs in this deployment: %w", service, err) + } + + if location == EndpointLocationExternal { + return fmt.Errorf("the %s database does not run in this deployment and %s. %s", + service, notPrepared, externalMisdiagnosisHint(service)) + } + + return nil +} + +// resolveExternalTarget is what both preparers do before they create anything: +// require the gate's evidence, resolve the endpoint, and decide TLS for it. +// verb names the operation in the refusal, "capture" or "restore into". +// +// The empty-host assertion is unreachable through ResolveEndpoint, which +// refuses an external endpoint it could not address. It is asserted anyway, +// because every path after it walks the host list and an empty one reaching +// them would be an operation addressed at nothing, reported as done. +func (iops *InfrahubOps) resolveExternalTarget(target databaseTarget, verb string) (*DatabaseEndpoint, externalTLSDecision, error) { + if err := target.requireGated(verb); err != nil { + return nil, externalTLSDecision{}, err + } + + endpoint, err := iops.ResolveEndpoint(target.Service) + if err != nil { + return nil, externalTLSDecision{}, err + } + + if len(endpoint.Hosts) == 0 { + return nil, externalTLSDecision{}, fmt.Errorf("cannot %s the external %s database: no endpoint was resolved for it", verb, target.Service) + } + + tls := resolveExternalTLS(target.Service, iops.config.ExternalDB, endpoint.TLS) + endpoint.Report() + tls.Report() + + return endpoint, tls, nil +} + +// --------------------------------------------------------------------------- +// Opening the capture +// --------------------------------------------------------------------------- + +// openExternalCapture creates the workload the capture will run in, sized from +// the store size the probe reported (FR-022), and returns nil for a database +// that runs inside the deployment. +func (iops *InfrahubOps) openExternalCapture(service string) (*externalCapture, error) { + source := iops.externalSourceFor(service) + if source == nil { + return nil, nil + } + + ops, err := iops.externalCaptureOps(service) + if err != nil { + return nil, err + } + + pod, release, err := ops.startCapture(service, source.Facts.StoreSizeBytes) + if err != nil { + return nil, err + } + + return &externalCapture{ + externalSource: source, + Pod: pod, + Bound: externalDBBound(iops.config), + ClientEnv: iops.transientClientEnv(service, pod, source.TLS), + release: release, + }, nil +} + +// transientClientEnv is the environment the clients in a freshly created +// transient workload need beyond their credentials, installing whatever trust +// material that takes into the pod first. +// +// Neo4j's arm is empty on purpose: its capture speaks the backup protocol, +// which authenticates through the server's own backup SSL policy rather than +// through INFRAHUB_DB_TLS_CA_FILE, so the CA the probe's Bolt clients need has +// nothing to do in the capture pod (see external_ca.go). +func (iops *InfrahubOps) transientClientEnv(service, pod string, tls externalTLSDecision) map[string]string { + if service == serviceTaskManagerDB { + return iops.postgresClientEnv(pod, tls) + } + + return nil +} + +// --------------------------------------------------------------------------- +// Binding the sequence to a cluster +// --------------------------------------------------------------------------- + +// externalCaptureOps are the things preparing an operation against an external +// database does to the cluster, and the one thing it does to the database, as +// function values. +// +// They are injected rather than called directly so the sequence above — probe +// first, refusal and gate before any data moves, capture workload sized from +// what the probe read — is assertable without a cluster and without a database. +// That sequence is the part FR-006, FR-008 and FR-022 actually constrain. +// +// The restore path shares them rather than growing its own. What it needs from +// the cluster is what a capture needs — the refusal for a run that may not +// create workloads, one run identifier, one sweep of earlier runs' strays, and +// a workload to execute in — and a second copy of that would be a second place +// a run identifier is minted, which is the invariant the reaper rests on. +type externalCaptureOps struct { + // startProbe creates the probe workload and returns the pod to run in and + // how to give it back. members is how many endpoints the probe may walk, + // which is what budgets the pod's deadline. + startProbe func(service string, members int) (string, func(), error) + + // startCapture creates the capture workload, sized from the store size the + // probe reported, and returns the pod to run in and how to give it back. + startCapture func(service string, storeSizeBytes int64) (string, func(), error) + + // startRestore creates the one workload a restore into an external database + // uses, and returns the pod to run in and how to give it back. It takes no + // size, because nothing a restore puts through the pod is sized by the + // server (see restoreWorkloadSpec). + startRestore func(service string) (string, func(), error) + + // probe reads the facts a capture cannot be built without. + probe func(pod string, endpoint *DatabaseEndpoint, tls externalTLSDecision) (probedFacts, error) +} + +// externalCaptureOps binds the sequence to the detected environment. +// +// External databases are a Kubernetes topology: a Compose deployment has no +// namespace to stand a workload up in, and locateServiceIn answers "internal" +// there by construction, so this is unreachable on Docker rather than refused +// there. +// +// services are the databases the caller will run tools for. They decide direct +// mode (FR-032): when this process runs as a pod in the namespace and has every +// tool those databases need, the operations run here and nothing is created in +// the namespace. +func (iops *InfrahubOps) externalCaptureOps(services ...string) (externalCaptureOps, error) { + backend, err := iops.ensureBackend() + if err != nil { + return externalCaptureOps{}, err + } + + k, ok := backend.(*KubernetesBackend) + if !ok { + return externalCaptureOps{}, fmt.Errorf("cannot reach a database outside the deployment from a %s deployment: reading or writing one needs a transient workload, which only a Kubernetes namespace can host", backend.Name()) + } + + // Before the reaper, which deletes. This is the first line of the only + // function in the tool from which a transient object can come, so a run + // that may not create one is turned back here and nothing in the namespace + // has been touched. + if err := iops.refuseForbiddenExternalCapture(k.namespace); err != nil { + return externalCaptureOps{}, err + } + + // Direct mode (FR-032), after the collect refusal so infrahub-collect is + // turned back in either mode, and before everything below, which exists + // only to create workloads: the image they run, the run identifier they + // are labelled with, and the reaper's listing and deletion of Jobs and + // Secrets. None of it happens, and none of those permissions is exercised, + // when the tools run in this process. + if iops.decideDirectMode(k.namespace, services) { + return iops.localCaptureOps(), nil + } + + // Also before the reaper, and for the same reason: a run with no image to + // give a workload cannot create one, so it must not delete anything either. + // Being here rather than in the spec builder is what makes the refusal land + // before the first kubectl write instead of at the first create, after the + // reaper's deletes (FR-031). + if err := refuseUnpublishedExternalDBImage(iops.config); err != nil { + return externalCaptureOps{}, err + } + + runID, err := iops.externalRunID() + if err != nil { + return externalCaptureOps{}, err + } + + // Strays are reaped once, here, before this run creates the first workload + // of its own (FR-011). + // + // Not earlier: this is the first moment the run knows it will create one at + // all, and reaping lists pods *and secrets* in the namespace — two calls, + // and two RBAC verbs — which an all-internal deployment neither needs nor + // necessarily grants. Putting it on every Kubernetes run would make FR-015's + // "unchanged in behaviour" false, and would fail runs that work today on + // clusters where this tool cannot list secrets. + iops.reapTransientStrays(k, runID) + + start := func(spec transientWorkloadSpec, specErr error) (string, func(), error) { + if specErr != nil { + return "", nil, specErr + } + + // Every role passes through here, so this is the one place the + // credentials' delivery is decided: the deployment's own Secret when + // the run can show it holds them, the owned Secret otherwise. The + // gate creates the first workload while the deployment is still up, + // which is when the component's pod spec can be read; the decision is + // remembered for the workloads created after a scale-down. + spec = iops.withDeploymentCredentialRefs(k, spec) + + workload, err := k.CreateTransientWorkload(spec) + if err != nil { + return "", nil, err + } + + // Asked for rather than read off the struct: ExecutionTarget is where + // the "only a ready workload may be an execution target" precondition is + // enforced, and reading PodName directly would be the caller deciding it + // had been met. + pod, err := workload.ExecutionTarget() + if err != nil { + workload.releaseAfterFailure() + + return "", nil, err + } + + return pod, func() { + if err := workload.Release(); err != nil { + logrus.Warnf("Could not give back the transient %s workload: %v", workload.Service, err) + } + }, nil + } + + return externalCaptureOps{ + startProbe: func(service string, members int) (string, func(), error) { + return start(probeWorkloadSpec(iops.config, k.namespace, runID, service, members)) + }, + startCapture: func(service string, storeSizeBytes int64) (string, func(), error) { + return start(captureWorkloadSpec(iops.config, k.namespace, runID, service, storeSizeBytes)) + }, + startRestore: func(service string) (string, func(), error) { + return start(restoreWorkloadSpec(iops.config, k.namespace, runID, service)) + }, + probe: iops.probeExternalDatabase, + }, nil +} + +// forbidExternalCapture marks this run as one that may not capture a database +// living outside the deployment. +// +// It exists for `infrahub-collect`. That binary is read-only with respect to +// the deployment's workloads and ADR-0003 states so without qualification — +// but `--include-backup` delegates to CreateBackup (ADR-0006), and CreateBackup +// on an external database creates a Pod and a Secret and then deletes them. The +// boundary is drawn here rather than by amending the ADR: a support engineer +// hands the bundle tool to a customer precisely because it provably cannot +// mutate their cluster, and that guarantee is worth more than a backup inside +// the bundle. The accepted cost is that an external-database deployment gets no +// backup collected with its diagnostics; `infrahub-backup create` takes it. +func (iops *InfrahubOps) forbidExternalCapture() { + iops.externalCaptureForbidden = true +} + +// refuseForbiddenExternalCapture is what the operator reads when a run that may +// not create workloads has reached a database that would need one. +// +// It names the condition (a database outside the deployment), the resource (a +// transient Pod and Secret in this namespace) and the action (take the backup +// with infrahub-backup, and re-run the collection without --include-backup), +// which is the three-part shape contracts/cli-surface.md requires of every +// failure on this path. +func (iops *InfrahubOps) refuseForbiddenExternalCapture(namespace string) error { + if !iops.externalCaptureForbidden { + return nil + } + + return fmt.Errorf( + "a database this run must read does not run in this deployment, and capturing it means creating and then deleting a transient pod and secret in namespace %s; "+ + "infrahub-collect never creates, deletes or scales a workload, so it will not take this backup. "+ + "Take it with `infrahub-backup create`, which performs the external capture, and re-run the collection without --include-backup for the diagnostics themselves", + namespace) +} + +// refuseUnpublishedExternalDBImage turns back a run that would have to create a +// transient workload without an image for it: a development build, which has +// no published image of its own, given no --external-db-image (FR-031). +// +// The default is this tool's image at the binary's version, and a development +// build has no version an image was published under (see publishedImageTag). +// Substituting `latest` or the nearest release would run tooling nobody chose +// against a binary it may not match, so the operator is asked to choose. +// +// It is reached through externalCaptureOps, which every external capture and +// restore passes through after the gate has settled where each database lives +// and before any Infrahub service is stopped — so the run ends with nothing +// created, nothing deleted and nothing stopped, which the message states. +func refuseUnpublishedExternalDBImage(cfg *Configuration) error { + if cfg != nil && strings.TrimSpace(cfg.ExternalDB.Image) != "" { + return nil + } + + // A released binary has a default, so an empty image there can only be the + // flag given an empty value. Calling that a development build would send + // the operator after the wrong cause. + if _, published := publishedImageTag(version); published { + return fmt.Errorf( + "a database this run must reach does not run in this deployment, and reaching it means running a transient workload, "+ + "but %s was given an empty value; omit it to run this release's own image, or name an image. Nothing was created or stopped", + externalDBImageOption) + } + + return fmt.Errorf( + "a database this run must reach does not run in this deployment, and reaching it means running a transient workload, "+ + "but this is a development build of infrahub-backup (version %q), which has no published image for that workload to run; "+ + "supply %s with an image built from this repository's Dockerfile, or use a released infrahub-backup. "+ + "Nothing was created or stopped", + BuildRevision(), externalDBImageOption) +} + +// externalRunID is the identifier every transient object this run creates is +// labelled with, minted once. +// +// Once per run, not once per database. Two external databases produce two +// workloads, and minting an identifier for each would label them as two runs — +// which breaks the invariant the reaper and the create-collision refusal both +// rest on, that a transient workload is adopted by exactly one run, and would +// make each database's own workloads look like strays to the other's sweep. +func (iops *InfrahubOps) externalRunID() (string, error) { + if iops.externalRun != "" { + return iops.externalRun, nil + } + + id, err := newRunID() + if err != nil { + return "", err + } + + iops.externalRun = id + + return id, nil +} + +// reapTransientStrays removes what earlier runs left behind, once per run. +// +// A failure to reap is reported and not returned. The objects it would have +// removed are already bounded — a pod by its own activeDeadlineSeconds, a secret +// by the owner reference to that pod — so failing this run over another run's +// litter would trade a working backup for a tidy namespace. +func (iops *InfrahubOps) reapTransientStrays(k *KubernetesBackend, runID string) { + if iops.transientStraysReaped { + return + } + iops.transientStraysReaped = true + + if err := k.ReapTransientStrays(runID); err != nil { + logrus.Warnf("Could not remove transient external-database objects left behind by earlier runs; each is bounded by its own deadline and owner reference: %v", err) + } +} + +// releaseTransientWorkloads gives back every transient workload this run +// created. It is the run's exit path for them (FR-011), deferred by every entry +// point that may create one, and it is safe to call more than once — after the +// individual releases the capture paths already make, and after an outer entry +// point's own deferred call, which then finds nothing left to give back. +// +// Every entry point, not one per run: `restore --latest --s3` runs the restore +// gate before its download and RestoreBackup runs it again, and the paths +// between the two — a failed download, a second database that fails after the +// first was recorded — never reach the inner defer. +func (iops *InfrahubOps) releaseTransientWorkloads() { + // Direct mode's scratch directories first: they hold database content on + // this pod's own disk, and removing them needs no cluster call. + iops.releaseLocalWorkloads() + + k, ok := iops.backend.(*KubernetesBackend) + if !ok { + return + } + + if err := k.ReleaseTransientWorkloads(); err != nil { + logrus.Warnf("Could not remove every transient external-database workload this run created; each carries its own deadline and the cluster reclaims it: %v", err) + } +} + +// --------------------------------------------------------------------------- +// The probe itself +// --------------------------------------------------------------------------- + +// probeExternalDatabase reads the facts from the running probe workload. +// +// Every command it runs is bounded by the probe bound rather than the capture +// bound (FR-025). A server that has accepted a connection and cannot report its +// own version is not slow, and sizing this wait to a capture's is what would let +// a run sit for hours before saying so. +func (iops *InfrahubOps) probeExternalDatabase(pod string, endpoint *DatabaseEndpoint, tls externalTLSDecision) (probedFacts, error) { + switch endpoint.Service { + case serviceNeo4j: + return iops.probeExternalNeo4j(pod, endpoint, tls) + case serviceTaskManagerDB: + return iops.probeExternalPostgres(pod, endpoint, tls) + default: + return probedFacts{}, errNotADatabaseService("probe", endpoint.Service) + } +} + +func (iops *InfrahubOps) probeExternalNeo4j(pod string, endpoint *DatabaseEndpoint, tls externalTLSDecision) (probedFacts, error) { + facts := probedFacts{} + + // The deployment's own certificate authority, carried into the workload + // before the first Bolt connection is opened, so that a server whose + // certificate a private CA issued can be verified rather than only trusted + // blind (see external_ca.go). It installs nothing where there is nothing to + // install, and every Bolt command below runs under whatever it returns. + opts := &ExecOptions{Pod: pod, Env: iops.installExternalCATrust(pod, tls)} + + utility, err := iops.probeUtilityVersion(opts, endpoint, neo4jUtilityVersionCommand()) + if err != nil { + return facts, err + } + facts.UtilityVersion = utility + + uris := endpoint.boltURIs(tls) + if len(uris) == 0 { + return facts, fmt.Errorf("no %s endpoint was resolved to probe", endpoint.Service) + } + + // The server's version and edition are the one probe result the run cannot + // proceed without, so they are what tries every member. The rest is read + // from the member that answered, because a fact one member reports about a + // database is a fact about the database. + uri, index, err := iops.probeNeo4jComponents(opts, endpoint, uris, &facts) + if err != nil { + return facts, err + } + facts.Answered = endpoint.Hosts[index] + + // A store size the server will not report is not fatal here. FR-022 places + // that failure at sizing time, where the message can name + // --external-db-scratch-size and where an operator who supplied it is not + // stopped at all. + if size, err := iops.probeStoreSize(opts, endpoint, uri); err != nil { + logrus.Warnf("Could not read the size of the external %s store, so the capture's scratch space cannot be derived from it: %v", endpoint.Service, err) + } else { + facts.StoreSizeBytes = size + } + + // Roles are likewise best-effort: roleUnknown is a value the run records, so + // a server that will not enumerate its members costs the capture's report a + // detail rather than costing the run. RolesObserved is set either way — the + // run asked, and a failed answer is recorded as unknown rather than as + // nothing having been looked at. + facts.RolesObserved = true + if members, err := iops.probeNeo4jMembers(opts, endpoint, uri); err != nil { + logrus.Warnf("Could not observe the cluster members of the external %s database, so their roles are recorded as %s: %v", endpoint.Service, roleUnknown, err) + } else { + facts.Members = members + } + + return facts, nil +} + +// firstAnsweringEndpoint walks a database's endpoints in try order and returns +// the index of the first one that answers *and* answers something the caller +// accepts. +// +// The second half is the whole reason this is one function. Both probes walk a +// member list, and both are the FR-006 gate's only source for the server +// version — but only the Neo4j arm validated what came back. The PostgreSQL arm +// took whatever the exec returned and stopped, so a member that connected and +// answered nothing ended the walk and the version gate then aborted the run +// with a healthy member never tried. That was found once already, before this +// path was rebuilt, and reintroduced by the rebuild; a shared walk is what stops +// it from being found a third time. What the two legitimately differ on is +// accept — how an answer is read and what makes it usable — and that is now the +// only thing they differ on. +// +// A candidate that fails is warned about and walked past, never returned: the +// error the walk ends with is the last failure, so an operator reading it sees +// why the final member did not answer rather than why the first did not. +func firstAnsweringEndpoint[T any](endpoints []T, what, target string, ask func(T) (string, error), accept func(T, string) error) (int, error) { + var lastErr error + + for index, endpoint := range endpoints { + output, err := ask(endpoint) + if err != nil { + lastErr = err + logrus.Warnf("Could not read %s from %v, trying the next endpoint: %v", what, endpoint, err) + + continue + } + + if err := accept(endpoint, output); err != nil { + lastErr = err + logrus.Warn(lastErr) + + continue + } + + return index, nil + } + + return 0, fmt.Errorf("failed to read %s of %s: %w", what, target, lastErr) +} + +// probeNeo4jComponents walks the members in try order until one reports the +// server's version and edition, which is the same "tried in order" behaviour the +// capture's own --from list has. It reports which member answered so the probes +// that follow do not repeat the walk. +func (iops *InfrahubOps) probeNeo4jComponents(opts *ExecOptions, endpoint *DatabaseEndpoint, uris []string, facts *probedFacts) (string, int, error) { + index, err := firstAnsweringEndpoint(uris, "the server version and edition", endpoint.endpointTarget(), + func(uri string) (string, error) { + return iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the server version", endpoint.endpointTarget(), endpoint.Service, + neo4jComponentsCommand(uri), opts, + ) + }, + func(uri, output string) error { + row := parseCypherRow(output, neo4jVersionColumn, neo4jEditionColumn) + if row[neo4jVersionColumn] == "" || row[neo4jEditionColumn] == "" { + return fmt.Errorf("%s answered %q when asked for its version and edition", uri, strings.TrimSpace(output)) + } + + facts.ServerVersion = row[neo4jVersionColumn] + facts.Edition = strings.ToLower(row[neo4jEditionColumn]) + + return nil + }, + ) + if err != nil { + return "", 0, err + } + + return uris[index], index, nil +} + +func (iops *InfrahubOps) probeExternalPostgres(pod string, endpoint *DatabaseEndpoint, tls externalTLSDecision) (probedFacts, error) { + facts := probedFacts{} + + utility, err := iops.probeUtilityVersion(&ExecOptions{Pod: pod}, endpoint, postgresUtilityVersionCommand()) + if err != nil { + return facts, err + } + facts.UtilityVersion = utility + + if len(endpoint.Hosts) == 0 { + return facts, fmt.Errorf("no %s endpoint was resolved to probe", endpoint.Service) + } + + opts := &ExecOptions{Pod: pod, Env: iops.postgresClientEnv(pod, tls)} + + index, err := firstAnsweringEndpoint(endpoint.Hosts, "the server version", endpoint.endpointTarget(), + func(host HostPort) (string, error) { + return iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the server version", endpoint.endpointTarget(), endpoint.Service, + postgresServerVersionCommand(host, endpoint.Database), opts, + ) + }, + func(host HostPort, output string) error { + // A member that connects and reports no version is not the member + // this capture can be taken from: the FR-006 gate compares the + // server version against the utility's, and an empty one fails it + // on a member that never said anything rather than on a mismatch. + version := strings.TrimSpace(output) + if version == "" { + return fmt.Errorf("%s answered nothing when asked for its version", host) + } + + facts.ServerVersion = version + + return nil + }, + ) + if err != nil { + return facts, err + } + facts.Answered = endpoint.Hosts[index] + + // The size is read from the member that answered, which is also the member + // the dump will be taken from — so the size sizing the scratch volume and + // the database being dumped are the same server's. + output, err := iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the database size", endpoint.endpointTarget(), endpoint.Service, + postgresStoreSizeCommand(facts.Answered, endpoint.Database), opts, + ) + if err != nil { + logrus.Warnf("Could not read the size of the external %s database, so the capture's scratch space cannot be derived from it: %v", endpoint.Service, err) + + return facts, nil + } + + size, err := parseStoreSizeValue(output) + if err != nil { + logrus.Warnf("Could not read the size of the external %s database, so the capture's scratch space cannot be derived from it: %v", endpoint.Service, err) + + return facts, nil + } + facts.StoreSizeBytes = size + + return facts, nil +} + +// probeUtilityVersion reads the version of the tooling in the workload's image. +// It touches no database, so a failure here is the image not carrying the +// utility it was selected for — which is worth saying plainly, because the +// remedy is the image flag rather than anything about the server. +func (iops *InfrahubOps) probeUtilityVersion(opts *ExecOptions, endpoint *DatabaseEndpoint, command []string) (string, error) { + output, err := iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the utility version", transientWorkloadTarget(endpoint.Service), endpoint.Service, + command, opts, + ) + if err != nil { + return "", fmt.Errorf("failed to read the version of %s in the transient %s workload's image: supply %s with an image that carries it: %w", command[0], endpoint.Service, externalDBImageOption, err) + } + + version, err := extractVersionToken(output) + if err != nil { + return "", fmt.Errorf("failed to read the version of %s in the transient %s workload's image: %w", command[0], endpoint.Service, err) + } + + return version, nil +} + +// probeStoreSize reads the store size that sizes the capture's scratch space +// (FR-022). +func (iops *InfrahubOps) probeStoreSize(opts *ExecOptions, endpoint *DatabaseEndpoint, uri string) (int64, error) { + command, err := neo4jStoreSizeCommand(uri, endpoint.Database) + if err != nil { + return 0, err + } + + output, err := iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the store size", endpoint.endpointTarget(), endpoint.Service, + command, opts, + ) + if err != nil { + return 0, err + } + + return parseStoreSizeValue(parseCypherScalar(output, neo4jStoreSizeColumn)) +} + +// probeNeo4jMembers observes which members host the database and in what role +// (FR-005). It records what it saw; it does not record, and cannot observe, +// which of them will serve the capture. +func (iops *InfrahubOps) probeNeo4jMembers(opts *ExecOptions, endpoint *DatabaseEndpoint, uri string) ([]observedMember, error) { + command, err := neo4jMemberRolesCommand(uri, endpoint.Database) + if err != nil { + return nil, err + } + + output, err := iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "observing the cluster members", endpoint.endpointTarget(), endpoint.Service, + command, opts, + ) + if err != nil { + return nil, err + } + + members := parseNeo4jMemberRoles(output) + if len(members) == 0 { + return nil, fmt.Errorf("the server reported no members hosting %s", endpoint.Database) + } + + return members, nil +} + +// copyFromTransientWorkload copies an artifact out of a named transient pod, +// bounded like every other call this path makes (FR-025). +// +// It names the pod rather than resolving the service, for the reason +// ExecOptions.Pod exists: the workload stands in for a database with no +// container, so resolving "database" here would find nothing — and registering +// the pod under that name, which is what the reverted version of this path did, +// is undone by any StartServices the run makes. +func (iops *InfrahubOps) copyFromTransientWorkload(pod, src, dest string) error { + return iops.copyAcrossTransientWorkload(copyOutOfTransientWorkload, pod, src, dest) +} diff --git a/src/internal/app/external_capture_test.go b/src/internal/app/external_capture_test.go new file mode 100644 index 00000000..f1a0e1be --- /dev/null +++ b/src/internal/app/external_capture_test.go @@ -0,0 +1,1610 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +// The capture path is built so that everything decidable without a database is +// decided by a pure function over values: the argv that reaches a customer's +// server, the refusal an unsupported edition produces, the verdict a capture's +// own report earns, and where the credentials are and are not. Those are what +// this file asserts. +// +// What it cannot assert is an actual capture. There is no external Neo4j or +// PostgreSQL here, so no test below establishes that `neo4j-admin database +// backup --from=…` succeeds against a real server, that the store-size JMX bean +// is exposed by a given deployment, or that a real partial capture produces the +// output these verdicts are written for. Those belong to the end-to-end +// topology (research R8, T072). + +// --------------------------------------------------------------------------- +// T034 / FR-015: one argv builder, and the internal one unchanged +// --------------------------------------------------------------------------- + +// TestNeo4jCaptureCommandInternalArgvIsUnchanged pins FR-015 character for +// character. Both in-container capture entry points now build their command +// through neo4jCaptureCommand, so the thing that could silently change an +// existing deployment's behaviour is this function, and these are the two argvs +// it must keep producing. +func TestNeo4jCaptureCommandInternalArgvIsUnchanged(t *testing.T) { + cases := []struct { + name string + req neo4jCaptureRequest + want []string + }{ + { + name: "the directory capture, as backupNeo4jEnterprise wrote it", + req: neo4jCaptureRequest{ + Database: "neo4j", + BackupMetadata: "all", + ToPath: neo4jTempBackupDir, + }, + want: []string{"neo4j-admin", "database", "backup", "--expand-commands", "--include-metadata=all", "--to-path=/tmp/infrahubops", "neo4j"}, + }, + { + name: "the streaming capture, as backupNeo4jEnterpriseStream wrote it", + req: neo4jCaptureRequest{ + Database: "neo4j", + BackupMetadata: "all", + ToPath: neo4jTempBackupDir, + Uncompressed: true, + }, + want: []string{"neo4j-admin", "database", "backup", "--expand-commands", "--include-metadata=all", "--compress=false", "--to-path=/tmp/infrahubops", "neo4j"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := neo4jCaptureCommand(tc.req) + if strings.Join(got, "\x00") != strings.Join(tc.want, "\x00") { + t.Errorf("neo4jCaptureCommand() = %q\nwant %q", got, tc.want) + } + }) + } +} + +// TestNeo4jCaptureCommandAddsTheOrderedFromList is T034. The members are one +// --from value, comma-separated, in the order they were resolved, because that +// is the order the command tries them in (FR-005). +func TestNeo4jCaptureCommandAddsTheOrderedFromList(t *testing.T) { + got := neo4jCaptureCommand(neo4jCaptureRequest{ + Database: "neo4j", + BackupMetadata: "all", + ToPath: externalNeo4jCaptureDir, + TempPath: externalNeo4jStagingDir, + From: []HostPort{ + {Host: "db-a.example.net", Port: 6362}, + {Host: "db-b.example.net", Port: 6362}, + {Host: "2001:db8::1", Port: 6362}, + }, + Uncompressed: true, + }) + + want := []string{ + "neo4j-admin", "database", "backup", + "--include-metadata=all", + "--compress=false", + "--from=db-a.example.net:6362,db-b.example.net:6362,[2001:db8::1]:6362", + "--temp-path=" + externalNeo4jStagingDir, + "--to-path=" + externalNeo4jCaptureDir, + "neo4j", + } + if strings.Join(got, "\x00") != strings.Join(want, "\x00") { + t.Errorf("neo4jCaptureCommand() = %q\nwant %q", got, want) + } +} + +// TestExpandCommandsIsOnlyForTheInContainerCapture pins a defect found by running +// the real thing rather than asserting a string. +// +// `neo4j-admin` refuses to read a configuration file whose permissions are looser +// than 0640 when it is given `--expand-commands`, and it reads both `neo4j.conf` +// and `neo4j-admin.conf`. The image this tool pins for the transient workload +// ships them world-writable — `neo4j:5-enterprise` has `neo4j.conf` at 0777 — +// so the flag aborted every external capture before the database was contacted, +// blaming a file permission the operator never set: +// +// Failed to read config /var/lib/neo4j/conf/neo4j.conf: … does not have the +// correct file permissions to evaluate commands. +// +// Verified 2026-09-03 against neo4j:5-enterprise (5.26.30): with the flag the +// capture fails on config permissions; without it the identical argv completes a +// full remote capture from a live backup listener. +// +// The in-container capture keeps the flag, because there the configuration being +// expanded is the deployment's own — and that is also what holds FR-015. +func TestExpandCommandsIsOnlyForTheInContainerCapture(t *testing.T) { + hasExpand := func(argv []string) bool { + for _, arg := range argv { + if arg == "--expand-commands" { + return true + } + } + + return false + } + + t.Run("the in-container capture keeps it", func(t *testing.T) { + argv := neo4jCaptureCommand(neo4jCaptureRequest{ + Database: "neo4j", + BackupMetadata: "all", + ToPath: neo4jTempBackupDir, + }) + if !hasExpand(argv) { + t.Errorf("the in-container capture must keep --expand-commands (FR-015); got %q", argv) + } + }) + + t.Run("a capture with a member list does not", func(t *testing.T) { + argv := neo4jCaptureCommand(neo4jCaptureRequest{ + Database: "neo4j", + BackupMetadata: "all", + ToPath: externalNeo4jCaptureDir, + From: []HostPort{{Host: "db.example.net", Port: 6362}}, + }) + if hasExpand(argv) { + t.Errorf("a remote capture must not pass --expand-commands: the pinned image's conf files are world-writable and neo4j-admin refuses to read them; got %q", argv) + } + }) +} + +// TestCaptureTargetsUsesEachServicesOwnPort is the arm the reverted version of +// this path did not have. Stamping the Neo4j backup port onto a PostgreSQL host +// recorded :6362 as the provenance of a dump that travelled over :5432 — a +// false statement about where the data came from. +func TestCaptureTargetsUsesEachServicesOwnPort(t *testing.T) { + t.Run("Neo4j members are re-pointed at the backup listener", func(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceNeo4j, + BackupPort: 6362, + // Deliberately carrying client ports of their own: an address list's + // ports are Bolt ports, never the backup listener's. + Hosts: []HostPort{{Host: "a", Port: 7687}, {Host: "b", Port: 7688}}, + } + + if got := renderHostPorts(endpoint.captureTargets()); got != "a:6362,b:6362" { + t.Errorf("captureTargets() = %q, want both members on the backup port", got) + } + }) + + t.Run("a Neo4j endpoint with no backup port falls back to the vendor default", func(t *testing.T) { + endpoint := DatabaseEndpoint{Service: serviceNeo4j, Hosts: []HostPort{{Host: "a", Port: 7687}}} + + if got := renderHostPorts(endpoint.captureTargets()); got != fmt.Sprintf("a:%d", defaultNeo4jBackupPort) { + t.Errorf("captureTargets() = %q, want the default backup port", got) + } + }) + + t.Run("PostgreSQL members keep the client port the dump travels over", func(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceTaskManagerDB, + // A backup port set on the configuration must not reach here: it is + // a Neo4j setting and PostgreSQL has no second listener. + BackupPort: 6362, + Hosts: []HostPort{{Host: "pg-a", Port: 5432}, {Host: "pg-b", Port: 5433}}, + } + + if got := renderHostPorts(endpoint.captureTargets()); got != "pg-a:5432,pg-b:5433" { + t.Errorf("captureTargets() = %q, want each member's own client port", got) + } + }) +} + +// --------------------------------------------------------------------------- +// T035 / FR-008: refusing an external Community capture +// --------------------------------------------------------------------------- + +func TestRefuseExternalCommunityCapture(t *testing.T) { + endpoint := &DatabaseEndpoint{Service: serviceNeo4j, Hosts: []HostPort{{Host: "db.example.net", Port: 7687}}} + + t.Run("a Community server is refused, naming the mechanism's requirement", func(t *testing.T) { + err := refuseExternalCommunityCapture(endpoint, "community") + if !errors.Is(err, errExternalCommunityCapture) { + t.Fatalf("err = %v, want %v", err, errExternalCommunityCapture) + } + + // The failure-message contract: the mechanism needs the database's own + // storage, the remote alternative is an Enterprise feature, and nothing + // is missing from the deployment. The last clause is the one the + // pre-feature "service not found" got wrong. + for _, want := range []string{ + "direct access to the database's own storage", + "Enterprise Edition feature", + "Nothing is missing from the deployment", + "No Infrahub service was stopped", + "db.example.net:7687", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to say %q", err, want) + } + } + }) + + t.Run("an Enterprise server is not refused", func(t *testing.T) { + if err := refuseExternalCommunityCapture(endpoint, "enterprise"); err != nil { + t.Errorf("refuseExternalCommunityCapture(enterprise) = %v, want nil", err) + } + }) + + t.Run("an edition the server did not report is not read as Community", func(t *testing.T) { + // The same asymmetry Neo4jEditionInfo documents: guessing the + // destructive branch from an answer that never came is what Principle II + // forbids. requireDeterminedEdition is what stops such a run instead. + for _, edition := range []string{"", " ", "unknown"} { + if err := refuseExternalCommunityCapture(endpoint, edition); err != nil { + t.Errorf("refuseExternalCommunityCapture(%q) = %v, want nil", edition, err) + } + } + }) +} + +// --------------------------------------------------------------------------- +// T036 / FR-012: completeness from what the operation reported +// --------------------------------------------------------------------------- + +// TestNeo4jArtifactProducedRequiresTheSeparator is the defect that made a run +// record capture_complete for a capture of nothing. `` as a bare prefix +// is satisfied by a sibling database's artifact, so the run reported a complete +// capture having captured a different database. +func TestNeo4jArtifactProducedRequiresTheSeparator(t *testing.T) { + cases := []struct { + name string + database string + listing string + want bool + }{ + {"the database's own artifact", "neo4j", "neo4j-2026-09-03T00-00-00.backup", true}, + {"a sibling database's artifact does not count", "neo4j", "neo4j2-2026-09-03T00-00-00.backup", false}, + {"a longer sibling name does not count", "infrahub", "infrahub-staging-2026-09-03T00-00-00.backup\n", false}, + {"an unrelated database does not count", "neo4j", "prefect-2026-09-03T00-00-00.backup", false}, + {"a non-artifact file does not count", "neo4j", "neo4j-2026-09-03T00-00-00.tmp", false}, + {"an empty listing", "neo4j", "", false}, + {"the artifact among other entries", "neo4j", "\nstaging\nneo4j-2026-09-03T00-00-00.backup\n", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := neo4jArtifactProduced(tc.database, tc.listing); got != tc.want { + t.Errorf("neo4jArtifactProduced(%q, %q) = %v, want %v", tc.database, tc.listing, got, tc.want) + } + }) + } +} + +// TestClassifyNeo4jCapture is the completeness signal itself. Exit status alone +// cannot establish it — code 1 means "failed, or succeeded with some servers +// uncontactable" and the vendor documents no marker separating them — so the +// signal is a reported success *and* the artifact being there, and nothing else +// may promote a capture to complete. +func TestClassifyNeo4jCapture(t *testing.T) { + failed := errors.New("exit status 1") + // The endpoints the capture was asked to try. They are what make an + // "unreachable" line attributable to this database rather than to the + // cluster the command ran through (see uncontactableEvidence). + captureEndpoints := []HostPort{{Host: "db-a", Port: 6362}, {Host: "db-b", Port: 6362}, {Host: "db-b.example.net", Port: 6362}} + + t.Run("reported success with the artifact is the only complete verdict", func(t *testing.T) { + verdict := classifyNeo4jCapture(nil, "Backup complete.", true, captureEndpoints) + if !verdict.Complete { + t.Errorf("verdict = %+v, want complete", verdict) + } + if verdict.Detail != "" { + t.Errorf("Detail = %q, want empty for a complete capture", verdict.Detail) + } + }) + + t.Run("a reported success that mentions an unreachable member is still the vendor's success", func(t *testing.T) { + // Not overridden on a phrase match: the markers corroborate, they never + // decide, because a phrasing that changes between releases must not be + // able to refuse work the server actually did. + verdict := classifyNeo4jCapture(nil, "Backup complete.\nServer db-b was uncontactable.", true, captureEndpoints) + if !verdict.Complete { + t.Errorf("verdict = %+v, want complete: the operation reported success and left the artifact", verdict) + } + }) + + t.Run("a reported success with no artifact is not complete", func(t *testing.T) { + verdict := classifyNeo4jCapture(nil, "Nothing to do.", false, captureEndpoints) + if verdict.Complete { + t.Fatalf("verdict = %+v, want incomplete", verdict) + } + for _, want := range []string{"reported success but left no artifact", "Nothing to do."} { + if !strings.Contains(verdict.Detail, want) { + t.Errorf("Detail = %q, want it to say %q", verdict.Detail, want) + } + } + }) + + t.Run("an artifact plus a non-zero exit is incomplete, and says why the status is ambiguous", func(t *testing.T) { + verdict := classifyNeo4jCapture(failed, "", true, captureEndpoints) + if verdict.Complete { + t.Fatalf("verdict = %+v, want incomplete", verdict) + } + if !strings.Contains(verdict.Detail, "no exit status distinguishes the two") { + t.Errorf("Detail = %q, want it to name the conflation rather than assert a failure", verdict.Detail) + } + }) + + t.Run("an artifact plus a named unreachable endpoint quotes the line", func(t *testing.T) { + verdict := classifyNeo4jCapture(failed, "Server db-b.example.net was uncontactable", true, captureEndpoints) + if verdict.Complete { + t.Fatalf("verdict = %+v, want incomplete", verdict) + } + if !strings.Contains(verdict.Detail, "db-b.example.net") { + t.Errorf("Detail = %q, want it to quote the endpoint the command named", verdict.Detail) + } + }) + + t.Run("no artifact and a non-zero exit is incomplete", func(t *testing.T) { + verdict := classifyNeo4jCapture(failed, "", false, captureEndpoints) + if verdict.Complete || verdict.ArtifactProduced { + t.Errorf("verdict = %+v, want incomplete with no artifact", verdict) + } + }) +} + +// TestUncontactableEvidenceReadsTheErrorToo is a consequence of bounded +// execution returning stdout alone. neo4j-admin writes its diagnostics to +// stderr, which withStderr folds into the error — so evidence read from stdout +// only, as the reverted version did back when the streams were merged, looks for +// the line in the one place a failing command does not write it. +func TestUncontactableEvidenceReadsTheErrorToo(t *testing.T) { + targets := []HostPort{{Host: "db-a", Port: 6362}, {Host: "db-b.example.net", Port: 6362}} + runErr := fmt.Errorf("exit status 1: %s", "Failed to connect to db-b.example.net:6362") + + if got := uncontactableEvidence("", runErr, targets); !strings.Contains(got, "db-b.example.net") { + t.Errorf("uncontactableEvidence() = %q, want the line from the error's stderr", got) + } + if got := uncontactableEvidence("Server db-a was unreachable", nil, targets); got != "Server db-a was unreachable" { + t.Errorf("uncontactableEvidence() = %q, want the stdout line", got) + } + if got := uncontactableEvidence("Backup complete.", nil, targets); got != "" { + t.Errorf("uncontactableEvidence() = %q, want no evidence", got) + } +} + +// TestUncontactableEvidenceIsAttributedToAnEndpoint is T115. The error chain +// carries kubectl's own stderr, and kubectl words a control-plane outage +// exactly the way a database that will not answer is worded — so an API server +// that stopped answering was reported to the operator as a named Neo4j endpoint +// the capture could not reach, pointing the diagnosis at the wrong system. +func TestUncontactableEvidenceIsAttributedToAnEndpoint(t *testing.T) { + targets := []HostPort{{Host: "db-a", Port: 6362}, {Host: "db-b", Port: 6362}} + + cases := []struct { + name string + output string + runErr error + want string + wantErr string + }{ + { + name: "an API-server outage is not evidence about the database", + runErr: errors.New("failed to exec in the transient database workload: Unable to connect to the server: dial tcp 10.0.0.1:6443: connect: connection refused"), + want: "", + }, + { + name: "a kubectl upgrade failure is not evidence about the database", + runErr: errors.New("error: unable to upgrade connection: pod does not exist"), + want: "", + }, + { + name: "the command's own line naming a supplied endpoint is evidence", + runErr: errors.New("exit status 1: Server db-b:7687 was uncontactable during the backup"), + want: "exit status 1: Server db-b:7687 was uncontactable during the backup", + }, + { + name: "an endpoint named on stdout is evidence", + output: "Backup failed.\nunable to connect to db-a:6362", + want: "unable to connect to db-a:6362", + }, + { + name: "a line naming an endpoint with no failure wording is not evidence", + output: "Backup of db-a:6362 completed in 4m12s", + want: "", + }, + { + // Both are present in one chain, and the one that says something + // about this database is the one the operator is given. + name: "the endpoint's line is preferred over the transport's", + runErr: errors.New("Unable to connect to the server: connection refused: Server db-a was uncontactable"), + want: "Unable to connect to the server: connection refused: Server db-a was uncontactable", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := uncontactableEvidence(tc.output, tc.runErr, targets); got != tc.want { + t.Errorf("uncontactableEvidence() = %q, want %q", got, tc.want) + } + }) + } + + // And the verdict that reads it: an API-server outage must not be reported + // as an endpoint the capture could not reach. The run still fails — the + // exit status and the missing artefact decide that — it is the diagnosis + // that must not name the wrong system. + t.Run("the verdict does not blame an endpoint for a control-plane outage", func(t *testing.T) { + outage := errors.New("Unable to connect to the server: dial tcp 10.0.0.1:6443: connect: connection refused") + + verdict := classifyNeo4jCapture(outage, "", true, targets) + if verdict.Complete { + t.Fatalf("verdict = %+v, want incomplete", verdict) + } + if strings.Contains(verdict.Detail, "could not reach") { + t.Errorf("Detail = %q, want it not to report an endpoint the capture could not reach", verdict.Detail) + } + if !strings.Contains(verdict.Detail, "no exit status distinguishes the two") { + t.Errorf("Detail = %q, want the ambiguous-status verdict", verdict.Detail) + } + }) +} + +// --------------------------------------------------------------------------- +// T037 / T038: reading the probe's answers +// --------------------------------------------------------------------------- + +// TestParseCypherRowLocatesItsHeader is the carried-over T085 defect and the +// half of it T116 found still open. The reverted parser skipped index 0; its +// replacement skipped the first *non-empty* line, which handles the blank line +// and gets the notice case — the one its own comment claimed — exactly wrong: +// the notice is skipped as the header and the header is returned as the value. +// A header parses far enough to be wrong rather than to fail, so a version +// reads as "version" and a store size as "attributes.Value.value". +func TestParseCypherRowLocatesItsHeader(t *testing.T) { + cases := []struct { + name string + output string + column string + want string + }{ + {"header then value", "version\n\"2025.10.1\"\n", neo4jVersionColumn, "2025.10.1"}, + {"a blank line before the header", "\nversion\n\"2025.10.1\"\n", neo4jVersionColumn, "2025.10.1"}, + {"several blank lines before the header", "\n\n\nversion\n2025.10.1\n", neo4jVersionColumn, "2025.10.1"}, + {"blank lines between header and value", "version\n\n2025.10.1\n", neo4jVersionColumn, "2025.10.1"}, + {"a header with no rows", "version\n", neo4jVersionColumn, ""}, + {"nothing at all", "", neo4jVersionColumn, ""}, + { + // The T116 case. Skipping one non-empty line returns "version". + name: "a notice before the header", + output: "This query has been deprecated and will be removed.\nversion\n\"2025.10.1\"\n", + column: neo4jVersionColumn, + want: "2025.10.1", + }, + { + name: "several notices and a blank line before the header", + output: "WARNING: routing table refreshed\n\nNotice: server is in maintenance\nversion\n2025.10.1\n", + column: neo4jVersionColumn, + want: "2025.10.1", + }, + { + // The store size is returned under the expression text, and it is + // the value the sizing arithmetic reads: the header returned as + // data is what made a store size parse as a column name. + name: "a notice before a store size", + output: "Notice: metric names change in a future release\nattributes.Value.value\n1073741824\n", + column: neo4jStoreSizeColumn, + want: "1073741824", + }, + { + // No header for what was asked, so no answer. Every caller turns + // that into a failure naming what it asked for, which is the safe + // direction — the alternative is a value the run made up. + name: "notices and nothing else", + output: "Notice: something happened\nNotice: something else happened\n", + column: neo4jVersionColumn, + want: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := parseCypherScalar(tc.output, tc.column); got != tc.want { + t.Errorf("parseCypherScalar(%q, %q) = %q, want %q", tc.output, tc.column, got, tc.want) + } + }) + } + + t.Run("both columns of the components row", func(t *testing.T) { + got := parseCypherRow("\nversion, edition\n\"2025.10.1\", \"enterprise\"\n", neo4jVersionColumn, neo4jEditionColumn) + if got[neo4jVersionColumn] != "2025.10.1" || got[neo4jEditionColumn] != "enterprise" { + t.Errorf("parseCypherRow() = %v, want the version and the edition", got) + } + }) + + t.Run("the components row read by name, not by position", func(t *testing.T) { + // The edition read as the version would be compared against the + // utility's version by FR-006 and matched against "enterprise" by the + // FR-008 gate. Both questions get the wrong field from a positional + // read of a server that returned the columns the other way round. + got := parseCypherRow("edition, version\n\"enterprise\", \"2025.10.1\"\n", neo4jVersionColumn, neo4jEditionColumn) + if got[neo4jVersionColumn] != "2025.10.1" || got[neo4jEditionColumn] != "enterprise" { + t.Errorf("parseCypherRow() = %v, want each column read under its own name", got) + } + }) + + t.Run("a notice before the components header", func(t *testing.T) { + got := parseCypherRow("Notice: deprecated procedure\nversion, edition\n\"2025.10.1\", \"enterprise\"\n", neo4jVersionColumn, neo4jEditionColumn) + if got[neo4jVersionColumn] != "2025.10.1" || got[neo4jEditionColumn] != "enterprise" { + t.Errorf("parseCypherRow() = %v, want the row after the header, not the header", got) + } + }) + + // The statement and its parser must ask for the same column names: a + // renamed alias in one and not the other finds no header, and "the server + // answered nothing" is a wrong answer that reads like a server problem. + t.Run("the statements ask for the columns the parsers look for", func(t *testing.T) { + if !strings.Contains(neo4jComponentsStatement, "AS "+neo4jVersionColumn) { + t.Errorf("neo4jComponentsStatement = %q, want it to alias the version as %q", neo4jComponentsStatement, neo4jVersionColumn) + } + if !strings.Contains(neo4jStoreSizeStatement("neo4j"), "RETURN "+neo4jStoreSizeColumn) { + t.Errorf("neo4jStoreSizeStatement() = %q, want it to return %q", neo4jStoreSizeStatement("neo4j"), neo4jStoreSizeColumn) + } + }) +} + +func TestParseStoreSizeValue(t *testing.T) { + t.Run("a byte count", func(t *testing.T) { + size, err := parseStoreSizeValue("1073741824") + if err != nil || size != 1073741824 { + t.Errorf("parseStoreSizeValue() = %d, %v, want 1073741824", size, err) + } + }) + + // FR-022's closing clause: a size that cannot be determined must reach + // resolveScratchSize as zero so the run fails naming the override, rather + // than as a guess. + for _, text := range []string{"", " ", "null", "NULL", "not-a-number", "0", "-1"} { + t.Run(fmt.Sprintf("%q is not a size", text), func(t *testing.T) { + if _, err := parseStoreSizeValue(text); err == nil { + t.Errorf("parseStoreSizeValue(%q) = nil error, want the size reported as undetermined", text) + } + }) + } +} + +func TestExtractVersionToken(t *testing.T) { + cases := map[string]string{ + "pg_dump (PostgreSQL) 18.1": "18.1", + "neo4j-admin 2025.10.1": "2025.10.1", + "5.26.1": "5.26.1", + "18.1 (Debian 18.1-1.pgdg120+1)": "18.1", + "\nneo4j-admin, version 2025.1.0": "2025.1.0", + } + + for output, want := range cases { + t.Run(output, func(t *testing.T) { + got, err := extractVersionToken(output) + if err != nil || got != want { + t.Errorf("extractVersionToken(%q) = %q, %v, want %q", output, got, err, want) + } + }) + } + + if _, err := extractVersionToken("command not found"); err == nil { + t.Error("extractVersionToken() = nil error for output with no version, want a failure") + } +} + +// TestParseNeo4jMemberRoles reads roles from the header rather than by position, +// so a server returning the columns in another order cannot make every member's +// role its address. +func TestParseNeo4jMemberRoles(t *testing.T) { + t.Run("the documented column order", func(t *testing.T) { + members := parseNeo4jMemberRoles("address, role, writer\n\"db-a:7687\", \"primary\", true\n\"db-b:7687\", \"secondary\", false\n") + if len(members) != 2 { + t.Fatalf("members = %+v, want two", members) + } + if members[0].Address != "db-a:7687" || members[0].Role != "primary" || !members[0].Writer { + t.Errorf("members[0] = %+v, want the primary writer", members[0]) + } + if members[1].Writer { + t.Errorf("members[1] = %+v, want writer false", members[1]) + } + }) + + t.Run("a server that returns the columns in another order", func(t *testing.T) { + members := parseNeo4jMemberRoles("role, writer, address\n\"primary\", true, \"db-a:7687\"\n") + if len(members) != 1 || members[0].Address != "db-a:7687" || members[0].Role != "primary" { + t.Errorf("members = %+v, want the columns read from the header", members) + } + }) + + t.Run("a role the server did not report becomes a value, not an absence", func(t *testing.T) { + members := parseNeo4jMemberRoles("address, role, writer\n\"db-a:7687\", null, false\n") + if len(members) != 1 || members[0].Role != roleUnknown { + t.Errorf("members = %+v, want the role recorded as %q", members, roleUnknown) + } + }) + + // The header side of the case question, which this parser has always got + // right: the header is normalised as it is indexed. The lookup side — the + // spelling the *caller* asks with — is the one that was fragile here, and + // it is pinned on the shared helper instead, because no member column is + // spelled with a capital today and a test of this parser therefore cannot + // reach it. See TestCypherColumnLookupIsIndifferentToCase. + t.Run("a server that spells a column with a capital", func(t *testing.T) { + members := parseNeo4jMemberRoles("Address, Role, Writer\n\"db-a:7687\", \"primary\", true\n") + if len(members) != 1 || members[0].Address != "db-a:7687" || members[0].Role != "primary" || !members[0].Writer { + t.Errorf("members = %+v, want the columns matched regardless of case: a role that silently vanishes is worse than a run that fails (FR-005)", members) + } + }) + + t.Run("output with no rows", func(t *testing.T) { + if members := parseNeo4jMemberRoles("address, role, writer\n"); members != nil { + t.Errorf("members = %+v, want none", members) + } + }) + + // The sibling of the T116 defect: taking the first non-empty line as the + // header put a notice there, found neither column in it, and reported that + // a cluster hosts no members — which the caller turns into a failed run for + // a server that answered perfectly well. + t.Run("a notice before the header", func(t *testing.T) { + members := parseNeo4jMemberRoles("Notice: SHOW DATABASE output changes in a future release\naddress, role, writer\n\"db-a:7687\", \"primary\", true\n") + if len(members) != 1 || members[0].Address != "db-a:7687" || members[0].Role != "primary" { + t.Errorf("members = %+v, want the row after the header the notice preceded", members) + } + }) +} + +// TestCaptureRolesRecordsEveryEndpointAndClaimsNoServer is FR-005. Every +// endpoint supplied appears, matched on the host because the supplied Neo4j +// endpoint carries the backup port while the server reports its client address; +// and no key or value says which endpoint served the capture, because nothing +// observed it. +func TestCaptureRolesRecordsEveryEndpointAndClaimsNoServer(t *testing.T) { + targets := []HostPort{{Host: "db-a", Port: 6362}, {Host: "db-b", Port: 6362}, {Host: "db-c", Port: 6362}} + members := []observedMember{ + {Address: "db-a:7687", Role: "primary", Writer: true}, + {Address: "db-b:7687", Role: "secondary"}, + } + + roles := captureRoles(targets, members) + if len(roles) != 3 { + t.Fatalf("roles = %v, want one entry per endpoint supplied", roles) + } + if roles["db-a:6362"] != "primary" || roles["db-b:6362"] != "secondary" { + t.Errorf("roles = %v, want the observed roles matched on host", roles) + } + if roles["db-c:6362"] != roleUnknown { + t.Errorf("roles[db-c:6362] = %q, want %q as a value rather than an absent key", roles["db-c:6362"], roleUnknown) + } + + // Nothing in the rendering asserts a server. The line an operator reads says + // so explicitly, and this pins that no role value is a claim of one. + rendered := describeCaptureRoles(targets, roles) + for _, forbidden := range []string{"served", "captured from", "source member"} { + if strings.Contains(rendered, forbidden) { + t.Errorf("describeCaptureRoles() = %q, want no claim about which member served the capture", rendered) + } + } + if rendered != "db-a:6362=primary,db-b:6362=secondary,db-c:6362=unknown" { + t.Errorf("describeCaptureRoles() = %q, want the endpoints in try order", rendered) + } +} + +// TestCaptureRolesWithTwoInstancesOnOneHost is T114. Keying members by host +// alone let two instances at one address collapse into whichever the server +// listed last, so a supplied endpoint could be recorded as `secondary` on a run +// that had read a primary there — the field FR-005 says bears on the recovery +// point, wrong in the artefact's own provenance. +func TestCaptureRolesWithTwoInstancesOnOneHost(t *testing.T) { + t.Run("instances in different roles at one host record no role", func(t *testing.T) { + targets := []HostPort{{Host: "db-a", Port: 6362}} + members := []observedMember{ + {Address: "db-a:7687", Role: "primary", Writer: true}, + {Address: "db-a:7688", Role: "secondary"}, + } + + if roles := captureRoles(targets, members); roles["db-a:6362"] != roleUnknown { + t.Errorf("roles = %v, want %q: the run cannot tell which instance at db-a it named", roles, roleUnknown) + } + }) + + t.Run("the answer does not depend on the order the server listed them", func(t *testing.T) { + // A role the server did not report is carried as roleUnknown, which is + // also the "nothing chosen yet" value a naive fold would start from — + // so a null before a primary recorded `primary` and the same pair + // reversed recorded `unknown`. + targets := []HostPort{{Host: "db-a", Port: 6362}} + forwards := captureRoles(targets, []observedMember{ + {Address: "db-a:7687", Role: roleUnknown}, + {Address: "db-a:7688", Role: "primary"}, + }) + backwards := captureRoles(targets, []observedMember{ + {Address: "db-a:7688", Role: "primary"}, + {Address: "db-a:7687", Role: roleUnknown}, + }) + + if forwards["db-a:6362"] != backwards["db-a:6362"] { + t.Errorf("roles = %v and %v for the same two instances, want one answer whichever order they were listed in", forwards, backwards) + } + if forwards["db-a:6362"] != roleUnknown { + t.Errorf("roles = %v, want %q where the two instances disagree", forwards, roleUnknown) + } + }) + + t.Run("instances that agree still record the role", func(t *testing.T) { + // Nothing is lost by collapsing them here: whichever of the two the + // endpoint was, the role is the same. + targets := []HostPort{{Host: "db-a", Port: 6362}} + members := []observedMember{ + {Address: "db-a:7687", Role: "secondary"}, + {Address: "db-a:7688", Role: "secondary"}, + } + + if roles := captureRoles(targets, members); roles["db-a:6362"] != "secondary" { + t.Errorf("roles = %v, want secondary: both instances at db-a are in that role", roles) + } + }) + + t.Run("a whole-address match beats the host fallback", func(t *testing.T) { + // Where the supplied endpoint is the address the server reports — a + // PostgreSQL endpoint, or an operator-supplied Neo4j client address — + // the run knows which instance it named and records that one's role. + targets := []HostPort{{Host: "db-a", Port: 7688}} + members := []observedMember{ + {Address: "db-a:7687", Role: "primary", Writer: true}, + {Address: "db-a:7688", Role: "secondary"}, + } + + if roles := captureRoles(targets, members); roles["db-a:7688"] != "secondary" { + t.Errorf("roles = %v, want the role of the instance at the address supplied", roles) + } + }) +} + +// --------------------------------------------------------------------------- +// T039: the PostgreSQL dump goes to the member that answered +// --------------------------------------------------------------------------- + +func TestExternalPostgresDumpRequestUsesTheMemberThatAnswered(t *testing.T) { + capture := &externalCapture{externalSource: &externalSource{ + Endpoint: &DatabaseEndpoint{ + Service: serviceTaskManagerDB, + Database: "prefect", + Hosts: []HostPort{{Host: "pg-a", Port: 5432}, {Host: "pg-b", Port: 5432}}, + }, + // The probe walked past pg-a and got its answer from pg-b. Dumping from + // Hosts[0] would point the dump at the member just watched to fail. + Facts: probedFacts{Answered: HostPort{Host: "pg-b", Port: 5432}}, + }} + + got := postgresDumpCommand(externalPostgresDumpRequest(capture, externalPostgresDumpFile, false)) + want := []string{"pg_dump", "-Fc", "-h", "pg-b", "-p", "5432", "-d", "prefect", "-f", externalPostgresDumpFile} + if strings.Join(got, "\x00") != strings.Join(want, "\x00") { + t.Errorf("postgresDumpCommand() = %q\nwant %q", got, want) + } +} + +// TestInternalPostgresDumpArgvIsUnchanged is the FR-015 half: the in-container +// dump still connects to localhost with -U and no -p, exactly as before. +func TestInternalPostgresDumpArgvIsUnchanged(t *testing.T) { + cfg := &Configuration{PostgresUsername: "postgres", PostgresDatabase: "prefect"} + + cases := []struct { + name string + req postgresDumpRequest + want []string + }{ + { + name: "the streaming dump", + req: internalPostgresDumpRequest(cfg, "", true), + want: []string{"pg_dump", "-Fc", "-Z0", "-h", "localhost", "-U", "postgres", "-d", "prefect"}, + }, + { + name: "the file dump", + req: internalPostgresDumpRequest(cfg, "/tmp/infrahubops_prefect.dump", false), + want: []string{"pg_dump", "-Fc", "-h", "localhost", "-U", "postgres", "-d", "prefect", "-f", "/tmp/infrahubops_prefect.dump"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := postgresDumpCommand(tc.req) + if strings.Join(got, "\x00") != strings.Join(tc.want, "\x00") { + t.Errorf("postgresDumpCommand() = %q\nwant %q", got, tc.want) + } + }) + } +} + +// --------------------------------------------------------------------------- +// T040 / FR-014: no credential on any command line +// --------------------------------------------------------------------------- + +// TestNoCredentialReachesAnExternalArgv builds every command the external path +// constructs, with credentials chosen to be unmistakable, and asserts none of +// them appears anywhere in the argv or in the exec environment. +// +// The exec environment is included because it is not a separate channel: +// prepareCommand turns ExecOptions.Env into an `env KEY=VALUE …` prefix on the +// command line, so a password placed there is a password on the command line — +// which is exactly how the in-container path passes PGPASSWORD today, and +// exactly what may not follow the dump to a server outside the deployment. +func TestNoCredentialReachesAnExternalArgv(t *testing.T) { + const ( + neo4jPassword = "s3cr3t-neo4j-passphrase" + neo4jUser = "neo4j-service-account" + postgresPassword = "s3cr3t-postgres-passphrase" + postgresUser = "postgres-service-account" + ) + + cfg := &Configuration{ + Neo4jUsername: neo4jUser, + Neo4jPassword: neo4jPassword, + Neo4jDatabase: "neo4j", + PostgresUsername: postgresUser, + PostgresPassword: postgresPassword, + PostgresDatabase: "prefect", + } + + neo4jEndpoint := &DatabaseEndpoint{ + Service: serviceNeo4j, Database: cfg.Neo4jDatabase, BackupPort: 6362, + Hosts: []HostPort{{Host: "db-a", Port: 7687}}, + } + postgresEndpoint := &DatabaseEndpoint{ + Service: serviceTaskManagerDB, Database: cfg.PostgresDatabase, + Hosts: []HostPort{{Host: "pg-a", Port: 5432}}, + } + + tls := resolveExternalTLS(serviceNeo4j, cfg.ExternalDB, ExternalDBTLS{Enabled: true}) + pgTLS := resolveExternalTLS(serviceTaskManagerDB, cfg.ExternalDB, ExternalDBTLS{}) + + neo4jCapture := &externalCapture{externalSource: &externalSource{Endpoint: neo4jEndpoint, TLS: tls}, Pod: "infrahub-transient-capture-abcd1234"} + postgresCapture := &externalCapture{externalSource: &externalSource{ + Endpoint: postgresEndpoint, TLS: pgTLS, + Facts: probedFacts{Answered: HostPort{Host: "pg-a", Port: 5432}}, + }, Pod: "infrahub-transient-capture-abcd1234"} + + uri := neo4jEndpoint.boltURIs(tls)[0] + + storeSize, err := neo4jStoreSizeCommand(uri, cfg.Neo4jDatabase) + if err != nil { + t.Fatalf("neo4jStoreSizeCommand failed: %v", err) + } + memberRoles, err := neo4jMemberRolesCommand(uri, cfg.Neo4jDatabase) + if err != nil { + t.Fatalf("neo4jMemberRolesCommand failed: %v", err) + } + + commands := map[string][]string{ + "the Neo4j capture": neo4jCaptureCommand(externalNeo4jCaptureRequest(neo4jCapture, "all", true)), + "the Neo4j components probe": neo4jComponentsCommand(uri), + "the Neo4j utility version": neo4jUtilityVersionCommand(), + "the Neo4j store size": storeSize, + "the Neo4j member roles": memberRoles, + "the PostgreSQL dump": postgresDumpCommand(externalPostgresDumpRequest(postgresCapture, externalPostgresDumpFile, false)), + "the PostgreSQL stream": postgresDumpCommand(externalPostgresDumpRequest(postgresCapture, "", true)), + "the PostgreSQL version": postgresServerVersionCommand(postgresEndpoint.Hosts[0], cfg.PostgresDatabase), + "the PostgreSQL size": postgresStoreSizeCommand(postgresEndpoint.Hosts[0], cfg.PostgresDatabase), + "the PostgreSQL utility": postgresUtilityVersionCommand(), + } + + secrets := []string{neo4jPassword, postgresPassword, neo4jUser, postgresUser} + + for name, command := range commands { + t.Run(name, func(t *testing.T) { + joined := strings.Join(command, " ") + for _, secret := range secrets { + if strings.Contains(joined, secret) { + t.Errorf("%s carries a credential on its command line: %q", name, joined) + } + } + }) + } + + // The exec environment is the other half of the same command line. Only the + // TLS mode belongs there; PGUSER and PGPASSWORD come from the secret the + // transient workload owns. + t.Run("the exec environment carries the TLS settings and nothing else", func(t *testing.T) { + opts := postgresCapture.execOptions(postgresCapture.ClientEnv) + if opts.Pod != postgresCapture.Pod { + t.Errorf("opts.Pod = %q, want the capture's workload named rather than the service resolved", opts.Pod) + } + for key := range opts.Env { + if key != "PGSSLMODE" && key != "PGSSLROOTCERT" { + t.Errorf("opts.Env carries %s, want the TLS mode and its trust anchor alone", key) + } + } + for key, value := range opts.Env { + for _, secret := range secrets { + if strings.Contains(value, secret) { + t.Errorf("opts.Env[%s] carries a credential", key) + } + } + } + }) + + // The credentials do reach the workload — through the secret it owns, which + // is what makes their absence above a relocation rather than a breakage + // (research R7). + t.Run("the credentials reach the workload through its secret", func(t *testing.T) { + for service, want := range map[string][]string{ + serviceNeo4j: {"NEO4J_USERNAME", "NEO4J_PASSWORD"}, + serviceTaskManagerDB: {"PGUSER", "PGPASSWORD"}, + } { + credentials, err := transientCredentials(cfg, service) + if err != nil { + t.Fatalf("transientCredentials(%s) failed: %v", service, err) + } + for _, key := range want { + if credentials[key] == "" { + t.Errorf("transientCredentials(%s) has no %s", service, key) + } + } + } + }) +} + +// --------------------------------------------------------------------------- +// T041 / FR-029: certificate verification +// --------------------------------------------------------------------------- + +func TestResolveExternalTLS(t *testing.T) { + t.Run("verification is the default for an encrypted channel", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{}, ExternalDBTLS{Enabled: true}) + if !decision.verify() || decision.OptedOut { + t.Errorf("decision = %+v, want verification on and no opt-out", decision) + } + if got := decision.boltScheme("neo4j"); got != "neo4j+s" { + t.Errorf("boltScheme() = %q, want the verifying scheme", got) + } + }) + + // The requirement in one assertion: the deployment's own insecure setting is + // about Infrahub's connections, and this tool sends credentials of its own. + t.Run("a discovered insecure setting does not disable verification", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{}, ExternalDBTLS{Enabled: true, Insecure: true}) + if !decision.verify() { + t.Errorf("decision = %+v, want verification still on: the deployment's setting was not made about this tool", decision) + } + if decision.OptedOut { + t.Errorf("decision = %+v, want the discovered setting kept distinct from the operator's opt-out", decision) + } + if got := decision.boltScheme("neo4j"); got != "neo4j+s" { + t.Errorf("boltScheme() = %q, want the verifying scheme", got) + } + }) + + t.Run("only the operator's own flag disables it", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{InsecureTLS: true}, ExternalDBTLS{Enabled: true}) + if decision.verify() || !decision.OptedOut { + t.Errorf("decision = %+v, want the opt-out honoured", decision) + } + if got := decision.boltScheme("neo4j"); got != "neo4j+ssc" { + t.Errorf("boltScheme() = %q, want the self-signed-accepting scheme", got) + } + }) + + t.Run("PostgreSQL negotiates TLS regardless, and the mode carries the decision", func(t *testing.T) { + verifying := resolveExternalTLS(serviceTaskManagerDB, ExternalDBConfig{}, ExternalDBTLS{}) + if !verifying.Encrypt || !verifying.verify() { + t.Errorf("decision = %+v, want an encrypted, verified channel", verifying) + } + if got := verifying.postgresSSLMode(); got != "verify-full" { + t.Errorf("postgresSSLMode() = %q, want verify-full", got) + } + + optedOut := resolveExternalTLS(serviceTaskManagerDB, ExternalDBConfig{InsecureTLS: true}, ExternalDBTLS{Insecure: true}) + if got := optedOut.postgresSSLMode(); got != "require" { + t.Errorf("postgresSSLMode() = %q, want require", got) + } + }) + + // T100: a deployment that leaves INFRAHUB_DB_TLS_ENABLED at Infrahub's own + // default exports nothing, so "absent" is the common observation — and it + // used to be read as licence to send this tool's database password out of + // the cluster in clear, with no flag able to prevent it. + t.Run("an unexported TLS setting does not turn encryption off", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{}, ExternalDBTLS{}) + if !decision.Encrypt || !decision.verify() { + t.Errorf("decision = %+v, want an encrypted, verified channel by default", decision) + } + if got := decision.boltScheme("bolt"); got != "bolt+s" { + t.Errorf("boltScheme() = %q, want the verifying scheme", got) + } + }) + + // The escape from the default above, and the only one: the deployment says + // the server does not speak TLS *and* the operator says to accept that. + t.Run("an unencrypted channel takes the operator's opt-out as well", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{InsecureTLS: true}, ExternalDBTLS{Enabled: false}) + if decision.Encrypt || decision.verify() { + t.Errorf("decision = %+v, want no encryption and no verification claim", decision) + } + if !decision.OptedOut { + t.Errorf("decision = %+v, want the opt-out recorded", decision) + } + if got := decision.boltScheme("bolt"); got != "bolt" { + t.Errorf("boltScheme() = %q, want the bare scheme", got) + } + }) + + // The opt-out does not reach past what the deployment describes: where the + // server does speak TLS, opting out gives up verification and keeps the + // encryption. + t.Run("the opt-out keeps an encrypted channel the deployment declares", func(t *testing.T) { + decision := resolveExternalTLS(serviceNeo4j, ExternalDBConfig{InsecureTLS: true}, ExternalDBTLS{Enabled: true}) + if !decision.Encrypt || decision.verify() { + t.Errorf("decision = %+v, want encryption kept and verification given up", decision) + } + if got := decision.boltScheme("bolt"); got != "bolt+ssc" { + t.Errorf("boltScheme() = %q, want the self-signed-accepting scheme", got) + } + }) +} + +// TestSafeDatabaseIdentifier keeps a discovered name from closing a quote and +// continuing a statement. +func TestSafeDatabaseIdentifier(t *testing.T) { + for _, name := range []string{"neo4j", "infrahub-main", "infrahub_main", "db.one", "n4"} { + if _, err := safeDatabaseIdentifier(name); err != nil { + t.Errorf("safeDatabaseIdentifier(%q) = %v, want it accepted", name, err) + } + } + for _, name := range []string{"", " ", `neo4j" YIELD x RETURN "`, "neo4j;DROP", "neo4j one", "neo4j`"} { + if _, err := safeDatabaseIdentifier(name); err == nil { + t.Errorf("safeDatabaseIdentifier(%q) = nil, want it refused", name) + } + } +} + +// --------------------------------------------------------------------------- +// The preparation sequence (FR-006, FR-008, FR-022, FR-011) +// --------------------------------------------------------------------------- + +// newExternalOps is a run whose Neo4j database lives outside the deployment at +// the supplied address. +func newExternalOps(t *testing.T, address string) (*InfrahubOps, *gatedBackend) { + t.Helper() + + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + + cfg := createRetentionConfig(t.TempDir(), RetentionConfig{}) + cfg.Neo4jUsername = "neo4j" + cfg.Neo4jPassword = "password" + cfg.Neo4jDatabase = "neo4j" + cfg.ExternalDB = ExternalDBConfig{Neo4jAddress: address, Neo4jBackupPort: defaultNeo4jBackupPort} + + return &InfrahubOps{config: cfg, backend: backend, executor: NewCommandExecutor()}, backend +} + +// externalCaptureTarget is the target the gate hands prepareExternalSourceWith: +// a database this deployment does not host, about to be captured. +// +// It comes out of databaseTargetWith rather than being written as a literal, +// because a literal is no longer a target the preparation accepts: the gate's +// verdict is carried by an unexported field only that constructor sets, so a +// hand-built value is refused (see databaseTarget.requireGated). A test that +// built its own was exactly the shape the witness exists to stop. +func externalCaptureTarget(t *testing.T, service string) databaseTarget { + t.Helper() + + target, err := databaseTargetWith(deploymentQueries{ + locate: func(string) (EndpointLocation, error) { return EndpointLocationExternal, nil }, + }, databaseCapture, service, ExternalRestoreAuth{}) + if err != nil { + t.Fatalf("databaseTargetWith(capture, %s) = %v, want a target", service, err) + } + + return target +} + +// recordingCaptureOps is externalCaptureOps with every step recorded, so the +// order the sequence performs them in is assertable without a cluster. +type recordingCaptureOps struct { + steps []string + facts probedFacts + err error +} + +func (r *recordingCaptureOps) ops() externalCaptureOps { + return externalCaptureOps{ + startProbe: func(_ string, members int) (string, func(), error) { + r.steps = append(r.steps, fmt.Sprintf("start-probe(members=%d)", members)) + + return "probe-pod", func() { r.steps = append(r.steps, "release-probe") }, nil + }, + startCapture: func(_ string, storeSizeBytes int64) (string, func(), error) { + r.steps = append(r.steps, fmt.Sprintf("start-capture(size=%d)", storeSizeBytes)) + + return "capture-pod", func() { r.steps = append(r.steps, "release-capture") }, nil + }, + probe: func(pod string, _ *DatabaseEndpoint, _ externalTLSDecision) (probedFacts, error) { + r.steps = append(r.steps, "probe("+pod+")") + + return r.facts, r.err + }, + } +} + +// TestPrepareExternalSourceWithSequencesTheGate pins the order the requirements +// force: probe first because nothing else can answer, then the edition refusal +// and the version gate on what it read, and the probe given back before the run +// goes on. No capture workload is created here at all — it is built later, +// around the capture itself, sized from what the probe reported. +func TestPrepareExternalSourceWithSequencesTheGate(t *testing.T) { + t.Run("a healthy Enterprise server records what the probe read", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a,db-b") + recorder := &recordingCaptureOps{facts: probedFacts{ + ServerVersion: "2025.10.1", + UtilityVersion: "2025.10.1", + Edition: "enterprise", + StoreSizeBytes: 4 << 30, + Members: []observedMember{{Address: "db-a:7687", Role: "primary", Writer: true}}, + }} + + if err := iops.prepareExternalSourceWith(recorder.ops(), externalCaptureTarget(t, serviceNeo4j)); err != nil { + t.Fatalf("prepareExternalSourceWith() = %v, want the source prepared", err) + } + + // The probe's deadline is budgeted from the number of endpoints it may + // walk, so the member count has to reach it. + want := []string{"start-probe(members=2)", "probe(probe-pod)", "release-probe"} + if strings.Join(recorder.steps, ",") != strings.Join(want, ",") { + t.Errorf("steps = %v\nwant %v", recorder.steps, want) + } + + source := iops.externalSourceFor(serviceNeo4j) + if source == nil { + t.Fatal("externalSourceFor() = nil, want the prepared source") + } + if source.Facts.StoreSizeBytes != 4<<30 { + t.Errorf("StoreSizeBytes = %d, want the size the probe read (FR-022)", source.Facts.StoreSizeBytes) + } + if got := renderHostPorts(source.captureTargets()); got != "db-a:6362,db-b:6362" { + t.Errorf("captureTargets() = %q, want both members on the backup port in order", got) + } + if roles := source.observedRoles(); roles["db-a:6362"] != "primary" || roles["db-b:6362"] != roleUnknown { + t.Errorf("observedRoles() = %v, want the observed role and %q for the member none matched", roles, roleUnknown) + } + }) + + t.Run("a Community server is refused before the version is even compared", func(t *testing.T) { + iops, backend := newExternalOps(t, "db-a") + // A utility older than the server, so a run that reached the version gate + // would fail there instead — which is how this test tells the two apart. + recorder := &recordingCaptureOps{facts: probedFacts{ + ServerVersion: "2025.10.1", UtilityVersion: "5.26.1", Edition: "community", + }} + + err := iops.prepareExternalSourceWith(recorder.ops(), externalCaptureTarget(t, serviceNeo4j)) + if !errors.Is(err, errExternalCommunityCapture) { + t.Fatalf("err = %v, want the FR-008 refusal rather than a version failure", err) + } + if iops.externalSourceFor(serviceNeo4j) != nil { + t.Error("a source was recorded for a database this run refused to capture") + } + // Principle II: the refusal costs a probe pod and nothing else. + if !contains(recorder.steps, "release-probe") { + t.Errorf("steps = %v, want the probe given back on the refusal path (FR-011)", recorder.steps) + } + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none before the refusal", backend.stopped) + } + }) + + t.Run("a utility older than the server aborts before any data moves", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + recorder := &recordingCaptureOps{facts: probedFacts{ + ServerVersion: "2025.10.1", UtilityVersion: "5.26.1", Edition: "enterprise", + }} + + err := iops.prepareExternalSourceWith(recorder.ops(), externalCaptureTarget(t, serviceNeo4j)) + if err == nil { + t.Fatal("prepareExternalSourceWith() = nil, want the FR-006 abort") + } + if !strings.Contains(err.Error(), externalDBImageOption) { + t.Errorf("err = %v, want it to name the flag that fixes it", err) + } + if contains(recorder.steps, "start-capture(size=0)") { + t.Error("a capture workload was created for a run the version gate aborted") + } + }) + + t.Run("a probe that failed gives the workload back and reports the endpoint", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + recorder := &recordingCaptureOps{err: errors.New("connection refused")} + + err := iops.prepareExternalSourceWith(recorder.ops(), externalCaptureTarget(t, serviceNeo4j)) + if err == nil { + t.Fatal("prepareExternalSourceWith() = nil, want the probe failure reported") + } + if !strings.Contains(err.Error(), "db-a") { + t.Errorf("err = %v, want it to name the endpoint it could not read", err) + } + if !contains(recorder.steps, "release-probe") { + t.Errorf("steps = %v, want the probe given back on the failure path (FR-011)", recorder.steps) + } + }) +} + +// TestOpenExternalCaptureSizesTheWorkloadFromTheProbe is FR-022's other half: a +// pod's storage is fixed when it is created, so the capture workload is the one +// built from the store size, and it is built only when the capture runs. +func TestOpenExternalCaptureSizesTheWorkloadFromTheProbe(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + iops.recordExternalSource(serviceNeo4j, &externalSource{ + Endpoint: &DatabaseEndpoint{Service: serviceNeo4j, Database: "neo4j", BackupPort: 6362, Hosts: []HostPort{{Host: "db-a", Port: 7687}}}, + Facts: probedFacts{StoreSizeBytes: 8 << 30}, + }) + + recorder := &recordingCaptureOps{} + pod, release, err := recorder.ops().startCapture(serviceNeo4j, iops.externalSourceFor(serviceNeo4j).Facts.StoreSizeBytes) + if err != nil { + t.Fatalf("startCapture failed: %v", err) + } + release() + + if pod != "capture-pod" { + t.Errorf("pod = %q, want the capture workload named", pod) + } + if !contains(recorder.steps, fmt.Sprintf("start-capture(size=%d)", int64(8<<30))) { + t.Errorf("steps = %v, want the workload sized from the store size the probe read", recorder.steps) + } +} + +// TestExternalSourceForIsNilForAnInternalDatabase is the FR-015 branch every +// capture path takes. Nothing was prepared, so nothing external happens. +func TestExternalSourceForIsNilForAnInternalDatabase(t *testing.T) { + iops := NewInfrahubOps() + + if iops.externalSourceFor(serviceNeo4j) != nil || iops.externalSourceFor(serviceTaskManagerDB) != nil { + t.Error("externalSourceFor() = non-nil on a run that prepared nothing, want the internal path") + } +} + +// TestExternalRunIDIsMintedOncePerRun is the invariant two external databases +// broke. A per-service identifier labels one run's two workloads as two runs, +// which is what the reaper and the create-collision refusal both read to decide +// whose objects they are looking at. +func TestExternalRunIDIsMintedOncePerRun(t *testing.T) { + iops := NewInfrahubOps() + + first, err := iops.externalRunID() + if err != nil { + t.Fatalf("externalRunID() failed: %v", err) + } + second, err := iops.externalRunID() + if err != nil { + t.Fatalf("externalRunID() failed: %v", err) + } + + if first == "" { + t.Fatal("externalRunID() = empty") + } + if first != second { + t.Errorf("externalRunID() = %q then %q, want one identifier for the whole run", first, second) + } + + // A different run gets a different one, or the reaper could not tell an + // earlier run's litter from this run's live workloads. + other, err := NewInfrahubOps().externalRunID() + if err != nil { + t.Fatalf("externalRunID() failed: %v", err) + } + if other == first { + t.Errorf("two runs minted the same identifier %q", first) + } +} + +// scriptedProbeBackend answers each bounded exec from a script keyed on the +// first word of the command, recording every argv it was handed. It is what +// lets the probe walk and the CA install be driven without a cluster. +type scriptedProbeBackend struct { + unboundedBackend + + // answers maps a command's first token to the outputs it returns, in call + // order; the last one is repeated once exhausted. + answers map[string][]string + // fails maps a command's first token to an error it returns instead. + fails map[string]error + + calls [][]string + envs []map[string]string +} + +func (b *scriptedProbeBackend) ExecSeparateContext(_ context.Context, _ time.Duration, _ string, command []string, opts *ExecOptions) (string, string, error) { + b.calls = append(b.calls, command) + if opts != nil { + b.envs = append(b.envs, opts.Env) + } else { + b.envs = append(b.envs, nil) + } + + key := "" + if len(command) > 0 { + key = command[0] + } + if err, ok := b.fails[key]; ok { + return "", "", err + } + + outputs := b.answers[key] + switch { + case len(outputs) == 0: + return "", "", nil + case len(outputs) == 1: + return outputs[0], "", nil + default: + b.answers[key] = outputs[1:] + + return outputs[0], "", nil + } +} + +func (b *scriptedProbeBackend) argvFor(token string) [][]string { + matched := [][]string{} + for _, call := range b.calls { + if len(call) > 0 && call[0] == token { + matched = append(matched, call) + } + } + + return matched +} + +// errTestCommandMissing stands in for a command the workload's image does not +// carry, which is what every best-effort step on this path must degrade around. +var errTestCommandMissing = errors.New("exec: \"keytool\": executable file not found in $PATH") + +// TestFirstAnsweringEndpointWalksPastAnEmptyAnswer is F7, and F16 before the +// rebuild. The Neo4j arm validated what a member returned and walked on; the +// PostgreSQL arm took whatever the exec produced and stopped — so a member that +// connected and answered nothing ended the walk, and the FR-006 version gate +// then aborted the run with a healthy member never tried. +func TestFirstAnsweringEndpointWalksPastAnEmptyAnswer(t *testing.T) { + endpoints := []string{"first", "second", "third"} + + t.Run("a member that answers nothing does not end the walk", func(t *testing.T) { + asked := []string{} + accepted := "" + index, err := firstAnsweringEndpoint(endpoints, "the server version", "the database", + func(endpoint string) (string, error) { + asked = append(asked, endpoint) + if endpoint == "first" { + return " \n", nil + } + + return "16.2", nil + }, + func(endpoint, output string) error { + if strings.TrimSpace(output) == "" { + return fmt.Errorf("%s answered nothing when asked for its version", endpoint) + } + accepted = endpoint + + return nil + }, + ) + if err != nil { + t.Fatalf("firstAnsweringEndpoint failed: %v", err) + } + if index != 1 || accepted != "second" { + t.Errorf("index = %d (accepted %q), want the second member", index, accepted) + } + if len(asked) != 2 { + t.Errorf("asked = %v, want the walk to have continued past the empty answer", asked) + } + }) + + t.Run("every member failing reports the last failure", func(t *testing.T) { + _, err := firstAnsweringEndpoint(endpoints, "the server version", "the database", + func(endpoint string) (string, error) { return "", fmt.Errorf("dial %s: refused", endpoint) }, + func(string, string) error { return nil }, + ) + if err == nil { + t.Fatal("firstAnsweringEndpoint() = nil, want the walk reported as a failure") + } + if !strings.Contains(err.Error(), "failed to read the server version of the database") { + t.Errorf("err = %v, want it to name what was being read and of what", err) + } + if !strings.Contains(err.Error(), "dial third") { + t.Errorf("err = %v, want the last member's own failure kept", err) + } + }) +} + +// TestProbeExternalPostgresWalksPastASilentMember drives the walk through the +// PostgreSQL probe itself, which is where the regression lived: it is the arm +// that accepted whatever came back. +func TestProbeExternalPostgresWalksPastASilentMember(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{ + "pg_dump": {"pg_dump (PostgreSQL) 16.2"}, + // The first member connects and reports nothing; the second answers. + "psql": {"", "16.2", "1024"}, + }} + iops := &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + + endpoint := &DatabaseEndpoint{ + Service: serviceTaskManagerDB, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "silent.example", Port: 5432}, {Host: "healthy.example", Port: 5432}}, + Database: "prefect", + } + + facts, err := iops.probeExternalPostgres("probe-pod", endpoint, externalTLSDecision{Service: serviceTaskManagerDB, Encrypt: true}) + if err != nil { + t.Fatalf("probeExternalPostgres failed: %v", err) + } + if facts.ServerVersion != "16.2" { + t.Errorf("ServerVersion = %q, want the version the second member reported", facts.ServerVersion) + } + if facts.Answered.Host != "healthy.example" { + t.Errorf("Answered = %v, want the member that actually answered", facts.Answered) + } +} + +// TestExternalDatabaseForReadsTheLocation is F4. The capture paths branched +// on "did preparation record something?", so an entry point that never ran +// prepareDatabaseCapture left externalSources empty and silently took the +// in-deployment path against a database with no container. +// +// The accessor answers for either direction (T148): a restore's record carries +// the same probe, and the decisions that ask what the probe read are reached +// from both. +func TestExternalDatabaseForReadsTheLocation(t *testing.T) { + t.Run("an internal database is nil and is not an error", func(t *testing.T) { + iops := newGatedOps(t, newGatedBackend(), BackendTarball) + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil || source != nil { + t.Errorf("externalDatabaseFor() = %v, %v; want nil, nil for a database inside the deployment (FR-015)", source, err) + } + }) + + t.Run("a prepared external database returns what was established", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + prepared := &externalSource{Endpoint: &DatabaseEndpoint{Service: serviceNeo4j}} + iops.recordExternalSource(serviceNeo4j, prepared) + + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + t.Fatalf("externalDatabaseFor failed: %v", err) + } + if source != prepared { + t.Errorf("source = %v, want the source the gate prepared", source) + } + }) + + t.Run("a prepared external restore returns what its probe established", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + probed := &externalSource{Endpoint: &DatabaseEndpoint{Service: serviceNeo4j}} + iops.recordExternalRestore(serviceNeo4j, &externalRestore{externalSource: probed}) + + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err != nil { + t.Fatalf("externalDatabaseFor failed: %v", err) + } + if source != probed { + t.Errorf("source = %v, want the source the restore's gate probed (T148)", source) + } + }) + + t.Run("located external with nothing prepared is an error, never internal", func(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := newGatedOps(t, backend, BackendTarball) + + source, err := iops.externalDatabaseFor(serviceNeo4j) + if err == nil { + t.Fatalf("externalDatabaseFor() = %v, nil; want an error rather than the in-deployment path", source) + } + if source != nil { + t.Errorf("source = %v, want none", source) + } + if !strings.Contains(err.Error(), "nothing was prepared") { + t.Errorf("err = %v, want it to name what was not done", err) + } + }) + + t.Run("a location that could not be established is reported", func(t *testing.T) { + backend := newGatedBackend() + backend.locateErr = errors.New("Unable to connect to the server") + iops := newGatedOps(t, backend, BackendTarball) + + if _, err := iops.externalDatabaseFor(serviceNeo4j); err == nil { + t.Error("externalDatabaseFor() = nil, want a cluster that did not answer reported rather than read as internal (FR-002)") + } + }) +} + +// TestPostgresClientEnvCarriesTrustMaterial is T110. `PGSSLMODE=verify-full` +// was set with no `PGSSLROOTCERT` beside it, so libpq looked for +// ~/.postgresql/root.crt in a workload that has no such file and refused the +// connection before reaching the server — a secure default that cannot be +// used, whose only escape drops verification altogether. The deployment's own +// `sslrootcert` was read into a log string and nowhere else. +func TestPostgresClientEnvCarriesTrustMaterial(t *testing.T) { + verifying := externalTLSDecision{Service: serviceTaskManagerDB, Encrypt: true} + + t.Run("the deployment's authority is installed and named", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{ + "cat": {"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"}, + }} + iops := &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + iops.config.ExternalDB.PostgresTLS.CAFile = "/etc/ssl/pg-ca.pem" + + env := iops.postgresClientEnv("capture-pod", verifying) + + if env["PGSSLMODE"] != "verify-full" { + t.Errorf("PGSSLMODE = %q, want verify-full", env["PGSSLMODE"]) + } + if env["PGSSLROOTCERT"] != externalPostgresCAPEMPath { + t.Errorf("PGSSLROOTCERT = %q, want the PEM this run wrote into the workload", env["PGSSLROOTCERT"]) + } + + // Read from the container whose environment named the path, and written + // into the pod the client will run in. + if len(backend.argvFor("cat")) != 1 { + t.Errorf("the deployment's CA was not read: calls = %v", backend.calls) + } + if len(backend.argvFor("sh")) != 1 { + t.Errorf("the CA was not written into the workload: calls = %v", backend.calls) + } + }) + + t.Run("a deployment naming no authority still verifies", func(t *testing.T) { + backend := &scriptedProbeBackend{} + iops := &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + + env := iops.postgresClientEnv("capture-pod", verifying) + + if env["PGSSLMODE"] != "verify-full" { + t.Errorf("PGSSLMODE = %q, want verify-full", env["PGSSLMODE"]) + } + if env["PGSSLROOTCERT"] != postgresSystemTrustStore { + t.Errorf("PGSSLROOTCERT = %q, want %q: verify-full with no anchor named is a mode nothing can satisfy", + env["PGSSLROOTCERT"], postgresSystemTrustStore) + } + if len(backend.calls) != 0 { + t.Errorf("calls = %v, want none: there is nothing to install", backend.calls) + } + }) + + t.Run("an unreadable authority degrades to verifying, never to not verifying", func(t *testing.T) { + backend := &scriptedProbeBackend{fails: map[string]error{"cat": errTestCommandMissing}} + iops := &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + iops.config.ExternalDB.PostgresTLS.CAFile = "/etc/ssl/pg-ca.pem" + + env := iops.postgresClientEnv("capture-pod", verifying) + + if env["PGSSLMODE"] != "verify-full" { + t.Errorf("PGSSLMODE = %q, want verify-full: a CA this run could not read is not an instruction to stop verifying", env["PGSSLMODE"]) + } + if env["PGSSLROOTCERT"] != postgresSystemTrustStore { + t.Errorf("PGSSLROOTCERT = %q, want %q", env["PGSSLROOTCERT"], postgresSystemTrustStore) + } + }) + + t.Run("the opt-out installs nothing and names nothing", func(t *testing.T) { + backend := &scriptedProbeBackend{} + iops := &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + iops.config.ExternalDB.PostgresTLS.CAFile = "/etc/ssl/pg-ca.pem" + + env := iops.postgresClientEnv("capture-pod", externalTLSDecision{ + Service: serviceTaskManagerDB, Encrypt: true, OptedOut: true, + }) + + if env["PGSSLMODE"] != "require" { + t.Errorf("PGSSLMODE = %q, want require", env["PGSSLMODE"]) + } + if _, named := env["PGSSLROOTCERT"]; named { + t.Errorf("PGSSLROOTCERT = %q, want none: nothing is verified, so an anchor would only mislead", env["PGSSLROOTCERT"]) + } + if len(backend.calls) != 0 { + t.Errorf("calls = %v, want none: a run that verifies nothing installs nothing", backend.calls) + } + }) +} diff --git a/src/internal/app/external_credential_refs.go b/src/internal/app/external_credential_refs.go new file mode 100644 index 00000000..6eb7349d --- /dev/null +++ b/src/internal/app/external_credential_refs.go @@ -0,0 +1,449 @@ +package app + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + + "github.com/sirupsen/logrus" +) + +// A transient workload has to authenticate to the database it stands in for, +// and FR-014 keeps the credentials off every command line and out of every +// inline value in a pod spec. Until this file, the only way to satisfy both was +// for this tool to write a Secret of its own — which made `create`, `list` and +// `delete` on secrets part of what a namespace had to grant before an external +// database could be backed up at all. +// +// Most deployments already hold those credentials in a Secret: the Infrahub +// server takes INFRAHUB_DB_USERNAME and INFRAHUB_DB_PASSWORD from one, and the +// prefect-server chart can take its database user and password from one +// (externalDatabase.usernameSecretKey / passwordSecretKey render +// PREFECT_SERVER_DATABASE_USER and PREFECT_SERVER_DATABASE_PASSWORD as +// secretKeyRefs). When the credentials this run is about to use are exactly +// the ones such a Secret holds, the Job can reference the same Secret key with +// `env[].valueFrom.secretKeyRef`. The kubelet resolves that reference, not this +// tool, so no Secret is created and no secrets verb is needed. A reference is +// not the value, so FR-014 holds unchanged. +// +// Reuse rests on evidence the run already has, never on a guess: +// +// - the credential came from the deployment component's own environment, not +// from the operator's environment and not from a default — an operator who +// supplied credentials asked for those, and the Secret may hold others; +// - the component's running pod declares the variable that carried it as a +// Secret key (pods `get`, which the run already needs); +// - for PostgreSQL, whose credentials arrive inside the Prefect connection +// URL, a Secret-backed variable on the task-manager pod held exactly the +// user or the password the URL carried when the environment was read. +// +// Anything short of that — an inline value, a ConfigMap, a variable reached +// only through envFrom (which names no keys), a pod that cannot be +// read — falls back to the owned Secret, which is the behaviour every run had +// before this file existed. + +// deploymentCredentialSource records where in the deployment the credentials a +// run holds for one database were read from: the component whose environment +// supplied them, and the variables in that environment which held each value. +// +// It is recorded only when *both* the username and the password came from the +// deployment in the same read. One from the operator and one from the +// deployment is a pair the deployment never held together, so no Secret of its +// can stand in for it. +type deploymentCredentialSource struct { + component string + // username and password are variable names in the component's + // environment, in preference order. Each held the value the run now uses + // at the moment the environment was read. + username []string + password []string +} + +// transientSecretKeyRef is one key of a Secret the deployment already has. +// Optional is never carried: the transient container always states it false, +// so a key that has gone missing since the deployment's pod started holds the +// container in a visible pending state rather than starting it unauthenticated. +type transientSecretKeyRef struct { + Name string + Key string +} + +// prefectCredentialEnvVars are the variables prefect-server renders the +// database user and password into when the chart takes them from a Secret. +// They are tried before any other variable that happens to hold the same +// value, because they are the ones that mean it. +var ( + prefectUserEnvVars = []string{"PREFECT_SERVER_DATABASE_USER"} + prefectPasswordEnvVars = []string{"PREFECT_SERVER_DATABASE_PASSWORD"} +) + +// transientCredentialNames is the pair of variables the database's own client +// reads its username and password from inside the transient container. +func transientCredentialNames(service string) (username, password string, err error) { + switch service { + case serviceNeo4j: + return "NEO4J_USERNAME", "NEO4J_PASSWORD", nil + case serviceTaskManagerDB: + return "PGUSER", "PGPASSWORD", nil + default: + return "", "", errNotADatabaseService("resolve credentials for", service) + } +} + +// recordDeploymentCredentials remembers that the credentials for service came +// from the deployment, and where. A source with no candidate variable for +// either value is not recorded: nothing could be mapped from it. +func (iops *InfrahubOps) recordDeploymentCredentials(service string, source deploymentCredentialSource) { + if len(source.username) == 0 || len(source.password) == 0 { + return + } + + if iops.deploymentCredentials == nil { + iops.deploymentCredentials = map[string]deploymentCredentialSource{} + } + iops.deploymentCredentials[service] = source +} + +// envNamesHolding lists the variables in an `env` dump whose value is exactly +// value: the preferred names first, in their order, then every other match in +// name order so the choice does not depend on the order the container printed +// its environment. The Prefect connection URLs are never candidates — they hold +// the whole URL, which is never equal to a user or a password, and naming them +// here would only invite reading one as if it were. +func envNamesHolding(env map[string]string, value string, preferred []string) []string { + if value == "" { + return nil + } + + names := []string{} + for _, name := range preferred { + if env[name] == value { + names = append(names, name) + } + } + + others := []string{} + for name, held := range env { + if held != value || slices.Contains(preferred, name) || slices.Contains(prefectConnectionEnvVars, name) { + continue + } + others = append(others, name) + } + sort.Strings(others) + + return append(names, others...) +} + +// --------------------------------------------------------------------------- +// Reading what a running pod declares about its environment +// --------------------------------------------------------------------------- + +// podDefaultContainerAnnotation names the container `kubectl exec` enters when +// no -c is given. The environment the credentials were read from is that +// container's, because the read was an exec with no -c. +const podDefaultContainerAnnotation = "kubectl.kubernetes.io/default-container" + +// podEnvDeclarationsJSONPath reads where each variable of each container comes +// from, and never what it holds. +// +// It is a jsonpath rather than `-o json` on purpose. A pod spec may carry a +// credential as an inline value — the case this file exists to fall back from +// — and `-o json` would hand that value to this process and to anything that +// logs a command's output. The jsonpath asks for names, Secret and ConfigMap +// references and field paths only, so an inline value never leaves the API +// server. Its presence is still detectable: an `E` line with no reference of +// any kind is a variable whose value the spec states inline. +// +// Every line carries a leading tag because the runner trims its output, and an +// empty annotation on an untagged first line would be trimmed away and shift +// every line after it. +var podEnvDeclarationsJSONPath = `jsonpath=` + + `A{"\t"}{.metadata.annotations.` + escapeJSONPathKey(podDefaultContainerAnnotation) + `}{"\n"}` + + `{range .spec.containers[*]}` + + `C{"\t"}{.name}{"\n"}` + + `{range .env[*]}E{"\t"}{.name}{"\t"}{.valueFrom.secretKeyRef.name}{"\t"}{.valueFrom.secretKeyRef.key}{"\t"}` + + `{.valueFrom.configMapKeyRef.name}{.valueFrom.fieldRef.fieldPath}{.valueFrom.resourceFieldRef.resource}{"\n"}{end}` + + `{range .envFrom[*]}F{"\t"}{.prefix}{"\t"}{.secretRef.name}{"\t"}{.configMapRef.name}{"\n"}{end}` + + `{end}` + +// podEnvVar is one `env` entry: the Secret key it comes from, or the fact that +// it comes from something else (a ConfigMap, a field of the pod) — or, with +// neither, an inline value. +type podEnvVar struct { + name string + secretName string + secretKey string + other bool +} + +// podEnvFrom is one `envFrom` entry. +type podEnvFrom struct { + prefix string + secretName string + configMapName string +} + +type podContainerEnv struct { + name string + env []podEnvVar + envFrom []podEnvFrom +} + +type podEnvDeclarations struct { + defaultContainer string + containers []podContainerEnv +} + +// parsePodEnvDeclarations reads podEnvDeclarationsJSONPath's output. A line it +// does not recognise is skipped rather than guessed at; the worst that can do +// is make a variable look undeclared, which falls back. +func parsePodEnvDeclarations(output string) podEnvDeclarations { + declarations := podEnvDeclarations{} + + for _, line := range strings.Split(output, "\n") { + fields := strings.Split(strings.TrimRight(line, "\r"), "\t") + field := func(i int) string { + if i < len(fields) { + return strings.TrimSpace(fields[i]) + } + + return "" + } + + switch field(0) { + case "A": + declarations.defaultContainer = field(1) + case "C": + declarations.containers = append(declarations.containers, podContainerEnv{name: field(1)}) + case "E": + if len(declarations.containers) == 0 || field(1) == "" { + continue + } + current := &declarations.containers[len(declarations.containers)-1] + current.env = append(current.env, podEnvVar{ + name: field(1), + secretName: field(2), + secretKey: field(3), + other: field(4) != "", + }) + case "F": + if len(declarations.containers) == 0 { + continue + } + current := &declarations.containers[len(declarations.containers)-1] + current.envFrom = append(current.envFrom, podEnvFrom{ + prefix: field(1), + secretName: field(2), + configMapName: field(3), + }) + } + } + + return declarations +} + +// execContainer is the container an exec with no -c enters, which is the one +// whose environment the credentials were read from: the one the default-container +// annotation names, or the first. +func (d podEnvDeclarations) execContainer() (podContainerEnv, bool) { + if d.defaultContainer != "" { + for _, container := range d.containers { + if container.name == d.defaultContainer { + return container, true + } + } + } + + if len(d.containers) == 0 { + return podContainerEnv{}, false + } + + return d.containers[0], true +} + +// secretKeyFor reports which Secret key a variable in this container comes +// from, or why it cannot be shown to come from one. +// +// It follows the precedence the kubelet applies: an `env` entry overrides +// anything from `envFrom`, and of two `env` entries with one name the later +// wins. A variable reached only through `envFrom` is never attributed to a +// Secret. An `envFrom` source names no keys and the kubelet silently skips a +// key the Secret lacks, so a name that matches a source's prefix (an empty +// prefix matches every name) does not show that the Secret holds the key: the +// value may come from the image instead. A `secretKeyRef` to a missing key +// would hold the transient container pending and block the run, and checking +// that the key exists would mean reading the Secret, which is the permission +// this file exists to avoid needing. Such a variable falls back. +func (c podContainerEnv) secretKeyFor(variable string) (transientSecretKeyRef, string, bool) { + for i := len(c.env) - 1; i >= 0; i-- { + entry := c.env[i] + if entry.name != variable { + continue + } + + switch { + case entry.secretName != "" && entry.secretKey != "": + return transientSecretKeyRef{Name: entry.secretName, Key: entry.secretKey}, "", true + case entry.other: + return transientSecretKeyRef{}, fmt.Sprintf("%s comes from a ConfigMap or a field of the pod, not a Secret", variable), false + default: + return transientSecretKeyRef{}, fmt.Sprintf("%s is stated inline in the pod spec", variable), false + } + } + + sources := 0 + for _, source := range c.envFrom { + if !strings.HasPrefix(variable, source.prefix) || len(variable) == len(source.prefix) { + continue + } + if source.secretName != "" || source.configMapName != "" { + sources++ + } + } + + if sources == 0 { + return transientSecretKeyRef{}, fmt.Sprintf("the pod spec does not declare %s, so the image or its entrypoint sets it", variable), false + } + + return transientSecretKeyRef{}, fmt.Sprintf("%s could come from %d envFrom source(s), which name no keys, so whether a Secret holds it would mean reading the Secret", variable, sources), false +} + +// firstSecretKey is secretKeyFor over the candidate variables in order, the +// first that maps winning. The reason returned when none does is the first +// candidate's: it is the variable that means the credential, and the others +// merely held the same value. +func (c podContainerEnv) firstSecretKey(candidates []string) (transientSecretKeyRef, string, bool) { + firstReason := "" + for _, variable := range candidates { + ref, reason, ok := c.secretKeyFor(variable) + if ok { + return ref, "", true + } + if firstReason == "" { + firstReason = reason + } + } + + return transientSecretKeyRef{}, firstReason, false +} + +// deploymentCredentialRefs maps the transient container's credential variables +// onto the Secret keys the deployment's pod declares, or reports why it cannot. +// It is pure over what was read, which is what lets every arm of the decision +// be exercised without a cluster. +func deploymentCredentialRefs(service string, source deploymentCredentialSource, declarations podEnvDeclarations) (map[string]transientSecretKeyRef, string) { + userVar, passwordVar, err := transientCredentialNames(service) + if err != nil { + return nil, err.Error() + } + + container, ok := declarations.execContainer() + if !ok { + return nil, fmt.Sprintf("the %s pod spec lists no containers", source.component) + } + + user, reason, ok := container.firstSecretKey(source.username) + if !ok { + return nil, "the username: " + reason + } + + password, reason, ok := container.firstSecretKey(source.password) + if !ok { + return nil, "the password: " + reason + } + + return map[string]transientSecretKeyRef{userVar: user, passwordVar: password}, "" +} + +// resolveDeploymentCredentialRefs decides, once per database per run, whether +// its transient workloads reference the deployment's own Secret or carry an +// owned one, and logs which — naming Secrets and keys, never a value. +// +// readPodEnv reads podEnvDeclarationsJSONPath off the component's running pod. +// A failed read is not remembered: it says the pod could not answer, not that +// the credentials are not Secret-backed, so a later workload asks again. A +// decision is remembered either way, which is also what keeps the answer +// available after a restore or an offline capture has scaled the component to +// zero — the gate creates the first workload while the deployment is up. +func (iops *InfrahubOps) resolveDeploymentCredentialRefs(service string, readPodEnv func(component string) (string, error)) map[string]transientSecretKeyRef { + if refs, decided := iops.deploymentCredentialRefs[service]; decided { + return refs + } + + decide := func(refs map[string]transientSecretKeyRef) map[string]transientSecretKeyRef { + if iops.deploymentCredentialRefs == nil { + iops.deploymentCredentialRefs = map[string]map[string]transientSecretKeyRef{} + } + iops.deploymentCredentialRefs[service] = refs + + return refs + } + + source, fromDeployment := iops.deploymentCredentials[service] + if !fromDeployment { + logrus.Infof("The %s credentials did not both come from the deployment (they were supplied by the operator or are defaults), so the transient workload carries them in a Secret of its own", service) + + return decide(nil) + } + + output, err := readPodEnv(source.component) + if err != nil { + logrus.Infof("Could not read how the %s pod declares the %s credentials, so the transient workload carries them in a Secret of its own: %v", source.component, service, err) + + return nil + } + + refs, reason := deploymentCredentialRefs(service, source, parsePodEnvDeclarations(output)) + if refs == nil { + logrus.Infof("The %s credentials cannot be referenced from a Secret the deployment already has (%s), so the transient workload carries them in a Secret of its own", service, reason) + + return decide(nil) + } + + described := make([]string, 0, len(refs)) + for variable, ref := range refs { + described = append(described, fmt.Sprintf("%s from %s/%s", variable, ref.Name, ref.Key)) + } + sort.Strings(described) + logrus.Infof("The transient %s workloads reference the deployment's own Secret for their credentials (%s), so this run creates no Secret for them", service, strings.Join(described, ", ")) + + return decide(refs) +} + +// withDeploymentCredentialRefs points a spec at the deployment's own Secret +// when the run can show that Secret holds the credentials it would otherwise +// copy, and leaves the spec as it was otherwise. The credential values are +// dropped from a spec that references the Secret, so they cannot reach the +// manifest by any path. +func (iops *InfrahubOps) withDeploymentCredentialRefs(k *KubernetesBackend, spec transientWorkloadSpec) transientWorkloadSpec { + refs := iops.resolveDeploymentCredentialRefs(spec.Service, k.readPodEnvDeclarations) + if len(refs) == 0 { + return spec + } + + spec.CredentialRefs = refs + spec.Credentials = nil + + return spec +} + +// readPodEnvDeclarations reads where the variables of a component's running +// pod come from. It is a `get` on a pod — a verb every run on this path already +// holds — and it is bounded on the control bound, being a call to the API +// server rather than to the database (FR-025). +func (k *KubernetesBackend) readPodEnvDeclarations(component string) (string, error) { + bound := externalDBControlBound(k.config) + ctx, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + + pod, err := k.getPodForServiceContext(ctx, bound, component) + if err != nil { + return "", err + } + + run := k.externalDBRunner(bound, externalDBTarget("the "+component+" pod", k.namespace)) + + return run("kubectl", "get", "pod", pod, "-n", k.namespace, "-o", podEnvDeclarationsJSONPath) +} diff --git a/src/internal/app/external_credential_refs_test.go b/src/internal/app/external_credential_refs_test.go new file mode 100644 index 00000000..a009a8c7 --- /dev/null +++ b/src/internal/app/external_credential_refs_test.go @@ -0,0 +1,531 @@ +package app + +import ( + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// podEnvListing renders what podEnvDeclarationsJSONPath returns for a pod, so +// the tests read the same shape the cluster produces. +func podEnvListing(defaultContainer string, lines ...string) string { + return "A\t" + defaultContainer + "\n" + strings.Join(lines, "\n") +} + +func neo4jDeploymentSource() deploymentCredentialSource { + return deploymentCredentialSource{ + component: componentInfrahubServer, + username: []string{"INFRAHUB_DB_USERNAME"}, + password: []string{"INFRAHUB_DB_PASSWORD"}, + } +} + +// TestPodEnvDeclarationsJSONPathReadsNoValues pins the property that makes the +// pod-spec read safe: it asks where each variable comes from and never what it +// holds, so an inline credential in the deployment's spec never reaches this +// process or its debug log (FR-014). +func TestPodEnvDeclarationsJSONPathReadsNoValues(t *testing.T) { + for _, forbidden := range []string{"{.value}", ".env[*].value", "{.data", "stringData"} { + if strings.Contains(podEnvDeclarationsJSONPath, forbidden) { + t.Errorf("podEnvDeclarationsJSONPath contains %q: it must read references only, never a value", forbidden) + } + } + for _, want := range []string{"secretKeyRef.name", "secretKeyRef.key", "envFrom", "secretRef.name", `kubectl\.kubernetes\.io/default-container`} { + if !strings.Contains(podEnvDeclarationsJSONPath, want) { + t.Errorf("podEnvDeclarationsJSONPath = %q, want it to read %q", podEnvDeclarationsJSONPath, want) + } + } +} + +// TestDeploymentCredentialRefs covers each way a deployment can declare the +// variables the credentials were read from, and which of them can be +// referenced from the transient container without this tool reading a Secret. +func TestDeploymentCredentialRefs(t *testing.T) { + tests := []struct { + name string + listing string + want map[string]transientSecretKeyRef + wantReason string + }{ + { + name: "env secretKeyRef maps onto the client's own variable names", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_ADDRESS\t\t\t", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t"), + want: map[string]transientSecretKeyRef{ + "NEO4J_USERNAME": {Name: "db-auth", Key: "username"}, + "NEO4J_PASSWORD": {Name: "db-auth", Key: "password"}, + }, + }, + { + // envFrom names no keys and the kubelet skips a key the Secret + // lacks, so a matching prefix does not show the Secret holds the + // variable; a mandatory secretKeyRef to a missing key would block + // the run, so the owned Secret is used instead. + name: "a sole envFrom secretRef with no prefix falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "F\t\tinfrahub-db\t", + "F\tAPP_\tunrelated\t"), + wantReason: "envFrom source", + }, + { + name: "a sole envFrom secretRef whose prefix the variable carries falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "F\tINFRAHUB_DB_\tneo4j-auth\t"), + wantReason: "envFrom source", + }, + { + name: "an inline value falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\t\t\t"), + wantReason: "inline", + }, + { + name: "an env entry overrides envFrom, so an inline override falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_USERNAME\t\t\t", + "E\tINFRAHUB_DB_PASSWORD\t\t\t", + "F\t\tinfrahub-db\t"), + wantReason: "inline", + }, + { + name: "of two env entries with one name the later wins", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t", + "E\tINFRAHUB_DB_PASSWORD\t\t\t"), + wantReason: "inline", + }, + { + name: "a ConfigMap reference falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_USERNAME\t\t\tsettings", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t"), + wantReason: "ConfigMap", + }, + { + name: "two envFrom sources that could supply the variable fall back", + listing: podEnvListing("", + "C\tinfrahub-server", + "F\t\tfirst\t", + "F\t\tsecond\t"), + wantReason: "envFrom source", + }, + { + name: "an envFrom Secret beside an envFrom ConfigMap falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "F\t\tinfrahub-db\t", + "F\t\t\tsettings"), + wantReason: "envFrom source", + }, + { + name: "a variable the spec does not declare falls back", + listing: podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_ADDRESS\t\t\t"), + wantReason: "does not declare", + }, + { + name: "the default-container annotation names the container exec reads", + listing: podEnvListing("server", + "C\tsidecar", + "E\tINFRAHUB_DB_USERNAME\t\t\t", + "E\tINFRAHUB_DB_PASSWORD\t\t\t", + "C\tserver", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t"), + want: map[string]transientSecretKeyRef{ + "NEO4J_USERNAME": {Name: "db-auth", Key: "username"}, + "NEO4J_PASSWORD": {Name: "db-auth", Key: "password"}, + }, + }, + { + name: "without the annotation the first container is the one exec reads", + listing: podEnvListing("", + "C\tserver", + "E\tINFRAHUB_DB_USERNAME\t\t\t", + "E\tINFRAHUB_DB_PASSWORD\t\t\t", + "C\tsidecar", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t"), + wantReason: "inline", + }, + { + name: "a pod with no containers falls back", + listing: podEnvListing(""), + wantReason: "no containers", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + refs, reason := deploymentCredentialRefs(serviceNeo4j, neo4jDeploymentSource(), parsePodEnvDeclarations(tt.listing)) + if !reflect.DeepEqual(refs, tt.want) { + t.Errorf("deploymentCredentialRefs() = %v (%s), want %v", refs, reason, tt.want) + } + if tt.wantReason != "" && !strings.Contains(reason, tt.wantReason) { + t.Errorf("reason = %q, want it to mention %q", reason, tt.wantReason) + } + }) + } +} + +// TestApplyNeo4jEnvironmentRecordsProvenance asserts the deployment is +// recorded as the credentials' source only when this read supplied both. +func TestApplyNeo4jEnvironmentRecordsProvenance(t *testing.T) { + dump := "INFRAHUB_DB_USERNAME=neo4j\nINFRAHUB_DB_PASSWORD=from-deployment\n" + + t.Run("both read from the deployment", func(t *testing.T) { + iops := NewInfrahubOps() + iops.applyNeo4jEnvironment(dump) + + if got, ok := iops.deploymentCredentials[serviceNeo4j]; !ok || !reflect.DeepEqual(got, neo4jDeploymentSource()) { + t.Errorf("deploymentCredentials[neo4j] = %#v, %v, want the infrahub-server source", got, ok) + } + }) + + t.Run("the operator's environment wins and nothing is recorded", func(t *testing.T) { + t.Setenv("INFRAHUB_DB_USERNAME", "operator") + t.Setenv("INFRAHUB_DB_PASSWORD", "operator-pw") + + iops := NewInfrahubOps() + iops.loadCredentialsFromEnvironment() + iops.applyNeo4jEnvironment(dump) + + if _, ok := iops.deploymentCredentials[serviceNeo4j]; ok { + t.Error("deploymentCredentials[neo4j] recorded for credentials the operator supplied") + } + if iops.config.Neo4jPassword != "operator-pw" { + t.Errorf("Neo4jPassword = %q, want the operator's", iops.config.Neo4jPassword) + } + }) + + t.Run("one from the operator and one from the deployment is not a pair the deployment holds", func(t *testing.T) { + t.Setenv("INFRAHUB_DB_USERNAME", "") + t.Setenv("INFRAHUB_DB_PASSWORD", "operator-pw") + + iops := NewInfrahubOps() + iops.loadCredentialsFromEnvironment() + iops.applyNeo4jEnvironment(dump) + + if _, ok := iops.deploymentCredentials[serviceNeo4j]; ok { + t.Error("deploymentCredentials[neo4j] recorded for a mixed pair") + } + }) +} + +// TestApplyPostgresEnvironmentRecordsProvenance covers the PostgreSQL finding: +// the credentials arrive inside the connection URL, which no client can be +// handed a username and a password from, so reuse needs variables beside the +// URL that held the same values — what prefect-server renders when its chart +// takes the user and password from Secret keys. +func TestApplyPostgresEnvironmentRecordsProvenance(t *testing.T) { + t.Run("component variables beside the URL are recorded, the chart's own names first", func(t *testing.T) { + iops := NewInfrahubOps() + iops.applyPostgresEnvironment(strings.Join([]string{ + "PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:pg-secret@db.example:5432/prefect", + "PREFECT_SERVER_DATABASE_USER=prefect", + "PREFECT_SERVER_DATABASE_PASSWORD=pg-secret", + "A_COPY_OF_THE_PASSWORD=pg-secret", + }, "\n")) + + got, ok := iops.deploymentCredentials[serviceTaskManagerDB] + if !ok { + t.Fatal("deploymentCredentials[task-manager-db] not recorded") + } + want := deploymentCredentialSource{ + component: componentTaskManager, + username: []string{"PREFECT_SERVER_DATABASE_USER"}, + password: []string{"PREFECT_SERVER_DATABASE_PASSWORD", "A_COPY_OF_THE_PASSWORD"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("deploymentCredentials[task-manager-db] = %#v, want %#v", got, want) + } + + listing := podEnvListing("", + "C\tprefect-server", + "E\tPREFECT_SERVER_DATABASE_USER\tpg-app\tusername\t", + "E\tPREFECT_SERVER_DATABASE_PASSWORD\tpg-app\tpassword\t", + "E\tPREFECT_SERVER_DATABASE_CONNECTION_URL\t\t\t") + refs, reason := deploymentCredentialRefs(serviceTaskManagerDB, got, parsePodEnvDeclarations(listing)) + wantRefs := map[string]transientSecretKeyRef{ + "PGUSER": {Name: "pg-app", Key: "username"}, + "PGPASSWORD": {Name: "pg-app", Key: "password"}, + } + if !reflect.DeepEqual(refs, wantRefs) { + t.Errorf("deploymentCredentialRefs() = %v (%s), want %v", refs, reason, wantRefs) + } + }) + + t.Run("a URL alone, as the default chart renders it, records nothing", func(t *testing.T) { + iops := NewInfrahubOps() + iops.applyPostgresEnvironment("PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:pg-secret@db.example:5432/prefect\n") + + if _, ok := iops.deploymentCredentials[serviceTaskManagerDB]; ok { + t.Error("deploymentCredentials[task-manager-db] recorded with nothing beside the URL to reference") + } + if iops.config.PostgresPassword != "pg-secret" { + t.Errorf("PostgresPassword = %q, want the URL's: the credentials themselves still resolve", iops.config.PostgresPassword) + } + }) + + t.Run("a partial operator URL is replaced by the deployment's pair, which is recorded", func(t *testing.T) { + iops := NewInfrahubOps() + iops.applyPrefectConnection("postgresql://operator@db.example:5432/prefect") + iops.applyPostgresEnvironment("PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql://prefect:pg-secret@db.example/prefect\nPREFECT_SERVER_DATABASE_USER=prefect\nPREFECT_SERVER_DATABASE_PASSWORD=pg-secret\n") + + if iops.config.PostgresUsername != "prefect" || iops.config.PostgresPassword != "pg-secret" { + t.Fatalf("credentials = %q/%q, want the deployment's prefect/pg-secret", iops.config.PostgresUsername, iops.config.PostgresPassword) + } + got, ok := iops.deploymentCredentials[serviceTaskManagerDB] + if !ok { + t.Fatal("deploymentCredentials[task-manager-db] not recorded although the deployment supplied both values in use") + } + want := deploymentCredentialSource{ + component: componentTaskManager, + username: []string{"PREFECT_SERVER_DATABASE_USER"}, + password: []string{"PREFECT_SERVER_DATABASE_PASSWORD"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("deploymentCredentials[task-manager-db] = %#v, want %#v", got, want) + } + }) + + t.Run("a deployment URL without a password keeps the operator's and records nothing", func(t *testing.T) { + t.Setenv("PGPASSWORD", "") + t.Setenv("PGPASSFILE", filepath.Join(t.TempDir(), "absent")) + iops := NewInfrahubOps() + iops.applyPrefectConnection("postgresql://operator:operator-pw@db.example:5432/prefect") + iops.applyPostgresEnvironment("PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql://prefect@db.example/prefect\nPREFECT_SERVER_DATABASE_USER=prefect\nPREFECT_SERVER_DATABASE_PASSWORD=operator-pw\n") + + if iops.config.PostgresPassword != "operator-pw" { + t.Fatalf("PostgresPassword = %q, want the operator's", iops.config.PostgresPassword) + } + if _, ok := iops.deploymentCredentials[serviceTaskManagerDB]; ok { + t.Error("deploymentCredentials[task-manager-db] recorded for a pair mixing operator and deployment values") + } + }) +} + +// TestResolveDeploymentCredentialRefs covers the decision and its memory: +// operator credentials never read the pod, a read pod is decided once, and a +// failed read is not mistaken for a decision. +func TestResolveDeploymentCredentialRefs(t *testing.T) { + secretBacked := podEnvListing("", + "C\tinfrahub-server", + "E\tINFRAHUB_DB_USERNAME\tdb-auth\tusername\t", + "E\tINFRAHUB_DB_PASSWORD\tdb-auth\tpassword\t") + + t.Run("operator credentials fall back without reading the pod", func(t *testing.T) { + iops := NewInfrahubOps() + refs := iops.resolveDeploymentCredentialRefs(serviceNeo4j, func(string) (string, error) { + t.Fatal("the pod spec was read for credentials the deployment did not supply") + + return "", nil + }) + if refs != nil { + t.Errorf("refs = %v, want nil: the operator's credentials are carried in an owned Secret", refs) + } + }) + + t.Run("a decision is remembered", func(t *testing.T) { + iops := NewInfrahubOps() + iops.recordDeploymentCredentials(serviceNeo4j, neo4jDeploymentSource()) + + reads := 0 + read := func(component string) (string, error) { + reads++ + if component != componentInfrahubServer { + t.Errorf("read the %s pod, want %s", component, componentInfrahubServer) + } + + return secretBacked, nil + } + + first := iops.resolveDeploymentCredentialRefs(serviceNeo4j, read) + second := iops.resolveDeploymentCredentialRefs(serviceNeo4j, read) + if len(first) != 2 || !reflect.DeepEqual(first, second) { + t.Errorf("refs = %v then %v, want the same two references both times", first, second) + } + if reads != 1 { + t.Errorf("read the pod %d times, want once: the component may be scaled to zero by the second workload", reads) + } + }) + + t.Run("a failed read is asked again", func(t *testing.T) { + iops := NewInfrahubOps() + iops.recordDeploymentCredentials(serviceNeo4j, neo4jDeploymentSource()) + + failing := func(string) (string, error) { return "", errors.New("connection refused") } + if refs := iops.resolveDeploymentCredentialRefs(serviceNeo4j, failing); refs != nil { + t.Errorf("refs = %v after a failed read, want nil", refs) + } + + refs := iops.resolveDeploymentCredentialRefs(serviceNeo4j, func(string) (string, error) { return secretBacked, nil }) + if len(refs) != 2 { + t.Errorf("refs = %v, want the references once the pod answered", refs) + } + }) +} + +func testReusingSpec() transientWorkloadSpec { + spec := testCaptureSpec() + spec.Credentials = nil + spec.CredentialRefs = map[string]transientSecretKeyRef{ + "NEO4J_USERNAME": {Name: "db-auth", Key: "username"}, + "NEO4J_PASSWORD": {Name: "db-auth", Key: "password"}, + } + + return spec +} + +// TestReusingWorkloadCreatesNoSecret is the reviewer's request asserted: a +// workload whose credentials the deployment already keeps in a Secret +// references that Secret, so the run creates a Job and nothing else, and no +// credential value appears in what it sends to the cluster. +func TestReusingWorkloadCreatesNoSecret(t *testing.T) { + spec := testReusingSpec() + + manifest, err := buildTransientWorkloadManifest(spec) + if err != nil { + t.Fatalf("buildTransientWorkloadManifest() error = %v", err) + } + + var job transientJobManifest + if err := json.Unmarshal(manifest, &job); err != nil { + t.Fatalf("manifest is not valid JSON: %v", err) + } + container := job.Spec.Template.Spec.Containers[0] + if len(container.EnvFrom) != 0 { + t.Errorf("envFrom = %#v, want none: there is no owned Secret to read", container.EnvFrom) + } + want := []transientEnvVar{ + {Name: "NEO4J_PASSWORD", ValueFrom: &transientEnvVarSource{SecretKeyRef: &transientSecretKeySelector{Name: "db-auth", Key: "password", Optional: boolPtr(false)}}}, + {Name: "NEO4J_USERNAME", ValueFrom: &transientEnvVarSource{SecretKeyRef: &transientSecretKeySelector{Name: "db-auth", Key: "username", Optional: boolPtr(false)}}}, + } + if !reflect.DeepEqual(container.Env, want) { + t.Errorf("env = %s, want secretKeyRefs onto the deployment's Secret, not optional", mustJSON(t, container.Env)) + } + + backend := testBackend() + cluster := &fakeCluster{uid: "8f14e45f-ea3b-4b0e-9d3e-2a1c5f6b7d80"} + workload, err := backend.createTransientWorkloadWith(cluster.ops(), spec) + if err != nil { + t.Fatalf("createTransientWorkloadWith() error = %v", err) + } + if got := cluster.createdKinds(); !reflect.DeepEqual(got, []string{"Job"}) { + t.Errorf("created %v, want the Job alone: a reusing workload needs no secrets verb", got) + } + if workload.SecretName != "" { + t.Errorf("SecretName = %q, want empty: the workload owns no Secret", workload.SecretName) + } + for _, created := range cluster.created { + for _, value := range []string{"s3cr3t-pw", "stringData"} { + if strings.Contains(string(created), value) { + t.Errorf("a created manifest carries %q: a reusing workload sends no credential value", value) + } + } + } +} + +// TestOwnedSecretWorkloadIsUnchanged pins the fallback: a spec carrying values +// still gets its envFrom of the owned Secret and no secretKeyRef. +func TestOwnedSecretWorkloadIsUnchanged(t *testing.T) { + manifest, err := buildTransientWorkloadManifest(testCaptureSpec()) + if err != nil { + t.Fatalf("buildTransientWorkloadManifest() error = %v", err) + } + + var job transientJobManifest + if err := json.Unmarshal(manifest, &job); err != nil { + t.Fatalf("manifest is not valid JSON: %v", err) + } + container := job.Spec.Template.Spec.Containers[0] + if len(container.Env) != 0 { + t.Errorf("env = %#v, want none on the owned-Secret path", container.Env) + } + if len(container.EnvFrom) != 1 || container.EnvFrom[0].SecretRef.Name != testCaptureSpec().secretName() { + t.Errorf("envFrom = %#v, want the owned Secret", container.EnvFrom) + } +} + +// TestReusingSpecRefusals covers the guards that keep a reusing spec from +// shipping a value or creating the Secret it exists to avoid. +func TestReusingSpecRefusals(t *testing.T) { + both := testReusingSpec() + both.Credentials = map[string]string{"NEO4J_USERNAME": "neo4j", "NEO4J_PASSWORD": "pw"} + if _, err := buildTransientWorkloadManifest(both); err == nil { + t.Error("buildTransientWorkloadManifest() with values and references = nil error, want a refusal") + } + + missing := testReusingSpec() + delete(missing.CredentialRefs, "NEO4J_PASSWORD") + if _, err := buildTransientWorkloadManifest(missing); err == nil { + t.Error("buildTransientWorkloadManifest() with no password reference = nil error, want a refusal") + } + + if _, err := buildTransientSecretManifest(testReusingSpec(), "uid"); err == nil { + t.Error("buildTransientSecretManifest() for a reusing spec = nil error, want a refusal") + } +} + +// TestReapToleratesForbiddenSecretListing is the other half of dropping the +// secrets verbs: a namespace that refuses the secrets listing still has its +// abandoned pods reaped, and the run is not failed over it. +func TestReapToleratesForbiddenSecretListing(t *testing.T) { + old := reapNow.Add(-4 * time.Hour).UTC().Format(time.RFC3339) + pods := fmt.Sprintf("%s-probe-cccc3333\tcccc3333\tFailed\t%s\t1200\n", transientObjectPrefix, old) + + t.Run("forbidden is a warning", func(t *testing.T) { + backend := testBackend() + cluster := &fakeReapCluster{ + pods: pods, + secretsErr: errors.New(`Error from server (Forbidden): secrets is forbidden: User "backup" cannot list resource "secrets" in API group "" in the namespace "infrahub"`), + } + + if err := backend.reapTransientStraysWith(cluster.ops(), "aaaa1111", reapNow); err != nil { + t.Fatalf("reapTransientStraysWith() error = %v, want nil when only the secrets listing is forbidden", err) + } + if got := cluster.deletedNames(); !reflect.DeepEqual(got, []string{"pod " + transientObjectPrefix + "-probe-cccc3333"}) { + t.Errorf("deleted %v, want the abandoned pod still reaped", got) + } + }) + + t.Run("any other secrets listing failure still stops the sweep", func(t *testing.T) { + backend := testBackend() + cluster := &fakeReapCluster{pods: pods, secretsErr: errors.New("connection refused")} + + if err := backend.reapTransientStraysWith(cluster.ops(), "aaaa1111", reapNow); err == nil { + t.Fatal("reapTransientStraysWith() = nil, want an error for a listing that failed rather than was refused") + } + if len(cluster.deleted) != 0 { + t.Errorf("deleted %v, want nothing: the listing was incomplete", cluster.deletedNames()) + } + }) +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + + out, err := json.Marshal(v) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + + return string(out) +} diff --git a/src/internal/app/external_endpoint.go b/src/internal/app/external_endpoint.go new file mode 100644 index 00000000..54808606 --- /dev/null +++ b/src/internal/app/external_endpoint.go @@ -0,0 +1,2107 @@ +package app + +import ( + "cmp" + "errors" + "fmt" + "net" + "net/netip" + "strconv" + "strings" + + "github.com/sirupsen/logrus" +) + +// A database that lives outside the deployment has no container to resolve, so +// everything the run needs to reach it has to come from somewhere else: the +// deployment's own components describe most of it, the operator overrides the +// rest, and one detail — the backup listener's port — is not recorded anywhere +// at all. This file is where those three sources become one endpoint, and where +// that endpoint is reported before anything is attempted against it. + +// The documented deployment service names an endpoint may stand for. FR-017 +// forbids introducing a new assumed service name, so an external database +// resolves under the same name its container would have had. +const ( + serviceNeo4j = "database" + serviceTaskManagerDB = "task-manager-db" +) + +// errNotADatabaseService refuses a service this feature has no database +// meaning for, naming the two that it does. action completes "cannot …". +// +// It is one function because the vocabulary sentence was written out at every +// site that guards on the pair, so a third database meant finding all of them. +func errNotADatabaseService(action, service string) error { + return fmt.Errorf("cannot %s %q: the databases are %s and %s", action, service, serviceNeo4j, serviceTaskManagerDB) +} + +// Infrahub's own defaults for the database settings discovery reads. They are +// repeated here rather than assumed present in a component's environment, +// because a deployment that leaves a setting at its default does not export it +// at all — so "absent" and "default" are the same observation. +const ( + // defaultNeo4jClientPort is the default of Infrahub's INFRAHUB_DB_PORT: the + // Bolt port the application talks to, used here for the version probe and + // for restore. It is emphatically not the backup listener's port + // (defaultNeo4jBackupPort) — different listener, different purpose. + defaultNeo4jClientPort = 7687 + + // defaultNeo4jProtocol is the default of Infrahub's INFRAHUB_DB_PROTOCOL. + // The routed neo4j:// form is what a multi-member deployment needs; bolt:// + // addresses one server. + defaultNeo4jProtocol = "bolt" + + // defaultPostgresPort is the port a PostgreSQL connection URL means when it + // carries none of its own. + defaultPostgresPort = 5432 +) + +// EndpointLocation says whether a logical database lives inside the deployment +// or outside it. Both values are stated explicitly and neither is the zero +// value, so a failure to query the deployment cannot pass for "external": +// FR-002 requires that failure to surface as an error rather than as evidence +// the database is elsewhere. +type EndpointLocation string + +const ( + EndpointLocationInternal EndpointLocation = "internal" + EndpointLocationExternal EndpointLocation = "external" +) + +// HostPort is one member of a database's address list, with the port it will +// actually be reached on — either the one the member carried or the client port +// it inherited. +type HostPort struct { + Host string + Port int +} + +// String renders the member the way a client expects it, bracketing an IPv6 +// literal so that host and port stay tellable apart. +func (hp HostPort) String() string { + return net.JoinHostPort(hp.Host, strconv.Itoa(hp.Port)) +} + +// DatabaseEndpoint is where one logical database actually lives: resolved once +// per run per database, and thereafter the single source of truth for both the +// backup and the restore path. +// +// It deliberately carries no credential material. Credentials keep the +// resolution they already have on Configuration and reach a transient workload +// through a secret it owns, so there is no path by which reporting an endpoint +// can print a password (FR-014). +type DatabaseEndpoint struct { + // Service is the documented deployment service name this endpoint stands + // for: serviceNeo4j or serviceTaskManagerDB, never a new name (FR-017). + Service string + + // Location is internal or external, positively established (FR-002). + Location EndpointLocation + + // Hosts is the address list in the order it will be tried — which is the + // order the deployment or the operator wrote it in, because that order is + // what the backup command consumes (FR-005). + Hosts []HostPort + + // ClientPort is the port of the application protocol: Bolt for Neo4j, the + // wire protocol for PostgreSQL. A member of Hosts written without a port of + // its own inherits it. Zero means the deployment records none, in which case + // Infrahub's own default applies. + ClientPort int + + // BackupPort is the port the Neo4j backup service listens on. Unused for + // PostgreSQL, whose dump travels over the client port. + BackupPort int + + // Protocol is bolt or neo4j, from INFRAHUB_DB_PROTOCOL. Empty for + // PostgreSQL, which has no such setting. + Protocol string + + // TLS describes the client connection's TLS as the deployment's own + // INFRAHUB_DB_TLS_* settings state it. + TLS ExternalDBTLS + + // Database names the database inside the server. It is carried rather than + // assumed: it names the entry in the artifact, and a cross-name restore is a + // trap this codebase has already had to fix once. + Database string + + // reported records that the endpoint has been logged, so that resolution + // reporting it and a connection path re-asserting it do not put the same + // line in front of the operator twice. + reported bool +} + +// resolveHosts fills the endpoint's Hosts from a comma-separated host[:port] +// list, applying the endpoint's client port to every member that carries no +// port of its own. +func (e *DatabaseEndpoint) resolveHosts(address string) error { + hosts, err := parseHostPortList(address, e.effectiveClientPort()) + if err != nil { + return fmt.Errorf("failed to resolve the %s endpoint: %w", e.Service, err) + } + + e.Hosts = hosts + + return nil +} + +// effectiveClientPort is the client port the run will actually use: the one the +// deployment records, or Infrahub's own default for the service when it records +// none. +func (e DatabaseEndpoint) effectiveClientPort() int { + if e.ClientPort > 0 { + return e.ClientPort + } + + if e.Service == serviceTaskManagerDB { + return defaultPostgresPort + } + + return defaultNeo4jClientPort +} + +// effectiveProtocol is the protocol the run will actually use. PostgreSQL has no +// equivalent setting, so it reports none rather than inventing one. +func (e DatabaseEndpoint) effectiveProtocol() string { + if e.Protocol != "" { + return e.Protocol + } + + if e.Service == serviceNeo4j { + return defaultNeo4jProtocol + } + + return "" +} + +// Report logs the endpoint the run resolved. Every path that opens a connection +// to an endpoint MUST call this before opening it, so that the address a run is +// about to talk to reaches the operator's terminal ahead of the attempt +// (FR-003): when a connection then hangs or is refused, the line naming the +// endpoint is already there instead of being lost with the failure. It is +// idempotent, which is what makes calling it defensively from a connection path +// free even when resolution has already reported. +func (e *DatabaseEndpoint) Report() { + if e.reported { + return + } + e.reported = true + + logrus.Infof("Resolved %s endpoint: %s", e.Service, e.Describe()) +} + +// Describe renders the endpoint in one line, for the report above and for a +// failure message that has to name the resource it failed on. It states the +// values the run will use rather than the ones it read, so a setting left at its +// default still appears; it names no credential — see the type's own note. +func (e DatabaseEndpoint) Describe() string { + hosts := cmp.Or(renderHostPorts(e.Hosts), "none") + + fields := []string{ + fmt.Sprintf("location=%s", e.Location), + fmt.Sprintf("hosts=%s", hosts), + } + if protocol := e.effectiveProtocol(); protocol != "" { + fields = append(fields, "protocol="+protocol) + } + fields = append(fields, fmt.Sprintf("client-port=%d", e.effectiveClientPort())) + if e.BackupPort > 0 { + fields = append(fields, fmt.Sprintf("backup-port=%d", e.BackupPort)) + } + if e.Database != "" { + fields = append(fields, "database="+e.Database) + } + fields = append(fields, "tls="+describeTLS(e.TLS)) + + return strings.Join(fields, " ") +} + +// describeTLS renders the client connection's TLS settings for the endpoint +// report. It reports tls_insecure because that is what the deployment says, not +// because this tool takes it as licence to skip verification for the connections +// it opens (FR-029). +func describeTLS(tls ExternalDBTLS) string { + if !tls.Enabled { + return "disabled" + } + + description := "enabled" + if tls.CAFile != "" { + description += ",ca=" + tls.CAFile + } + if tls.Insecure { + description += ",deployment-insecure" + } + + return description +} + +// resolveNeo4jBackupPort is the port the backup service will be reached on: the +// operator's --neo4j-backup-port when they set one, and defaultNeo4jBackupPort +// otherwise. +// +// This is the one endpoint detail discovery cannot supply at all. Infrahub never +// connects to the backup listener, so no setting in the deployment records its +// port and there is nothing to read (research R4) — which also means the default +// here is the database vendor's rather than the deployment's, and a server whose +// backup listener was configured onto another port needs the flag. +func resolveNeo4jBackupPort(cfg ExternalDBConfig) int { + if cfg.Neo4jBackupPort > 0 { + return cfg.Neo4jBackupPort + } + + return defaultNeo4jBackupPort +} + +// discoveredEndpoint is what the deployment's own components say about where a +// database lives: INFRAHUB_DB_ADDRESS and INFRAHUB_DB_PORT for Neo4j, the host +// and port of the Prefect connection URL for PostgreSQL. +// +// It is kept apart from ExternalDBConfig on purpose. That struct carries the +// operator's overrides, and an override has to survive discovery rather than be +// overwritten by it, so the two are recorded separately and reconciled when the +// endpoint is resolved. +type discoveredEndpoint struct { + // Address is a comma-separated host[:port] list for Neo4j, and a single host + // for PostgreSQL. Empty when the deployment exports no address. + Address string + + // ClientPort is the port members of Address inherit when they carry none. + // Zero when the deployment exports no port. + ClientPort int + + // read records that the deployment's own configuration for this database has + // already been read, so the external path asks the deployment once rather + // than once per consultation. It is distinct from "Address is non-empty": + // a deployment that exports no address at all has still been asked, and + // asking again would not change the answer. + read bool +} + +// parseHostPortList parses a comma-separated host[:port] list the way Infrahub +// itself reads INFRAHUB_DB_ADDRESS: "Database host, or a comma-separated list of +// cluster members in host[:port] format. Members without an explicit port use +// the value of the port setting." defaultPort is that port setting, and it is +// what a member without a port of its own inherits. +// +// Order is preserved, because it is the order the backup command will try the +// members in (FR-005). An empty list is not an error — it is a deployment that +// configures no address, which the caller distinguishes from a broken one — but +// a member that cannot be read is, since dropping it silently would hand the +// caller a shorter list than the operator wrote and a capture from fewer +// endpoints than they asked for. +func parseHostPortList(address string, defaultPort int) ([]HostPort, error) { + if strings.TrimSpace(address) == "" { + return nil, nil + } + + members := strings.Split(address, ",") + hosts := make([]HostPort, 0, len(members)) + for _, member := range members { + host, err := parseHostPort(strings.TrimSpace(member), defaultPort) + if err != nil { + return nil, fmt.Errorf("invalid address list %q: %w", address, err) + } + + hosts = append(hosts, host) + } + + return hosts, nil +} + +// parseHostPort parses one member of an address list. The member may be a host +// name, an IPv4 address or an IPv6 literal, with or without a port; a bare IPv6 +// literal has to be recognised by its colons, since it looks exactly like a +// host:port pair to a naive split. +func parseHostPort(member string, defaultPort int) (HostPort, error) { + switch { + case member == "": + return HostPort{}, errors.New("it has an empty member") + + // A bracketed IPv6 literal carrying no port: [::1]. net.SplitHostPort + // rejects it for having no port, and the brackets are notation rather than + // part of the address. + case strings.HasPrefix(member, "[") && strings.HasSuffix(member, "]"): + host := strings.TrimSuffix(strings.TrimPrefix(member, "["), "]") + switch { + case host == "": + return HostPort{}, fmt.Errorf("member %q has no host", member) + case !isIPLiteral(host): + return HostPort{}, fmt.Errorf("member %q is bracketed but %q is not an IPv6 address: brackets are the notation for an IPv6 literal, so a host name is written without them", member, host) + } + + return HostPort{Host: host, Port: defaultPort}, nil + + // A bare IPv6 literal: ::1, fe80::1. More than one colon with no brackets + // means every colon belongs to the address, so there is no port to split off + // and the member inherits defaultPort. + // + // The literal is verified, not assumed from the colon count. Counting alone + // reads anything with two colons as an address, so `db-a:7687;db-b:7687` — + // a semicolon where the operator meant a comma — became a single host whose + // name contains the whole typo, and that host reached the capture's --from + // list and the artefact's own provenance metadata as a member this run + // claims to have read. A name that cannot resolve is a failure either way; + // what this stops is the failure arriving as a recorded endpoint rather + // than as a refused address list. + case !strings.HasPrefix(member, "[") && strings.Count(member, ":") > 1: + if !isIPLiteral(member) { + return HostPort{}, fmt.Errorf("member %q is not in host[:port] form: it holds more than one colon, which is only valid for an IPv6 literal, and it is not one — an IPv6 address with a port is written as [address]:port, and a list of members is separated by commas", member) + } + + return HostPort{Host: member, Port: defaultPort}, nil + + // host:port, or [::1]:7687. + case strings.Contains(member, ":"): + host, portText, err := net.SplitHostPort(member) + if err != nil { + return HostPort{}, fmt.Errorf("member %q is not in host[:port] form: %w", member, err) + } + if host == "" { + return HostPort{}, fmt.Errorf("member %q has no host", member) + } + if strings.HasPrefix(member, "[") && !isIPLiteral(host) { + return HostPort{}, fmt.Errorf("member %q is bracketed but %q is not an IPv6 address: brackets are the notation for an IPv6 literal, so a host name is written without them", member, host) + } + + port, err := parsePort(portText) + if err != nil { + return HostPort{}, fmt.Errorf("member %q: %w", member, err) + } + + return HostPort{Host: host, Port: port}, nil + + // A host with no port of its own, which is the inheritance case. + default: + return HostPort{Host: member, Port: defaultPort}, nil + } +} + +// parsePort reads a port number, rejecting anything outside the range a listener +// can occupy. Zero is rejected along with the rest: a port the caller cannot +// connect to is not a usable default, and accepting it would push the failure +// down to the connection attempt where it reads as an unreachable server. +func parsePort(text string) (int, error) { + port, err := strconv.Atoi(text) + if err != nil { + return 0, fmt.Errorf("port %q is not a number: %w", text, err) + } + if port < 1 || port > 65535 { + return 0, fmt.Errorf("port %d is outside the range 1-65535", port) + } + + return port, nil +} + +// isIPLiteral reports whether text is an IP address rather than a host name. +// +// Its callers ask it only where the notation says the member must be a literal +// — inside brackets, or unbracketed with more than one colon — so in practice +// the question it answers is "is this really an IPv6 address". It is not +// narrowed to IPv6 on purpose: an IPv4-mapped literal such as `::ffff:10.0.0.1` +// carries the colons that make it look like host:port, and refusing it would +// refuse an address that resolves perfectly well. +// +// A zone identifier is accepted and kept: `fe80::1%eth0` is how a link-local +// address names the interface it is reachable on, and dropping it would leave +// an address that cannot be connected to — which is why this parses with +// netip, which takes the zone, rather than net.ParseIP, which does not. +func isIPLiteral(text string) bool { + _, err := netip.ParseAddr(text) + + return err == nil +} + +// serviceLocation answers the one question the internal-versus-external +// decision rests on: does this deployment service have a container of its own? +// +// It is a function value rather than a method so resolution can be driven in a +// test without a cluster, the way getAllPodsWith takes its runner. +type serviceLocation func(service string) (EndpointLocation, error) + +// deploymentQueries are the two questions endpoint resolution puts to the +// deployment: where a service's container is, and — only once a database has +// been established as external — what the deployment's own components say about +// the database behind it. +type deploymentQueries struct { + locate serviceLocation + discover func(service string) error +} + +// ResolveEndpoint resolves where one logical database lives. It is called once +// per database per run, and each database is resolved independently of the +// other, so a deployment with one internal and one external database needs no +// additional operator input (FR-001). +func (iops *InfrahubOps) ResolveEndpoint(service string) (*DatabaseEndpoint, error) { + queries, err := iops.deploymentQueries() + if err != nil { + return nil, err + } + + return iops.resolveEndpointWith(queries, service) +} + +// deploymentQueries binds the two queries to the detected environment. The +// location query is backend-specific; discovery is not, because it reads a +// component's environment through the shared Exec primitive. +func (iops *InfrahubOps) deploymentQueries() (deploymentQueries, error) { + backend, err := iops.ensureBackend() + if err != nil { + return deploymentQueries{}, fmt.Errorf("cannot resolve database endpoints without a detected environment: %w", err) + } + + return deploymentQueries{ + locate: locateServiceIn(backend), + discover: iops.discoverEndpoint, + }, nil +} + +// serviceLocator is a backend that can say whether a deployment service runs +// inside it. Taking the capability as an interface rather than as the concrete +// Kubernetes backend is what lets the paths that gate on the location be driven +// in a test without a cluster; in production only KubernetesBackend implements +// it. +type serviceLocator interface { + locateService(service string) (EndpointLocation, error) +} + +// locateServiceIn is the backend's answer to "does this service run here?". +// +// Only the Kubernetes backend can answer it meaningfully: external databases are +// Kubernetes-only for this feature, so for Docker Compose the question does not +// arise and the answer is internal by construction. Reporting internal there is +// not an assumption about the deployment — it is the scope, and it keeps a +// Compose deployment on exactly the path it is on today (FR-015). +func locateServiceIn(backend EnvironmentBackend) serviceLocation { + if locator, ok := backend.(serviceLocator); ok { + return locator.locateService + } + + return func(string) (EndpointLocation, error) { + return EndpointLocationInternal, nil + } +} + +// A path that operates on a database has to establish that it can reach it, +// and it gets exactly one chance to do so: before anything is stopped. +// +// That used to be a step — a gate each entry point remembered to call. Which is +// how both capture entry points had one while all three restore entry points +// shipped without one, and how the next entry point would have shipped without +// one too. A step can be forgotten; a property cannot. So the location and the +// refusal live together on the target the operation resolved, and a +// databaseTarget exists only where the location was established and this build +// can actually perform the operation against it. Holding one is the evidence, +// and there is one way to obtain one. + +// databaseOperation is what a run intends to do to a database. +// +// The two are not interchangeable. Their refusals differ in the fact that +// matters most on each path — a restore's says no data was changed — and the +// two capabilities do not become available together, because restoring into a +// database the deployment does not manage additionally requires FR-009's +// authorisation. +type databaseOperation string + +const ( + databaseCapture databaseOperation = "capture" + databaseRestore databaseOperation = "restore" + + // databaseInspect reads where a database lives and says so, and is the only + // operation that moves no data and creates nothing. It exists so that + // `environment detect` resolves locations through databaseTargetWith, the + // constructor every other path reaches through resolveDatabaseTargets, + // rather than through a second resolver of its own — + // and so that its failures name the report rather than a backup the + // operator did not ask for. `infrahub-collect` runs it, so it must stay one + // that creates no workload (ADR-0003). + databaseInspect databaseOperation = "inspect" +) + +// describe names the run in the failure a location that could not be +// established produces: "cannot start the backup", "cannot start the restore". +func (o databaseOperation) describe() string { + switch o { + case databaseRestore: + return "restore" + case databaseInspect: + return "environment report" + } + + return "backup" +} + +// databaseTarget is one database an operation may act on: which database, where +// the deployment says it lives, and what is about to be done to it. +// +// Holding one is meant to be the evidence that the location was established and +// that this run may perform the operation. That was true of every target +// production builds, and true only by convention: with four exported fields and +// nothing else, `databaseTarget{Service: serviceNeo4j, Location: +// EndpointLocationExternal, Operation: databaseRestore}` was a value any caller +// could write, and it would have carried a zero Restore — no authorisation — +// into prepareExternalRestoreWith, which takes a target by value and +// deliberately does not re-check it. An unauthorised external restore, refused +// nowhere. +// +// So the evidence is now a field only databaseTargetWith can set. See gated. +type databaseTarget struct { + Service string + Location EndpointLocation + Operation databaseOperation + + // Restore is FR-009's authorisation, carried on the target because that is + // the only thing every path that could write to a database has to hold. + // It is consulted only for a restore into an external database; nothing + // else in the tool reads it, and an internal restore is unaffected by it. + Restore ExternalRestoreAuth + + // gated is set by databaseTargetWith, after reachable has answered, and by + // nothing else — it is unexported, so no code outside this package can + // produce a target that claims to have passed the gate, and inside the + // package the compiler makes the one assignment easy to find. + // + // It is checked by the two preparers, which are what a target authorises. + // This is the same shape the two defects on this branch had: an invariant + // that held because every constructor happened to be correct, with the + // reader trusting a value the type could not promise. Both times the fix + // was to make the type carry the promise (see requireGated). + gated bool +} + +// requireGated refuses a target that did not come from databaseTargetWith. +// +// It is what the preparers hold instead of re-deriving the location and the +// authorisation for themselves. Re-deriving would be a second answer to a +// question the gate exists to settle once — and the reason the gate exists is +// that all three restore entry points shipped with no reachability check at all +// (T083). So the preparers check the evidence rather than the facts, and this +// is what makes that sound. +func (t databaseTarget) requireGated(action string) error { + if t.gated { + return nil + } + + return fmt.Errorf( + "cannot %s the external %s database: this target did not come from the gate, so neither its location nor FR-009's authorisation has been established. "+ + "No data was changed", action, t.Service) +} + +// reachable reports whether this run may operate on this target, and it is the +// refusal itself rather than a check that produces one. An internal database is +// reachable by construction: the container is there, and every existing path +// reaches it through the deployment. +func (t databaseTarget) reachable() error { + if t.Location != EndpointLocationExternal { + return nil + } + + switch t.Operation { + case databaseCapture: + // Reachable: the capture path exists, and what it needs is prepared by + // prepareDatabaseCaptureWith immediately after this resolution. Holding + // the target is still the evidence that the location was established + // positively rather than inferred from a failed query (FR-002). + return nil + case databaseInspect: + // Always reachable, and it is the one arm where that needs no + // qualification: an inspection reads the location and reports it, + // creates nothing, writes nothing, and so has no capability to + // establish and nothing to authorise. FR-009's authorisation is + // emphatically not consulted — requiring it would make `environment + // detect` refuse to describe a deployment the operator is diagnosing + // precisely because they cannot back it up. + return nil + case databaseRestore: + // FR-009's authorisation, and this is the one place it is read. It is + // required for every restore into a database the deployment does not + // manage, and this constructor is the one thing every such path holds — + // so the check cannot be omitted by a path that lands later, which is + // exactly how the three restore entry points shipped without a + // reachability check at all (see T083). + // + // Nothing else in the tool consults it, and in particular no restore + // path asks again: holding the target *is* the answer. That is what + // keeps --force and the scheduled-restore setting from conferring it by + // accident — neither is on this line, and there is no second line. + if !t.Restore.allowed() { + return externalRestoreUnauthorised(t.Service) + } + + // Authorised, and the restore path for an external database exists: + // what it needs is prepared by prepareDatabaseRestoreWith immediately + // after this resolution. Holding the target is still the evidence that + // the location was established positively rather than inferred from a + // failed query (FR-002). + return nil + } + + return fmt.Errorf("cannot operate on the %s database: %q is not an operation this build knows", t.Service, t.Operation) +} + +// databaseTargets are the databases one operation will act on, in the order +// they were gated. +// +// It is a named type because the targets are the run's evidence and not a +// by-product of producing an error: every caller of resolveDatabaseTargets used +// to discard them — `if _, err := …` has a caller and no consumer — so the +// location each target carried was established, thrown away, and then asked for +// again by the preparation that followed. The methods below are what the +// preparation reads instead of re-querying the deployment. +type databaseTargets []databaseTarget + +// external returns the targets whose database lives outside the deployment. +// +// This is the answer prepareExternalSources used to recompute with a second +// locate call per database, having been handed the service names rather than +// the targets. The locations are memoised (see locateService), so the duplicate +// was a memo read rather than a second API call — but two readings of one +// question is the shape this phase exists to remove, and only one of them was +// the gate. +func (t databaseTargets) external() databaseTargets { + external := make(databaseTargets, 0, len(t)) + for _, target := range t { + if target.Location == EndpointLocationExternal { + external = append(external, target) + } + } + + return external +} + +// services names the databases these targets cover, for the one line an +// operator reads about what the gate established. +func (t databaseTargets) services() []string { + services := make([]string, 0, len(t)) + for _, target := range t { + services = append(services, target.Service) + } + + return services +} + +// databaseTargetWith resolves one database an operation is about to act on, +// against the supplied queries, following resolveEndpointWith's injection idiom +// so the decision is testable without a cluster. +// +// A location that could not be established is an error, not a location: FR-002 +// forbids reading a failure to query the deployment as evidence that the +// database is elsewhere, and here that would be the difference between stopping +// and scaling Infrahub down for nothing. +func databaseTargetWith(queries deploymentQueries, operation databaseOperation, service string, auth ExternalRestoreAuth) (databaseTarget, error) { + location, err := queries.locate(service) + if err != nil { + return databaseTarget{}, fmt.Errorf("cannot start the %s: %w", operation.describe(), err) + } + + target := databaseTarget{Service: service, Location: location, Operation: operation, Restore: auth} + if err := target.reachable(); err != nil { + return databaseTarget{}, err + } + + // The one assignment of the witness, and it is here rather than in the + // literal above so that it cannot be set by a target reachable refused. + target.gated = true + + return target, nil +} + +// resolveEndpointWith is the resolution itself, against the supplied queries. +// +// The order is deliberate: the location is established first, and everything +// external mode needs is read only afterwards. That is what makes FR-015 +// structural rather than a promise — a deployment whose databases are all +// internal returns from here before any new query is issued or any new setting +// is consulted. +func (iops *InfrahubOps) resolveEndpointWith(queries deploymentQueries, service string) (*DatabaseEndpoint, error) { + if service != serviceNeo4j && service != serviceTaskManagerDB { + return nil, errNotADatabaseService("resolve a database endpoint for", service) + } + + location, err := queries.locate(service) + if err != nil { + return nil, fmt.Errorf("failed to resolve the %s endpoint: %w", service, err) + } + + endpoint := &DatabaseEndpoint{Service: service, Location: location} + + if location != EndpointLocationExternal { + // The container is there, so there is nothing to resolve: every existing + // path reaches it through the deployment. Recorded at debug level so an + // all-internal deployment's output is the output it has always had. + logrus.Debugf("Resolved %s endpoint: %s", service, endpoint.Describe()) + + return endpoint, nil + } + + // Discovery can fail without the run failing: an operator who supplied the + // address explicitly has given the one thing that cannot be defaulted, and + // refusing work they fully specified would be worse than proceeding on the + // documented defaults. The warning names the reason, and the endpoint report + // below states the values the run will actually use. + // + // The failure is carried forward rather than only logged, because it decides + // which refusal an operator who supplied *nothing* gets. "The settings were + // read and none of them records an address" (FR-004) and "the settings could + // not be read" (FR-019) are two conditions with different remedies, and a + // warning in a terminal is not where a refused run says which it met. + discoveryErr := queries.discover(service) + if discoveryErr != nil { + logrus.Warnf("Could not read the %s settings from the deployment, falling back to the supplied configuration: %v", service, discoveryErr) + } + + if err := iops.applyDiscoveredFacts(endpoint, discoveryErr); err != nil { + return nil, err + } + + // FR-003: the address reaches the operator's terminal before anything is + // attempted against it, so a connection that then hangs or is refused has + // already been preceded by the endpoint it was made to. + endpoint.Report() + + return endpoint, nil +} + +// ReportDatabaseLocations states, per database, whether it runs in this +// deployment and — where it does not — the endpoint it was resolved to +// (US2 scenario 5). +// +// `environment detect` is the first command an operator reaches for when a +// backup fails, and until now it named the deployment and its credentials and +// said nothing at all about where the databases were. An operator whose backup +// had just failed on an external database therefore learnt nothing from the one +// command written to tell them what the tool sees. +// +// It is shared by all three binaries through AttachEnvironmentCommands, so it +// must be correct for the read-only one: it resolves the location, which is a +// pod listing, and then reads the deployment's own configuration through the +// existing exec — no workload is created, and databaseInspect is what makes +// that structural rather than a property of this function (ADR-0003). +func (iops *InfrahubOps) ReportDatabaseLocations() error { + queries, err := iops.deploymentQueries() + if err != nil { + return err + } + + return iops.reportDatabaseLocationsWith(queries) +} + +// reportDatabaseLocationsWith is the report against the supplied queries, +// following resolveEndpointWith's injection idiom so it is testable without a +// cluster. +// +// Neither half of it fails the command. A location that could not be +// established is reported as exactly that — never as a database outside the +// deployment (FR-002) — because this is the diagnostic an operator runs when +// something else has already failed, and a detect that exits on the first +// unanswered query tells them less than the deployment and credentials it would +// otherwise have printed. An endpoint that could not be resolved is reported +// with the reason, which for an external database is FR-004's refusal naming +// where discovery looked and the override that supplies it. +func (iops *InfrahubOps) reportDatabaseLocationsWith(queries deploymentQueries) error { + // Both databases, not the run's selection: nothing here reads or writes + // either one, and an operator diagnosing a deployment wants the whole + // picture rather than the part some later --exclude-task-manager would have + // covered. + // + // Each database is resolved on its own, through the same constructor + // resolveDatabaseTargets calls, rather than through resolveDatabaseTargets + // itself. That function stops at the first database it cannot locate, which + // is right for a run — nothing may be stopped once one location is unknown — + // and wrong here: a query that failed for one database says nothing about + // the other, and the operator diagnosing the failure needs whatever the + // deployment could still answer. + for _, service := range databaseServicesForBackup(true) { + target, err := databaseTargetWith(queries, databaseInspect, service, ExternalRestoreAuth{}) + if err != nil { + logrus.Warnf("Database %s: could not establish where it lives: %v", service, err) + + continue + } + + if target.Location != EndpointLocationExternal { + logrus.Infof("Database %s: runs in this deployment", target.Service) + + continue + } + + endpoint, err := iops.resolveEndpointWith(queries, target.Service) + if err != nil { + logrus.Warnf("Database %s: does not run in this deployment, and its endpoint could not be resolved: %v", target.Service, err) + + continue + } + + // The values repeat the line resolveEndpointWith has just logged for + // FR-003, and that is deliberate here: US2 scenario 5 asks for a + // statement per database that carries both the location and where it + // resolved to, and an operator reading a failing deployment should not + // have to join two lines to get one answer. The duplication is one line + // per external database, on the one command whose whole purpose is to + // describe what the tool sees. + logrus.Infof("Database %s: does not run in this deployment; resolved to %s", target.Service, endpoint.Describe()) + } + + return nil +} + +// discoverEndpoint reads the deployment's own configuration for one database. +// It runs only on the external path, and only once per database per run. +// +// Its final arm is unreachable through resolveEndpointWith, which is the only +// caller and which refuses anything that is not one of the two databases before +// issuing a query. It reports that rather than returning nil all the same: a +// nil here says "the deployment's own settings were read", and the one thing +// that could ever reach this arm is a third database arriving, whose settings +// would then be silently unread while the endpoint claimed to describe it. Its +// sibling applyDiscoveredFacts already draws the line here. +func (iops *InfrahubOps) discoverEndpoint(service string) error { + switch service { + case serviceNeo4j: + return iops.ensureNeo4jDiscovery() + case serviceTaskManagerDB: + return iops.ensurePostgresDiscovery() + default: + return errNotADatabaseService("read the settings from the deployment for", service) + } +} + +// applyDiscoveredFacts fills an external endpoint from the two sources that +// describe it: the operator's overrides on Configuration, and what the +// deployment's own components reported. The override wins wherever both speak, +// which is the reason the two are recorded separately in the first place. +// +// discoveryErr is why the second source is empty, when it is. It is carried in +// rather than looked up because only these functions know whether it mattered: +// an override supplies the address and the failure is a warning, while nothing +// at all makes it the whole reason the run cannot proceed (see +// undiscoverableEndpoint and unreadableDeploymentSettings). +func (iops *InfrahubOps) applyDiscoveredFacts(endpoint *DatabaseEndpoint, discoveryErr error) error { + switch endpoint.Service { + case serviceNeo4j: + return iops.applyNeo4jFacts(endpoint, discoveryErr) + case serviceTaskManagerDB: + return iops.applyPostgresFacts(endpoint, discoveryErr) + default: + return errNotADatabaseService("resolve a database endpoint for", endpoint.Service) + } +} + +// applyNeo4jFacts resolves the external Neo4j endpoint. The client port, +// protocol and TLS settings are the deployment's own, reconciled with the +// operator's overrides when chunk-2a's discovery read them; the backup port is +// the one detail no deployment records (research R4). +func (iops *InfrahubOps) applyNeo4jFacts(endpoint *DatabaseEndpoint, discoveryErr error) error { + endpoint.ClientPort = iops.discoveredNeo4j.ClientPort + endpoint.BackupPort = resolveNeo4jBackupPort(iops.config.ExternalDB) + endpoint.Protocol = iops.config.ExternalDB.Neo4jProtocol + endpoint.TLS = iops.config.ExternalDB.Neo4jTLS + endpoint.Database = iops.config.Neo4jDatabase + + address := cmp.Or(iops.config.ExternalDB.Neo4jAddress, iops.discoveredNeo4j.Address) + if address == "" { + return noAddressFor(serviceNeo4j, componentInfrahubServer, + []string{neo4jAddressEnvVar, neo4jPortEnvVar}, neo4jAddressFlag, discoveryErr) + } + + return endpoint.resolveHosts(address) +} + +// applyPostgresFacts resolves the external task-manager endpoint. It carries no +// protocol or TLS settings because the deployment records none for PostgreSQL — +// its host, port and database name all come from the Prefect connection URL. +func (iops *InfrahubOps) applyPostgresFacts(endpoint *DatabaseEndpoint, discoveryErr error) error { + endpoint.ClientPort = iops.discoveredPostgres.ClientPort + endpoint.Database = iops.config.PostgresDatabase + + address := cmp.Or(iops.config.ExternalDB.PostgresAddress, iops.discoveredPostgres.Address) + if address == "" { + return noAddressFor(serviceTaskManagerDB, componentTaskManager, + prefectConnectionEnvVars, postgresAddressFlag, discoveryErr) + } + + return endpoint.resolveHosts(address) +} + +// noAddressFor is the one place the two reasons a run has no address for an +// external database are told apart. +// +// Both arms are reached with the location positively established — the database +// demonstrably does not run in this deployment — and with nothing to address it +// by. What differs is whether the deployment was asked and answered: +// +// - it answered, and none of its settings records an address: FR-004, and +// undiscoverableEndpoint says so. +// - it could not be asked at all — `pods/exec` denied, the component's own +// pod gone, the control bound expired: FR-019, and asserting the settings +// "were read for it" would be a claim about the customer's configuration +// made on evidence this run never obtained. That conflation is what caused +// this feature's original reset, and it is the same one +// KubernetesBackend.undeterminedLocation exists to prevent one question +// earlier. +// +// The two have different remedies — set the address, versus restore the +// permission — so which one an operator reads decides where they spend the +// outage. +func noAddressFor(service, component string, settings []string, flag string, discoveryErr error) error { + if discoveryErr != nil { + return unreadableDeploymentSettings(service, component, settings, flag, discoveryErr) + } + + return undiscoverableEndpoint(service, component, settings, flag) +} + +// errDeploymentSettingsUnreadable is FR-019's refusal at this point, as a value +// callers and tests can identify rather than a message they have to match. It is +// deliberately a different value from errEndpointUndiscoverable: a caller that +// treats "not configured" and "could not be read" alike is the bug this +// separation exists for. +var errDeploymentSettingsUnreadable = errors.New("the deployment's own settings for it could not be read, so whether one records an address is unknown") + +// unreadableDeploymentSettings is what an operator reads when a database is +// demonstrably not in the deployment, nothing was supplied for it, and the +// deployment could not be asked what it uses. +// +// It states the three elements the failure-message contract requires against +// that condition: what happened (the component could not be read, with the +// cluster's own reason attached), the resource (the settings that would have +// answered, and where they live) and the action — which is two, in the order an +// operator can act on them: supply the address, which unblocks the run now, or +// restore the ability to read the component, which is what the tool was relying +// on. +func unreadableDeploymentSettings(service, component string, settings []string, flag string, err error) error { + return fmt.Errorf( + "the %s database does not run in this deployment and %w: %s would have been read from the %s component, and it could not be read (%v). "+ + "Supply the endpoint with --%s, or set %s for an unattended run, to proceed without reading it; "+ + "otherwise restore this run's access to that component — this is not a report that the setting is unset. %s", + service, errDeploymentSettingsUnreadable, + strings.Join(settings, ", "), component, err, + flag, flagEnvVar(flag), + externalMisdiagnosisHint(service)) +} + +// errEndpointUndiscoverable is FR-004's refusal, as a value callers and tests +// can identify rather than a message they have to match. +var errEndpointUndiscoverable = errors.New("no address for it could be discovered from the deployment's own configuration") + +// undiscoverableEndpoint is what an operator reads when a database is +// demonstrably not in the deployment and nothing records where it is. +// +// The three elements the failure-message contract requires are the condition +// (discovery ran and found nothing), the resource (the settings that were read, +// named, in the component they were read out of) and the action (the override +// flag, plus its configuration key, which is the only channel an unattended run +// has). Stating where discovery looked is the part that was missing: "supply +// --neo4j-address" on its own leaves an operator whose deployment *does* export +// the address unable to tell a setting they never set from a component this run +// could not read, and those two have different remedies. +// +// It is deliberately distinct from the refusal a denied deployment query +// produces (see KubernetesBackend.undeterminedLocation). This one is reached +// only after the location was positively established, so it says the deployment +// was asked and answered; conflating the two is what once reported an RBAC +// failure as a database nobody could find. +// +// "Asked and answered" is now a property of the call rather than of the comment: +// a run whose discovery *failed* reaches unreadableDeploymentSettings instead, +// because this message's central claim — that the settings were read — was +// false for it, and noAddressFor is what routes between them. +func undiscoverableEndpoint(service, component string, settings []string, flag string) error { + return fmt.Errorf( + "the %s database does not run in this deployment and %w: %s in the %s component were read for it, and none of them records an address. "+ + "Supply the endpoint with --%s, or set %s for an unattended run. %s", + service, errEndpointUndiscoverable, + strings.Join(settings, ", "), component, + flag, flagEnvVar(flag), + externalMisdiagnosisHint(service)) +} + +// flagEnvVar is the configuration key a flag also reads from, under the binding +// convention ConfigureRootCommand sets up: the INFRAHUB_ prefix, with dashes +// replaced by underscores. Derived rather than written out beside each flag, so +// a message naming both cannot state a key viper does not bind. +func flagEnvVar(flag string) string { + return "INFRAHUB_" + strings.ToUpper(strings.ReplaceAll(flag, "-", "_")) +} + +// The tooling that will read a database and the server it will read are two +// versions the run has to reconcile before it moves any data (FR-006). That +// reconciliation cannot be one numeric comparison, because Neo4j numbers its +// releases two different ways — 5.x.y up to the 5.26 LTS, and YYYY.MM.p since — +// and which of the two a version belongs to is a fact about the vendor's release +// timeline rather than something the digits state. So the scheme is classified +// first and the comparison happens within it. +// +// The repository's other version comparison, in internal/updater, is strict +// semver over this tool's own release tags. It is deliberately not reused here: +// it requires a leading "v", has no notion of calendar versioning, and would +// read the "-enterprise" of an image tag as a pre-release — see +// prereleaseMarkers for why that particular reading would abort working runs. + +// versionScheme is the numbering scheme a database version belongs to. +type versionScheme int + +const ( + // versionSchemeSequential is the traditional major[.minor[.patch]] form: + // Neo4j 5.26.1, PostgreSQL 18.1, or a bare major series. + versionSchemeSequential versionScheme = iota + + // versionSchemeCalendar is the YYYY.MM[.p] form Neo4j has published since + // 2025. + versionSchemeCalendar +) + +// calendarYearFloor is the first-field value at or above which a version reads +// as a calendar year rather than a major series. Nothing in a version string +// declares its scheme, so the classification rests on the one property the two +// schemes cannot share: no database this tool supports numbers a major series in +// the thousands, and every calendar release the vendor has published carries a +// four-digit year. +const calendarYearFloor = 2000 + +// prereleaseMarkers are the qualifiers that mean "these numbers, before their +// finished release". +// +// They are matched by name rather than by "the version carries a qualifier at +// all", because the qualifiers this project meets most are -enterprise, +// -community and -alpine, which say which build of a release an image carries +// rather than that it precedes one. Ranking those below the bare numbers, the +// way semver ranks any pre-release, would make neo4j:2025.10.1-enterprise +// tooling — the build the infrahub-backup image copies its Neo4j tools from — +// older than a 2025.10.1 server and abort a run that would have worked. +var prereleaseMarkers = []string{"alpha", "beta", "rc", "preview", "snapshot"} + +// databaseVersion is a database version parsed far enough to be ordered. +type databaseVersion struct { + // Original is the version as it was read, so that a message about it quotes + // what the server or the operator actually said rather than a normalised + // rendering of it. + Original string + + // scheme is which numbering scheme the version belongs to, which is what + // orders two versions drawn from different ones. + scheme versionScheme + + // fields are the numeric components in the order they were written, only as + // many as the version states. They are deliberately not padded to three: a + // version that stops at its major states nothing about a minor, and reading + // the absent field as zero is what would make a postgres:18-alpine image's + // tooling look older than the 18.1 server it reads perfectly well. + fields []int + + // prerelease records that the version named itself one of + // prereleaseMarkers. + prerelease bool +} + +// String renders the version as the string it was read from, so it can be +// dropped into a message with %s. +func (v databaseVersion) String() string { + return v.Original +} + +// parseDatabaseVersion reads a version string into an orderable value. +// +// It is tolerant of the shapes these servers and their images actually produce — +// a build qualifier (5.26.1-enterprise, 18-alpine) and the packaging detail +// PostgreSQL appends to its own server_version ("18.1 (Debian +// 18.1-1.pgdg120+1)") — and refuses everything else rather than guessing at it. +// The gate below turns that refusal into an abort: a version that cannot be +// ordered is not evidence of compatibility. +func parseDatabaseVersion(text string) (databaseVersion, error) { + original := strings.TrimSpace(text) + if original == "" { + return databaseVersion{}, errors.New("version was not reported") + } + + core, qualifier := splitVersionQualifier(original) + + fields, err := parseVersionFields(core) + if err != nil { + return databaseVersion{}, fmt.Errorf("version %q cannot be read: %w", original, err) + } + + return databaseVersion{ + Original: original, + scheme: classifyVersionScheme(fields), + fields: fields, + prerelease: isPrereleaseQualifier(qualifier), + }, nil +} + +// splitVersionQualifier separates the numeric core of a version from whatever +// follows it. The core ends at the first hyphen, plus or whitespace, whichever +// comes first: those are the separators that start a build qualifier, a build +// metadata suffix, and the packaging detail a server appends to its version. +func splitVersionQualifier(version string) (core, qualifier string) { + if index := strings.IndexAny(version, "-+ \t"); index >= 0 { + return version[:index], version[index+1:] + } + + return version, "" +} + +// parseVersionFields reads the dot-separated numeric components of a version +// core, keeping exactly as many as were written. An empty or non-numeric +// component is an error rather than a zero, because a version this tool cannot +// read is one it must not pretend to have compared. +func parseVersionFields(core string) ([]int, error) { + if core == "" { + return nil, errors.New("it states no version number") + } + + parts := strings.Split(core, ".") + fields := make([]int, 0, len(parts)) + for _, part := range parts { + value, err := strconv.Atoi(part) + if err != nil { + return nil, fmt.Errorf("component %q is not a number", part) + } + + fields = append(fields, value) + } + + return fields, nil +} + +// classifyVersionScheme decides which scheme a parsed version belongs to. It is +// a step of its own because that answer, rather than the digits, is what orders +// two versions from different schemes. +func classifyVersionScheme(fields []int) versionScheme { + if len(fields) > 0 && fields[0] >= calendarYearFloor { + return versionSchemeCalendar + } + + return versionSchemeSequential +} + +// isPrereleaseQualifier reports whether a version's qualifier names it a +// pre-release. The match is on the qualifier's opening text so that alpha09, rc1 +// and beta.2 are recognised alongside the bare markers. +func isPrereleaseQualifier(qualifier string) bool { + qualifier = strings.ToLower(strings.TrimSpace(qualifier)) + for _, marker := range prereleaseMarkers { + if strings.HasPrefix(qualifier, marker) { + return true + } + } + + return false +} + +// majorOnly returns v reduced to its leading numeric component, so that two +// versions compare on their major alone. The Original is kept, because it is +// what a message about the version has to quote; the scheme is unaffected, +// since it is decided by that same leading field. +func (v databaseVersion) majorOnly() databaseVersion { + if len(v.fields) <= 1 { + return v + } + + reduced := v + reduced.fields = v.fields[:1] + + return reduced +} + +// compare orders v against other: negative when v is the older of the two, zero +// when neither is older, positive when v is the newer. +func (v databaseVersion) compare(other databaseVersion) int { + // The scheme decides first. A calendar release is later than any 5.x one + // because the vendor moved from the one to the other; that 2025 also happens + // to exceed 5 is a coincidence of the digits, and relying on it would leave + // the ordering resting on an accident — while comparing the same two as text + // orders them backwards outright. + if v.scheme != other.scheme { + if v.scheme == versionSchemeCalendar { + return 1 + } + + return -1 + } + + // Within a scheme, only the fields both versions state are compared. An + // absent field is unstated rather than zero — see databaseVersion.fields. + for i := 0; i < len(v.fields) && i < len(other.fields); i++ { + switch { + case v.fields[i] < other.fields[i]: + return -1 + case v.fields[i] > other.fields[i]: + return 1 + } + } + + // The same numbers to the precision both state, so the only thing left that + // can separate them is one being a pre-release of those numbers. Two + // pre-releases of the same numbers are not ordered against each other: the + // markers carry no documented sequence, and inventing one would put a guess + // inside an abort condition. + switch { + case v.prerelease && !other.prerelease: + return -1 + case other.prerelease && !v.prerelease: + return 1 + default: + return 0 + } +} + +// checkVersionCompatibility is the FR-006 gate, and it runs before any data is +// read or written: the utility that will handle a database must not be older +// than the server it handles. +// +// The rule is asymmetric on purpose. An older utility against a newer server is +// the direction with a known failure mode — pg_dump refuses it outright, and +// nothing documents Neo4j's backup client tolerating it — so that direction +// aborts, naming both versions so the operator can see which end to move. The +// other direction is documented as tolerated for restore and merely +// undocumented for backup, so it warns and continues (research R3): enforcing +// the stricter symmetric rule would encode a guess in an abort condition and +// decline work this tool can do. +// +// The precision of the comparison differs by service, because the two servers +// state different rules. PostgreSQL's is major-only and written down: pg_dump +// sets maxRemoteVersion to (PG_VERSION_NUM / 100) * 100 + 99, so an 18.1 client +// reads every 18.x server, and comparing the patch would abort on a +// provider-patched 18.2 while naming an image flag that cannot fix it — the +// official images publish only the current patch and the bare major, so no +// 18.2-alpine exists to move to. Neo4j publishes no equivalent rule, so its arm +// compares every field both versions state; loosening it would be a guess, and +// the tolerated direction only warns anyway. +// +// The decision is pure, over the two version strings. Reading the server's +// version needs a client that exists only once the transient workload is +// running, so the caller supplies both — the server version from the probe it +// runs there, and the utility version from the tooling in that same image. +func checkVersionCompatibility(service, utilityVersion, serverVersion string) error { + // A version neither side can order is not a compatible one. FR-006 requires + // the comparison to have happened before data moves, and a comparison that + // could not be made has not happened, so an unreadable version on either + // side aborts rather than passing the gate. + utility, err := parseDatabaseVersion(utilityVersion) + if err != nil { + return fmt.Errorf("cannot compare the %s utility and server versions before moving data (the server reported %q): the utility's %w", service, serverVersion, err) + } + + server, err := parseDatabaseVersion(serverVersion) + if err != nil { + return fmt.Errorf("cannot compare the %s utility and server versions before moving data (the utility reported %q): the server's %w", service, utilityVersion, err) + } + + // The versions in the messages stay as they were reported: the operator has + // to see the versions their deployment states, not the truncation this + // comparison happens to make of them. + switch ordering := comparedAt(service, utility).compare(comparedAt(service, server)); { + case ordering < 0: + // The image carries this tool's pinned tooling, so the usual remedy is a + // newer release of this tool; the flag is the remedy when the operator + // builds or mirrors the image themselves. + return fmt.Errorf("the %s utility (%s) is older than the server (%s): use an infrahub-backup release whose image carries tooling at least as new as the server, or supply %s with an image whose tooling is at least as new as the server", service, utility, server, externalDBImageOption) + case ordering > 0: + logrus.Warnf("The %s utility (%s) is newer than the server (%s); continuing, since no documented rule forbids that direction", service, utility, server) + default: + logrus.Infof("The %s utility (%s) is compatible with the server (%s)", service, utility, server) + } + + return nil +} + +// comparedAt reduces a version to the precision its service's own compatibility +// rule is written at. PostgreSQL's is major-only; every other service is +// compared at the full precision both versions state. +func comparedAt(service string, version databaseVersion) databaseVersion { + if service == serviceTaskManagerDB { + return version.majorOnly() + } + + return version +} + +// externalDBImageOption is the flag that selects the transient workload's image, +// as an operator types it, so an abort tells the operator which end to move +// rather than only that the two versions disagree. There is one for both +// databases, because both run the same image (FR-031). +const externalDBImageOption = "--" + ExternalDBImageFlag + +// --------------------------------------------------------------------------- +// Certificate verification (FR-029) +// --------------------------------------------------------------------------- + +// The deployment records whether Infrahub verifies its own database server's +// certificate. That is a statement about Infrahub, not a licence for this tool, +// and the difference is the whole of FR-029: this tool sends database +// credentials of its own, so the decision to stop checking who it is sending +// them to has to be made about this tool, explicitly, by the operator. +// +// So a discovered tls_insecure never disables verification here. It is reported +// — silence would be worse, since an operator who set it probably expects it to +// apply — together with the flag that does disable verification, so the remedy +// is in the same line as the observation (SC-008). + +// insecureTLSFlagName is the only opt-out from protecting a connection to an +// external database — from verifying the certificate, and from encrypting the +// channel at all — as cobra registers it. insecureTLSFlag is the same flag as a +// message names it, derived rather than restated so the flag an operator is +// told to supply and the flag that exists cannot drift apart. +const ( + insecureTLSFlagName = "external-db-insecure-tls" + insecureTLSFlag = "--" + insecureTLSFlagName +) + +// neo4jSystemDatabase is the database every probe statement runs against. It is +// the one database a server always has, so a probe does not depend on the +// database being captured being online at the moment it runs. +const neo4jSystemDatabase = "system" + +// roleUnknown is a real observed role rather than the absence of one: a member +// the server did not report a role for is distinguishable from a capture that +// never observed roles at all (contracts/artifact-and-metadata.md). +const roleUnknown = "unknown" + +// externalTLSDecision is what this tool will do about TLS on the connections it +// opens to one external database. It is resolved once per database per run and +// passed to every command builder, so no builder decides it for itself. +type externalTLSDecision struct { + // Service is the database the decision applies to. + Service string + + // Encrypt says the channel will be encrypted at all. It is true by default + // for both services: for PostgreSQL because libpq negotiates TLS and the + // mode below is what decides the outcome, and for Neo4j because the + // connection carries this tool's own database password out of the cluster + // and an unencrypted channel is not something a deployment's setting gets + // to choose on the operator's behalf. + // + // It used to follow the deployment's own tls_enabled for Neo4j, on the + // reasoning that a server not speaking TLS on its client port cannot be + // made to by this tool asking. That is true and it is not the question: the + // setting is usually absent rather than false — a deployment that leaves it + // at Infrahub's default exports nothing — so the common case sent the + // password in clear with no flag able to prevent it, in a file that argues + // at length that a discovered tls_insecure must be read and never obeyed. + // The same reasoning applies a step further along: a channel that is not + // encrypted verifies nothing, so FR-029's "unless the operator explicitly + // opts out" has to cover it too. It now does, through OptedOut below, which + // is the one input that can turn encryption off. + Encrypt bool + + // OptedOut records that the operator supplied insecureTLSFlag, which is + // distinct from verify() being false for want of an encrypted channel. + OptedOut bool + + // deploymentInsecure records that the deployment configures insecure TLS + // for Infrahub's own connections. It exists to be reported, never to + // decide. + deploymentInsecure bool +} + +// resolveExternalTLS decides TLS for one external database. +// +// The default is a verified, encrypted channel, and the operator's own opt-out +// is the only thing that changes it. What the opt-out then gets is the weakest +// channel the deployment's own settings describe: where the deployment says it +// speaks TLS, an encrypted channel with the certificate unverified; where it +// says it does not — or says nothing, which is the same observation — an +// unencrypted one. So a deployment whose external Neo4j has no TLS at all is +// still reachable, but reaching it takes the operator saying so, rather than an +// absent environment variable deciding that this tool's password may travel in +// clear. +func resolveExternalTLS(service string, cfg ExternalDBConfig, tls ExternalDBTLS) externalTLSDecision { + return externalTLSDecision{ + Service: service, + Encrypt: service == serviceTaskManagerDB || !cfg.InsecureTLS || tls.Enabled, + OptedOut: cfg.InsecureTLS, + deploymentInsecure: tls.Insecure, + } +} + +// verify says whether the server's certificate will be verified. It is the +// default, and only the operator's own opt-out clears it — the same opt-out +// being the only thing that can clear Encrypt, so an unencrypted channel +// verifies nothing without a second field having to record it. +func (d externalTLSDecision) verify() bool { + return d.Encrypt && !d.OptedOut +} + +// Report states the decision before any connection is opened, for the same +// reason DatabaseEndpoint.Report exists: a handshake that then fails should be +// preceded by what this run was going to do about certificates. +func (d externalTLSDecision) Report() { + switch { + case !d.Encrypt: + logrus.Warnf("Opening unencrypted connections to the external %s database: %s was supplied and the deployment records TLS as disabled for it, so the credentials this run sends travel in clear and there is no certificate to verify", d.Service, insecureTLSFlag) + case d.OptedOut: + logrus.Warnf("Not verifying the external %s server's certificate because %s was supplied", d.Service, insecureTLSFlag) + default: + logrus.Infof("Verifying the external %s server's certificate", d.Service) + } + + // Reported whenever the deployment said insecure and this tool did not + // follow it, which is the case FR-029 exists for. + if d.deploymentInsecure && !d.OptedOut { + logrus.Warnf("The deployment configures insecure TLS for Infrahub's own %s connections; this run still verifies the server's certificate, because that setting was not made about the credentials this tool sends. Supply %s to opt out for this tool as well", d.Service, insecureTLSFlag) + } +} + +// boltScheme is the URI scheme the Bolt client is pointed at, which is where the +// decision above becomes an actual verification behaviour: +s verifies the +// certificate, +ssc accepts a self-signed one, and the bare scheme is +// unencrypted. +func (d externalTLSDecision) boltScheme(protocol string) string { + base := protocol + if base == "" { + base = defaultNeo4jProtocol + } + + switch { + case !d.Encrypt: + return base + case d.OptedOut: + return base + "+ssc" + default: + return base + "+s" + } +} + +// postgresSSLMode is the same decision for libpq. verify-full is the mode that +// checks both the chain and the host name; require encrypts and checks neither, +// which is what the opt-out asks for. +func (d externalTLSDecision) postgresSSLMode() string { + if d.OptedOut { + return "require" + } + + return "verify-full" +} + +// --------------------------------------------------------------------------- +// Addressing an external database (FR-005) +// --------------------------------------------------------------------------- + +// captureTargets is the endpoint's members addressed on the port the capture +// will actually travel over, in the order they will be tried (FR-005). +// +// The port depends on the service, and stamping one service's port onto the +// other is not a cosmetic error. Neo4j's online backup speaks to a listener of +// its own, whose port no deployment records (research R4) and which +// --neo4j-backup-port is the only source of; a member's own port, read from +// Infrahub's address setting, is a *client* port and is therefore never the +// right answer for that list. PostgreSQL has no second listener at all — the +// dump travels over the client port each member already carries — so giving it +// the Neo4j backup port would record :6362 in provenance metadata for a dump +// that went over :5432, which is a false statement about where the data came +// from rather than a wrong flag value. +func (e DatabaseEndpoint) captureTargets() []HostPort { + if e.Service != serviceNeo4j { + return append([]HostPort(nil), e.Hosts...) + } + + port := e.BackupPort + if port <= 0 { + port = defaultNeo4jBackupPort + } + + targets := make([]HostPort, 0, len(e.Hosts)) + for _, host := range e.Hosts { + targets = append(targets, HostPort{Host: host.Host, Port: port}) + } + + return targets +} + +// renderHostPortList is members in try order, ports explicit, one string each. +// It is the form provenance metadata records, where the order is the fact being +// recorded and joining it away would lose it. +func renderHostPortList(hosts []HostPort) []string { + if len(hosts) == 0 { + return nil + } + + rendered := make([]string, 0, len(hosts)) + for _, host := range hosts { + rendered = append(rendered, host.String()) + } + + return rendered +} + +// renderHostPorts joins members the way both --from and an operator-facing +// report want them: in try order, comma-separated, ports explicit. +func renderHostPorts(hosts []HostPort) string { + return strings.Join(renderHostPortList(hosts), ",") +} + +// boltURIs is one client URI per member, in try order. There is one per member +// rather than one for the endpoint because a probe against an unresponsive +// member should fall through to the next exactly as the capture's own --from +// list does, and a single URI would make the first member's health decide the +// run. +func (e DatabaseEndpoint) boltURIs(d externalTLSDecision) []string { + scheme := d.boltScheme(e.effectiveProtocol()) + + uris := make([]string, 0, len(e.Hosts)) + for _, host := range e.Hosts { + uris = append(uris, scheme+"://"+host.String()) + } + + return uris +} + +// safeDatabaseIdentifier refuses a database name that carries anything but a +// name before it is interpolated into a statement. +// +// Infrahub pattern-validates the name it sets, so this is a belt rather than a +// braces: the value reaches here from a component's environment, and a name +// that could close a quote and continue the statement must not be the thing +// standing between a discovered value and a Cypher session. +func safeDatabaseIdentifier(name string) (string, error) { + trimmed := strings.TrimSpace(name) + if trimmed == "" { + return "", errors.New("no database name was resolved") + } + + for _, r := range trimmed { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '-', r == '_': + default: + return "", fmt.Errorf("database name %q contains %q, which a database name may not carry", trimmed, string(r)) + } + } + + return trimmed, nil +} + +// --------------------------------------------------------------------------- +// Probe queries (FR-006, FR-008, FR-022, FR-005) +// --------------------------------------------------------------------------- + +// Four facts have to be read from an external server before its capture can be +// built, and none of them can be read from outside the deployment: the server's +// version and the utility's, which are the FR-006 gate; the edition, which +// decides whether a remote capture is possible at all (FR-008); and the store +// size, which sizes the capture's scratch space (FR-022). The commands are +// built here as pure values so that what is sent to a customer's database is +// assertable without one, and so that no credential can drift onto one of them +// (FR-014). +// +// The credentials are conspicuously absent from every command below. The +// transient workload mounts them as environment from a secret it owns, under the +// names both clients read of their own accord — NEO4J_USERNAME/NEO4J_PASSWORD +// and PGUSER/PGPASSWORD — so authentication needs no argument and no ExecOptions +// environment, which is the exec path that would put them on a command line. + +// cypherShellCommand builds the argv for one statement against an external Neo4j +// server. --format plain keeps the output to a header and its rows, which is +// what the parsers below read. +func cypherShellCommand(uri, database, statement string) []string { + return []string{ + "cypher-shell", + "--non-interactive", + "--format", "plain", + "-a", uri, + "-d", database, + statement, + } +} + +// neo4jComponentsStatement reads the server's version and edition in one +// round trip. +// +// They are asked for together rather than separately because they are read from +// the same call, they are needed at the same moment, and the probe pod's +// deadline is budgeted from the number of calls the probe issues +// (externalDBProbeDeadline). Splitting them would spend a whole extra bound to +// learn a second field of a result already in hand. +const neo4jComponentsStatement = "CALL dbms.components() YIELD versions, edition RETURN versions[0] AS " + + neo4jVersionColumn + ", edition AS " + neo4jEditionColumn + +// The column names the statements above and below ask for, which are also the +// names cypher-shell prints as the result's header and therefore what the +// parsers locate that header by. +// +// They are constants shared by the statement and its parser because that is +// the only thing that keeps the two in step: a statement whose column is +// renamed and a parser still looking for the old name do not fail, they find no +// header, and "the server answered nothing" is a wrong answer that reads like a +// server problem. +const ( + neo4jVersionColumn = "version" + neo4jEditionColumn = "edition" + + // The store size is returned as the expression itself rather than under an + // alias, so the header carries the expression text. + neo4jStoreSizeColumn = "attributes.Value.value" + + neo4jMemberAddressColumn = "address" + neo4jMemberRoleColumn = "role" + neo4jMemberWriterColumn = "writer" +) + +// neo4jComponentsCommand runs the statement above. It runs against the system +// database because that is the one database every server has, and the target +// database may be offline at the moment the version is read. +func neo4jComponentsCommand(uri string) []string { + return cypherShellCommand(uri, neo4jSystemDatabase, neo4jComponentsStatement) +} + +// neo4jUtilityVersionCommand reads the version of the tooling that will perform +// the capture, which is the other half of the FR-006 comparison. It is read from +// the image rather than assumed from the image's tag, because a tag is a label +// an operator chose and the version is a fact about what is installed. +func neo4jUtilityVersionCommand() []string { + return []string{"neo4j-admin", "--version"} +} + +// neo4jStoreSizeStatement asks for the store size through the server's own +// metrics, which is the only interface that reports it to a client with no +// access to the store's filesystem. +// +// The store-size metric is not part of SHOW DATABASES — that clause reports the +// store *format*, not its size — so this reads the metric bean instead. Where +// the metric is not exposed, the query answers nothing and sizing falls to +// FR-022's closing clause: the run fails naming --external-db-scratch-size +// rather than guessing a size. That fallback is the reason this query is allowed +// to be the one part of the probe whose availability depends on the server's +// configuration. +func neo4jStoreSizeStatement(database string) string { + return fmt.Sprintf(`CALL dbms.queryJmx("neo4j.metrics:name=neo4j.database.%s.store.size.total") YIELD attributes RETURN %s`, database, neo4jStoreSizeColumn) +} + +// neo4jStoreSizeCommand runs the statement above against the system database. +func neo4jStoreSizeCommand(uri, database string) ([]string, error) { + name, err := safeDatabaseIdentifier(database) + if err != nil { + return nil, fmt.Errorf("cannot query the store size of the external %s database: %w", serviceNeo4j, err) + } + + return cypherShellCommand(uri, neo4jSystemDatabase, neo4jStoreSizeStatement(name)), nil +} + +// neo4jMemberRolesCommand asks the server which of its members hosts the +// database being captured and in what role (FR-005). +// +// It yields the address, the role and the writer flag and nothing else. It +// deliberately does not ask, and there is nothing to ask, which member will +// serve the capture: --from tries its list in order and reports no such thing, +// so a field asserting it would be a guess presented as provenance. +func neo4jMemberRolesCommand(uri, database string) ([]string, error) { + name, err := safeDatabaseIdentifier(database) + if err != nil { + return nil, fmt.Errorf("cannot query the cluster members of the external %s database: %w", serviceNeo4j, err) + } + + statement := fmt.Sprintf("SHOW DATABASE %s YIELD %s, %s, %s", name, + neo4jMemberAddressColumn, neo4jMemberRoleColumn, neo4jMemberWriterColumn) + + return cypherShellCommand(uri, neo4jSystemDatabase, statement), nil +} + +// psqlCommand builds the argv for one statement against an external PostgreSQL +// server. -t -A -q keeps the output to the value alone, with no header, no +// alignment and no status line. +func psqlCommand(host HostPort, database, statement string) []string { + return []string{ + "psql", + "-tAq", + "-h", host.Host, + "-p", strconv.Itoa(host.Port), + "-d", database, + "-c", statement, + } +} + +// postgresServerVersionCommand and postgresStoreSizeCommand are the PostgreSQL +// halves of the same two questions. pg_database_size is the server's own +// accounting of the database's size, in bytes. +func postgresServerVersionCommand(host HostPort, database string) []string { + return psqlCommand(host, database, "SHOW server_version") +} + +func postgresStoreSizeCommand(host HostPort, database string) []string { + return psqlCommand(host, database, "SELECT pg_database_size(current_database())") +} + +// postgresUtilityVersionCommand reads the version of the client that will take +// the dump. pg_dump is the utility whose version FR-006 compares, not psql. +func postgresUtilityVersionCommand() []string { + return []string{"pg_dump", "--version"} +} + +// --------------------------------------------------------------------------- +// Probe parsing +// --------------------------------------------------------------------------- + +// extractVersionToken pulls the version out of what a utility or a server says +// when asked for it. The shapes differ — "pg_dump (PostgreSQL) 18.1", +// "neo4j-admin 2025.10.1", a bare "5.26.1", and PostgreSQL's own +// "18.1 (Debian 18.1-1.pgdg120+1)" — and what they share is that the first +// token beginning with a digit is the version. +// +// Taking the *first* such token rather than the last is what keeps PostgreSQL's +// parenthesised packaging detail from being read as the version, which is the +// one shape where the two rules disagree. +func extractVersionToken(output string) (string, error) { + for _, field := range strings.Fields(output) { + if field == "" { + continue + } + if r := field[0]; r >= '0' && r <= '9' { + return field, nil + } + } + + return "", fmt.Errorf("no version number appears in %q", strings.TrimSpace(output)) +} + +// cypherResult locates a statement's result inside everything cypher-shell +// wrote, and returns the header's fields and the data lines that follow it. +// +// The header is found by the column names the statement asked for, not by +// position — not even by position among the non-empty lines. Skipping one +// non-empty line, which is what this did, handles the blank line the format +// sometimes emits and gets the *notice* case exactly wrong: cypher-shell writes +// deprecation and routing notices above the result, and one of them makes the +// notice the line that is skipped and the header the line returned as data. So +// a version reads as "version", an edition as "edition" and a store size as +// "attributes.Value.value" — each of which parses far enough to be acted on +// (the FR-006 comparison against a version token that is not one) rather than +// far enough to fail. Locating by name cannot shift: however many notices +// precede the result, the line naming the columns is the header. +// +// An empty result is not an error here — a metric that is not exposed returns no +// rows, and the caller distinguishes "the server answered nothing" from "the +// query failed" — so it reports no rows rather than refusing. A result whose +// header never appears is reported the same way, which is the safe direction: +// every caller turns no answer into a failure naming what it asked for, and +// none of them can be handed a value the run made up. +// +// The header comes back already indexed, so a caller reads its columns from +// the same index that recognised the line — and the columns it asked for are +// present by construction, which is what let the readers stop re-checking. +func cypherResult(output string, columns ...string) (cypherColumns, []string) { + if len(columns) == 0 { + return nil, nil + } + + lines := nonEmptyLines(output) + for index, line := range lines { + if present := indexCypherColumns(splitPlainRow(line)); present.has(columns...) { + return present, lines[index+1:] + } + } + + logrus.Debugf("No result header naming %s appears in the %d line(s) the server returned, so the statement is treated as having answered nothing", + strings.Join(columns, ", "), len(lines)) + + return nil, nil +} + +// cypherColumns is a result header indexed so that a column can be found by +// the name the statement asked for it under. +// +// It exists because the same question had three answers. Two callers indexed +// the header the same way — lowercased and trimmed — and then read it +// differently: parseNeo4jDatabaseStatuses lowercased its column constant at the +// lookup, parseNeo4jMemberRoles used its constants raw. Raw works only for as +// long as every constant is itself already lowercase, which today's `address`, +// `role` and `writer` are. It is the *next* column that breaks it, and this is +// not a hypothetical spelling: the constant read from the same kind of result +// on the sibling path is `currentStatus`. A member column named that way would +// simply not be found, parseNeo4jMemberRoles would return nil, every member's +// role would vanish from SourceRoles, and FR-005's provenance would be silently +// empty with no error anywhere — the opposite of what recording an observed +// role is for. +// +// cypherResult and parseCypherRow answered the same question a third way, with +// strings.EqualFold at the comparison. There is one answer now, and both the +// indexing and the lookup go through cypherColumnKey, so neither side can +// normalise differently from the other again. +type cypherColumns map[string]int + +// cypherColumnKey is that one normalisation. Case-insensitive and +// space-trimming, because the name a server prints under --format plain is the +// expression the statement gave it, padded to the column width. +func cypherColumnKey(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +// indexCypherColumns indexes a header by position. +func indexCypherColumns(header []string) cypherColumns { + columns := make(cypherColumns, len(header)) + for index, name := range header { + columns[cypherColumnKey(name)] = index + } + + return columns +} + +// at is the position of a column, under whatever spelling the caller asks with. +func (columns cypherColumns) at(name string) (int, bool) { + index, ok := columns[cypherColumnKey(name)] + + return index, ok +} + +// has reports whether the header carries every column asked for — which is +// what makes one line the header of a result rather than a notice above it. +func (columns cypherColumns) has(names ...string) bool { + for _, name := range names { + if _, ok := columns.at(name); !ok { + return false + } + } + + return true +} + +// parseCypherRow reads the first data row of a statement's result under +// --format plain — which prints the column names and then the rows — and +// returns it keyed by the column names the caller asked for. Naming them is +// what tells the header from anything printed above it. +// +// Keyed rather than positional for the reason parseNeo4jMemberRoles already +// reads its columns by name: a two-column result read by position records the +// edition as the version if a server ever returns them the other way round, +// and that pair feeds the FR-006 version comparison and the FR-008 edition +// gate. The keys are the caller's own spellings, so a column constant is both +// what the statement asks for and what the answer is read back under. +func parseCypherRow(output string, columns ...string) map[string]string { + present, rows := cypherResult(output, columns...) + if len(rows) == 0 { + return nil + } + + fields := splitPlainRow(rows[0]) + row := make(map[string]string, len(columns)) + for _, column := range columns { + if index, ok := present.at(column); ok && index < len(fields) { + row[column] = fields[index] + } + } + + return row +} + +// parseCypherScalar reads the single value a one-column statement returned. +func parseCypherScalar(output, column string) string { + return parseCypherRow(output, column)[column] +} + +// parseStoreSizeValue reads a byte count a server reported. A missing or +// non-numeric value is "the size could not be determined", which FR-022 turns +// into a failure naming the override rather than into a guessed size. +func parseStoreSizeValue(output string) (int64, error) { + trimmed := strings.TrimSpace(output) + if trimmed == "" || strings.EqualFold(trimmed, "null") { + return 0, errors.New("the server reported no store size") + } + + size, err := strconv.ParseInt(trimmed, 10, 64) + if err != nil { + return 0, fmt.Errorf("the server reported the store size as %q, which is not a byte count", trimmed) + } + if size <= 0 { + return 0, fmt.Errorf("the server reported the store size as %d bytes", size) + } + + return size, nil +} + +// observedMember is one member's role for the database being captured, as the +// server reported it at the moment of capture (FR-005). +type observedMember struct { + // Address is the member's client address, which is what SHOW DATABASE + // reports. It is not the backup listener's address, so matching it against + // a supplied endpoint is done on the host. + Address string + + // Role is primary, secondary, or one of the other values the server may + // report — carried through as the server worded it rather than mapped onto + // a vocabulary of this tool's own. + Role string + + // Writer records the copy that accepts writes, which is the member a reader + // of the metadata will care about most. + Writer bool +} + +// parseNeo4jMemberRoles reads the rows of `SHOW DATABASE … YIELD address, role, +// writer` under --format plain. +// +// It locates its columns from the header rather than by position. The statement +// names them itself, so position would work today; reading the header is what +// keeps a server that returns them in another order from silently recording +// every member's role as its address. +// +// The header itself is located by cypherResult, for the reason recorded there: +// taking the first non-empty line as the header put a notice in that position +// and then found neither column in it, so a cluster whose server had something +// to say reported no members at all. +func parseNeo4jMemberRoles(output string) []observedMember { + columns, rows := cypherResult(output, neo4jMemberAddressColumn, neo4jMemberRoleColumn) + if len(rows) == 0 { + return nil + } + + // The header was located by these two names, so both are present. + addressAt, _ := columns.at(neo4jMemberAddressColumn) + roleAt, _ := columns.at(neo4jMemberRoleColumn) + writerAt, hasWriter := columns.at(neo4jMemberWriterColumn) + + members := make([]observedMember, 0, len(rows)) + for _, line := range rows { + fields := splitPlainRow(line) + if addressAt >= len(fields) || roleAt >= len(fields) { + continue + } + + member := observedMember{ + Address: fields[addressAt], + Role: fields[roleAt], + } + if member.Role == "" || strings.EqualFold(member.Role, "null") { + member.Role = roleUnknown + } + if hasWriter && writerAt < len(fields) { + member.Writer = strings.EqualFold(fields[writerAt], "true") + } + + members = append(members, member) + } + + return members +} + +// splitPlainRow splits one --format plain row into its values, honouring the +// quotes the format puts around strings so that an address's own colon-and-port +// or a name containing a comma does not split a field in two. +func splitPlainRow(line string) []string { + fields := []string{} + current := strings.Builder{} + quoted := false + + for _, r := range strings.TrimSpace(line) { + switch { + case r == '"': + quoted = !quoted + case r == ',' && !quoted: + fields = append(fields, strings.TrimSpace(current.String())) + current.Reset() + default: + current.WriteRune(r) + } + } + + return append(fields, strings.TrimSpace(current.String())) +} + +// captureRoles maps every endpoint the run supplied to the role observed for it +// (FR-005, and `source_roles` in contracts/artifact-and-metadata.md). +// +// The keys are the endpoints as they were supplied, in try order, because those +// are what the run is accountable for having named. +// +// Matching takes the address whole first and falls back to the host. Whole is +// the stronger evidence and it is sometimes available — a PostgreSQL endpoint +// travels over the same port the server reports, and an operator may have +// supplied Neo4j client addresses — while the host-only fallback is what makes +// the ordinary Neo4j case work at all: a supplied endpoint there carries the +// backup listener's port and the server reports its client address, so +// comparing the two whole is comparing two different ports on one machine. +// +// The fallback is only sound while a host holds one instance, and this keys by +// host to a *set* for that reason. Two instances on one host — a colocated +// pair, or the same server reported twice — collapsed into whichever the +// listing mentioned last, so a supplied endpoint could be recorded as +// `secondary` when the run had read a primary at that host. That is the field +// FR-005 says bears on the recovery point, wrong in the artefact's own +// provenance, with nothing in the run to contradict it. Where the instances at +// a host disagree the answer is roleUnknown and the ambiguity is logged, since +// "the run could not tell which of these it named" is a fact and a coin toss +// between primary and secondary is not. Where they agree the role is recorded, +// because then it does not matter which of them the endpoint was. +// +// An endpoint no member matched gets roleUnknown, which is a value rather than +// an absence. Nothing here says which endpoint served the capture, because +// nothing observed it. +func captureRoles(targets []HostPort, members []observedMember) map[string]string { + if len(targets) == 0 { + return nil + } + + byAddress := map[string]observedMember{} + byHost := map[string][]observedMember{} + for _, member := range members { + address := strings.TrimSpace(member.Address) + byAddress[strings.ToLower(address)] = member + + host := address + if parsed, _, err := net.SplitHostPort(address); err == nil { + host = parsed + } + key := strings.ToLower(host) + byHost[key] = append(byHost[key], member) + } + + roles := make(map[string]string, len(targets)) + for _, target := range targets { + roles[target.String()] = observedRoleFor(target, byAddress, byHost) + } + + return roles +} + +// observedRoleFor is the per-endpoint half of captureRoles: the role the run +// may record for one supplied endpoint, or roleUnknown where the evidence does +// not single one out. +func observedRoleFor(target HostPort, byAddress map[string]observedMember, byHost map[string][]observedMember) string { + if member, ok := byAddress[strings.ToLower(target.String())]; ok && member.Role != "" { + return member.Role + } + + // The empty string is the "nothing chosen yet" state rather than + // roleUnknown, because roleUnknown is also a role a member can be reported + // in: parseNeo4jMemberRoles maps a null role onto it. Starting from it made + // the answer depend on the order the server happened to list the instances + // in — a null before a primary recorded `primary`, the same pair the other + // way round recorded `unknown`. + chosen := "" + for _, member := range byHost[strings.ToLower(target.Host)] { + switch { + case member.Role == "": + continue + case chosen == "": + chosen = member.Role + case !strings.EqualFold(chosen, member.Role): + logrus.Warnf("The server reports more than one instance at %s in different roles, so no role is recorded for the endpoint %s: a role attributed to the wrong instance would misstate the recovery point", target.Host, target) + + return roleUnknown + } + } + + if chosen == "" { + return roleUnknown + } + + return chosen +} + +// describeCaptureRoles renders the observed roles in try order, for the line the +// operator reads before a capture starts. It is deliberately ordered by the +// endpoint list rather than by map iteration, so two runs against the same +// cluster print the same line. +func describeCaptureRoles(targets []HostPort, roles map[string]string) string { + rendered := make([]string, 0, len(targets)) + for _, target := range targets { + role := roles[target.String()] + if role == "" { + role = roleUnknown + } + rendered = append(rendered, target.String()+"="+role) + } + + return strings.Join(rendered, ",") +} diff --git a/src/internal/app/external_endpoint_test.go b/src/internal/app/external_endpoint_test.go new file mode 100644 index 00000000..4ee177b7 --- /dev/null +++ b/src/internal/app/external_endpoint_test.go @@ -0,0 +1,1626 @@ +package app + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +// TestParseHostPortList covers the address shapes Infrahub's own INFRAHUB_DB_ADDRESS +// accepts, and the malformed ones a deployment or an operator can write. The +// inheritance rule is the one that carries the most weight: a member without a +// port takes the client port, which is how Infrahub reads the same setting. +func TestParseHostPortList(t *testing.T) { + tests := []struct { + name string + address string + defaultPort int + want []HostPort + wantErr bool + }{ + { + name: "single host inherits the client port", + address: "neo4j.example.com", + defaultPort: 7687, + want: []HostPort{{Host: "neo4j.example.com", Port: 7687}}, + }, + { + name: "host with an explicit port keeps it", + address: "neo4j.example.com:7688", + defaultPort: 7687, + want: []HostPort{{Host: "neo4j.example.com", Port: 7688}}, + }, + { + name: "mixed list preserves order and inherits per member", + address: "core-1:7688,core-2,core-3:7689", + defaultPort: 7687, + want: []HostPort{ + {Host: "core-1", Port: 7688}, + {Host: "core-2", Port: 7687}, + {Host: "core-3", Port: 7689}, + }, + }, + { + name: "whitespace around members is not part of the host", + address: " core-1 , core-2:7688 ", + defaultPort: 7687, + want: []HostPort{ + {Host: "core-1", Port: 7687}, + {Host: "core-2", Port: 7688}, + }, + }, + { + name: "IPv4 address with a port", + address: "10.0.0.7:7688", + defaultPort: 7687, + want: []HostPort{{Host: "10.0.0.7", Port: 7688}}, + }, + { + name: "bracketed IPv6 literal with a port", + address: "[::1]:7687", + defaultPort: 5432, + want: []HostPort{{Host: "::1", Port: 7687}}, + }, + { + name: "bracketed IPv6 literal without a port inherits", + address: "[fe80::1]", + defaultPort: 7687, + want: []HostPort{{Host: "fe80::1", Port: 7687}}, + }, + { + name: "bare IPv6 literal is a host, not a host:port pair", + address: "::1", + defaultPort: 7687, + want: []HostPort{{Host: "::1", Port: 7687}}, + }, + { + name: "IPv6 members mixed with a named host", + address: "[2001:db8::1]:7688,core-2,fe80::2", + defaultPort: 7687, + want: []HostPort{ + {Host: "2001:db8::1", Port: 7688}, + {Host: "core-2", Port: 7687}, + {Host: "fe80::2", Port: 7687}, + }, + }, + { + name: "empty address configures no endpoint", + address: "", + defaultPort: 7687, + }, + { + name: "whitespace-only address configures no endpoint", + address: " ", + defaultPort: 7687, + }, + { + name: "empty member is an error rather than a silently shorter list", + address: "core-1,,core-2", + defaultPort: 7687, + wantErr: true, + }, + { + name: "trailing separator is an error", + address: "core-1,", + defaultPort: 7687, + wantErr: true, + }, + { + name: "member with no host is an error", + address: ":7687", + defaultPort: 7687, + wantErr: true, + }, + { + name: "non-numeric port is an error", + address: "core-1:bolt", + defaultPort: 7687, + wantErr: true, + }, + { + name: "empty port is an error", + address: "core-1:", + defaultPort: 7687, + wantErr: true, + }, + { + name: "out-of-range port is an error", + address: "core-1:99999", + defaultPort: 7687, + wantErr: true, + }, + { + name: "port zero is an error", + address: "core-1:0", + defaultPort: 7687, + wantErr: true, + }, + { + name: "unterminated IPv6 bracket is an error", + address: "[::1:7687", + defaultPort: 7687, + wantErr: true, + }, + { + // T113. Two colons were read as "every colon belongs to an IPv6 + // address", so a semicolon where the operator meant a comma became + // one host whose name is the whole typo — and that host reached the + // capture's --from list and the artefact's provenance metadata as a + // member this run claims to have read. + name: "a semicolon in place of a separator is an error, not a host", + address: "db-a:7687;db-b:7687", + defaultPort: 7687, + wantErr: true, + }, + { + name: "a member with two ports is an error", + address: "core-1:7687:6362", + defaultPort: 7687, + wantErr: true, + }, + { + name: "a host name with a stray colon is an error", + address: "core-1:7687,core-2:bolt:7687", + defaultPort: 7687, + wantErr: true, + }, + { + // Still a literal, and still carrying the colons that make it look + // like host:port, so the check is "is this an IP" rather than "is + // this IPv6-shaped". + name: "IPv4-mapped IPv6 literal is a host", + address: "::ffff:10.0.0.1", + defaultPort: 7687, + want: []HostPort{{Host: "::ffff:10.0.0.1", Port: 7687}}, + }, + { + // The zone names the interface the address is reachable on, and an + // address without it cannot be connected to — so it is kept. + name: "link-local literal keeps its zone identifier", + address: "fe80::1%eth0", + defaultPort: 7687, + want: []HostPort{{Host: "fe80::1%eth0", Port: 7687}}, + }, + { + name: "a bracketed host name is an error", + address: "[db-a.example.net]", + defaultPort: 7687, + wantErr: true, + }, + { + name: "a bracketed host name with a port is an error", + address: "[db-a.example.net]:7687", + defaultPort: 7687, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseHostPortList(tt.address, tt.defaultPort) + + if tt.wantErr { + if err == nil { + t.Fatalf("parseHostPortList(%q) = %v, want an error", tt.address, got) + } + if got != nil { + t.Errorf("parseHostPortList(%q) returned %v alongside the error, want nil", tt.address, got) + } + + return + } + + if err != nil { + t.Fatalf("parseHostPortList(%q) returned an unexpected error: %v", tt.address, err) + } + if len(got) != len(tt.want) { + t.Fatalf("parseHostPortList(%q) = %v, want %v", tt.address, got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("member %d = %v, want %v", i, got[i], tt.want[i]) + } + } + }) + } +} + +// TestHostPortString checks the rendered form, since it is what reaches both the +// endpoint report and the backup command's endpoint list — an unbracketed IPv6 +// literal there would be read as a host and a port. +func TestHostPortString(t *testing.T) { + tests := []struct { + name string + host HostPort + want string + }{ + {name: "named host", host: HostPort{Host: "core-1", Port: 7687}, want: "core-1:7687"}, + {name: "IPv4", host: HostPort{Host: "10.0.0.7", Port: 6362}, want: "10.0.0.7:6362"}, + {name: "IPv6 is bracketed", host: HostPort{Host: "2001:db8::1", Port: 7687}, want: "[2001:db8::1]:7687"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.host.String(); got != tt.want { + t.Errorf("String() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestDatabaseEndpointResolveHosts is the inheritance rule at the endpoint level: +// the port a member inherits is the endpoint's client port, and when the +// deployment records none it is Infrahub's own default for that service. +func TestDatabaseEndpointResolveHosts(t *testing.T) { + tests := []struct { + name string + endpoint DatabaseEndpoint + address string + want []HostPort + }{ + { + name: "members inherit the discovered client port", + endpoint: DatabaseEndpoint{Service: serviceNeo4j, ClientPort: 7688}, + address: "core-1,core-2:7690", + want: []HostPort{ + {Host: "core-1", Port: 7688}, + {Host: "core-2", Port: 7690}, + }, + }, + { + name: "Neo4j with no discovered port falls back to Bolt's default", + endpoint: DatabaseEndpoint{Service: serviceNeo4j}, + address: "core-1", + want: []HostPort{{Host: "core-1", Port: defaultNeo4jClientPort}}, + }, + { + name: "task-manager database falls back to the PostgreSQL default", + endpoint: DatabaseEndpoint{Service: serviceTaskManagerDB}, + address: "pg-1", + want: []HostPort{{Host: "pg-1", Port: defaultPostgresPort}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + endpoint := tt.endpoint + if err := endpoint.resolveHosts(tt.address); err != nil { + t.Fatalf("resolveHosts(%q) returned an unexpected error: %v", tt.address, err) + } + + if len(endpoint.Hosts) != len(tt.want) { + t.Fatalf("Hosts = %v, want %v", endpoint.Hosts, tt.want) + } + for i := range tt.want { + if endpoint.Hosts[i] != tt.want[i] { + t.Errorf("member %d = %v, want %v", i, endpoint.Hosts[i], tt.want[i]) + } + } + }) + } +} + +// TestDatabaseEndpointResolveHostsError names the service in its failure, since +// a deployment with one external database of each kind gives the operator two +// address lists to tell apart. +func TestDatabaseEndpointResolveHostsError(t *testing.T) { + endpoint := DatabaseEndpoint{Service: serviceNeo4j} + + err := endpoint.resolveHosts("core-1:bolt") + if err == nil { + t.Fatal("resolveHosts with a malformed port returned no error") + } + if !strings.Contains(err.Error(), serviceNeo4j) { + t.Errorf("error = %q, want it to name the %q service", err.Error(), serviceNeo4j) + } +} + +// TestResolveNeo4jBackupPort is the port discovery cannot supply: the vendor +// default unless the operator names another one (research R4). +func TestResolveNeo4jBackupPort(t *testing.T) { + tests := []struct { + name string + cfg ExternalDBConfig + want int + }{ + {name: "unset falls back to the vendor default", cfg: ExternalDBConfig{}, want: defaultNeo4jBackupPort}, + {name: "default carries through", cfg: ExternalDBConfig{Neo4jBackupPort: defaultNeo4jBackupPort}, want: 6362}, + {name: "flag override wins", cfg: ExternalDBConfig{Neo4jBackupPort: 6363}, want: 6363}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := resolveNeo4jBackupPort(tt.cfg); got != tt.want { + t.Errorf("resolveNeo4jBackupPort() = %d, want %d", got, tt.want) + } + }) + } +} + +// TestDatabaseEndpointDescribe checks the reported line states the values the run +// will use — including the ports and protocol a deployment left at their +// defaults, which are absent from its environment but not from the connection. +func TestDatabaseEndpointDescribe(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "core-1", Port: 7687}, {Host: "2001:db8::1", Port: 7687}}, + BackupPort: defaultNeo4jBackupPort, + Database: "infrahub", + TLS: ExternalDBTLS{Enabled: true, CAFile: "/certs/ca.pem", Insecure: true}, + } + + got := endpoint.Describe() + + for _, want := range []string{ + "location=external", + "hosts=core-1:7687,[2001:db8::1]:7687", + "protocol=bolt", + "client-port=7687", + "backup-port=6362", + "database=infrahub", + "tls=enabled,ca=/certs/ca.pem,deployment-insecure", + } { + if !strings.Contains(got, want) { + t.Errorf("Describe() = %q, want it to contain %q", got, want) + } + } +} + +// TestDatabaseEndpointDescribeInternal is the internal case: no address list, and +// no protocol invented for a service that has no such setting. +func TestDatabaseEndpointDescribeInternal(t *testing.T) { + endpoint := DatabaseEndpoint{Service: serviceTaskManagerDB, Location: EndpointLocationInternal} + + got := endpoint.Describe() + + if !strings.Contains(got, "location=internal") { + t.Errorf("Describe() = %q, want it to contain %q", got, "location=internal") + } + if !strings.Contains(got, "hosts=none") { + t.Errorf("Describe() = %q, want it to contain %q", got, "hosts=none") + } + if !strings.Contains(got, "client-port=5432") { + t.Errorf("Describe() = %q, want it to contain %q", got, "client-port=5432") + } + if strings.Contains(got, "protocol=") { + t.Errorf("Describe() = %q, want no protocol for the task-manager database", got) + } + if !strings.Contains(got, "tls=disabled") { + t.Errorf("Describe() = %q, want it to contain %q", got, "tls=disabled") + } +} + +// TestDatabaseEndpointReport is FR-003's reporting half: the resolved endpoint +// reaches the log, and calling Report again — as a connection path does +// defensively — does not repeat the line. +func TestDatabaseEndpointReport(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "core-1", Port: 7687}}, + BackupPort: defaultNeo4jBackupPort, + } + + logged := captureLogrus(t, func() { + endpoint.Report() + }) + + if !strings.Contains(logged, "core-1:7687") { + t.Errorf("log = %q, want it to name the resolved host", logged) + } + if !strings.Contains(logged, serviceNeo4j) { + t.Errorf("log = %q, want it to name the %q service", logged, serviceNeo4j) + } + + repeated := captureLogrus(t, func() { + endpoint.Report() + }) + + if repeated != "" { + t.Errorf("second Report() logged %q, want nothing", repeated) + } +} + +// mixedTopologyOps is a deployment whose two databases have both already been +// discovered, so the cases below exercise resolution rather than the exec that +// feeds it. +func mixedTopologyOps() *InfrahubOps { + iops := &InfrahubOps{config: &Configuration{ + Neo4jDatabase: "infrahub", + PostgresDatabase: "prefect", + ExternalDB: ExternalDBConfig{ + Neo4jBackupPort: defaultNeo4jBackupPort, + Neo4jProtocol: "neo4j", + Neo4jTLS: ExternalDBTLS{Enabled: true}, + }, + }} + iops.discoveredNeo4j = discoveredEndpoint{Address: "core-1,core-2:7688", ClientPort: 7687, read: true} + iops.discoveredPostgres = discoveredEndpoint{Address: "pg.example.com", ClientPort: 5433, read: true} + + return iops +} + +// stubQueries answers the location question from a fixed map and records which +// databases discovery was asked about, so a case can assert that an internal +// database is never discovered for — the structural half of FR-015. +func stubQueries(locations map[string]EndpointLocation, discovered *[]string) deploymentQueries { + return deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + location, ok := locations[service] + if !ok { + return "", fmt.Errorf("no stubbed location for %q", service) + } + + return location, nil + }, + discover: func(service string) error { + *discovered = append(*discovered, service) + + return nil + }, + } +} + +// TestResolveEndpointMixedTopology is FR-001: each database is resolved on its +// own evidence, so a deployment with one database inside and one outside works +// with no additional operator input. Both combinations are covered, because a +// resolver that quietly decided once for the whole deployment would pass one of +// them. +func TestResolveEndpointMixedTopology(t *testing.T) { + t.Run("Neo4j external, task-manager internal", func(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{ + serviceNeo4j: EndpointLocationExternal, + serviceTaskManagerDB: EndpointLocationInternal, + }, &discovered) + + neo4j, err := iops.resolveEndpointWith(queries, serviceNeo4j) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceNeo4j, err) + } + if neo4j.Location != EndpointLocationExternal { + t.Errorf("%s location = %q, want %q", serviceNeo4j, neo4j.Location, EndpointLocationExternal) + } + wantHosts := []HostPort{{Host: "core-1", Port: 7687}, {Host: "core-2", Port: 7688}} + if !reflect.DeepEqual(neo4j.Hosts, wantHosts) { + t.Errorf("%s hosts = %v, want %v", serviceNeo4j, neo4j.Hosts, wantHosts) + } + if neo4j.BackupPort != defaultNeo4jBackupPort { + t.Errorf("%s backup port = %d, want %d", serviceNeo4j, neo4j.BackupPort, defaultNeo4jBackupPort) + } + if neo4j.Protocol != "neo4j" || !neo4j.TLS.Enabled || neo4j.Database != "infrahub" { + t.Errorf("%s endpoint = %+v, want the deployment's protocol, TLS and database name carried through", serviceNeo4j, neo4j) + } + + taskManager, err := iops.resolveEndpointWith(queries, serviceTaskManagerDB) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceTaskManagerDB, err) + } + if taskManager.Location != EndpointLocationInternal { + t.Errorf("%s location = %q, want %q", serviceTaskManagerDB, taskManager.Location, EndpointLocationInternal) + } + if len(taskManager.Hosts) != 0 { + t.Errorf("%s hosts = %v, want none for a database with a container", serviceTaskManagerDB, taskManager.Hosts) + } + + // The internal database is never discovered for: an all-internal + // deployment issues no query this feature added (FR-015). + if !reflect.DeepEqual(discovered, []string{serviceNeo4j}) { + t.Errorf("discovery was asked about %v, want only %v", discovered, []string{serviceNeo4j}) + } + }) + + t.Run("Neo4j internal, task-manager external", func(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{ + serviceNeo4j: EndpointLocationInternal, + serviceTaskManagerDB: EndpointLocationExternal, + }, &discovered) + + neo4j, err := iops.resolveEndpointWith(queries, serviceNeo4j) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceNeo4j, err) + } + if neo4j.Location != EndpointLocationInternal { + t.Errorf("%s location = %q, want %q", serviceNeo4j, neo4j.Location, EndpointLocationInternal) + } + if len(neo4j.Hosts) != 0 { + t.Errorf("%s hosts = %v, want none for a database with a container", serviceNeo4j, neo4j.Hosts) + } + // The address the deployment happens to record for an in-deployment + // database must not turn into an endpoint the run dials over the network. + if neo4j.BackupPort != 0 { + t.Errorf("%s backup port = %d, want none asserted for a database reached through its container", serviceNeo4j, neo4j.BackupPort) + } + + taskManager, err := iops.resolveEndpointWith(queries, serviceTaskManagerDB) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceTaskManagerDB, err) + } + if taskManager.Location != EndpointLocationExternal { + t.Errorf("%s location = %q, want %q", serviceTaskManagerDB, taskManager.Location, EndpointLocationExternal) + } + wantHosts := []HostPort{{Host: "pg.example.com", Port: 5433}} + if !reflect.DeepEqual(taskManager.Hosts, wantHosts) { + t.Errorf("%s hosts = %v, want %v", serviceTaskManagerDB, taskManager.Hosts, wantHosts) + } + if taskManager.Database != "prefect" { + t.Errorf("%s database = %q, want prefect", serviceTaskManagerDB, taskManager.Database) + } + if taskManager.Protocol != "" || taskManager.BackupPort != 0 { + t.Errorf("%s endpoint = %+v, want no protocol or backup port invented for PostgreSQL", serviceTaskManagerDB, taskManager) + } + + if !reflect.DeepEqual(discovered, []string{serviceTaskManagerDB}) { + t.Errorf("discovery was asked about %v, want only %v", discovered, []string{serviceTaskManagerDB}) + } + }) +} + +// TestResolveEndpointLocationFailure is FR-002 through the resolver: a location +// query that fails is an error, and no endpoint comes back for a caller to +// connect to. +func TestResolveEndpointLocationFailure(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := deploymentQueries{ + locate: func(string) (EndpointLocation, error) { + return "", errors.New(`pods is forbidden: cannot list resource "pods"`) + }, + discover: func(service string) error { + discovered = append(discovered, service) + + return nil + }, + } + + endpoint, err := iops.resolveEndpointWith(queries, serviceNeo4j) + if err == nil { + t.Fatal("resolveEndpointWith succeeded despite an unanswered location query, want error") + } + if endpoint != nil { + t.Errorf("endpoint = %+v, want none alongside an error", endpoint) + } + if len(discovered) != 0 { + t.Errorf("discovery ran for %v after the location query failed, want none", discovered) + } + if !strings.Contains(err.Error(), "is forbidden") { + t.Errorf("err = %v, want it to keep the reason the query failed", err) + } +} + +// TestResolveEndpointOverrideWins covers the reason the operator's overrides and +// the deployment's own facts are recorded separately: discovery must not +// overwrite what the operator asked for. +func TestResolveEndpointOverrideWins(t *testing.T) { + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{ + serviceNeo4j: EndpointLocationExternal, + serviceTaskManagerDB: EndpointLocationExternal, + }, &discovered) + + iops := mixedTopologyOps() + iops.config.ExternalDB.Neo4jAddress = "operator-1:7690,operator-2" + iops.config.ExternalDB.PostgresAddress = "operator-pg:6000" + iops.config.ExternalDB.Neo4jBackupPort = 7000 + + neo4j, err := iops.resolveEndpointWith(queries, serviceNeo4j) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceNeo4j, err) + } + wantHosts := []HostPort{{Host: "operator-1", Port: 7690}, {Host: "operator-2", Port: 7687}} + if !reflect.DeepEqual(neo4j.Hosts, wantHosts) { + t.Errorf("%s hosts = %v, want the operator's list %v", serviceNeo4j, neo4j.Hosts, wantHosts) + } + if neo4j.BackupPort != 7000 { + t.Errorf("%s backup port = %d, want the operator's 7000", serviceNeo4j, neo4j.BackupPort) + } + + taskManager, err := iops.resolveEndpointWith(queries, serviceTaskManagerDB) + if err != nil { + t.Fatalf("resolving the %s endpoint failed: %v", serviceTaskManagerDB, err) + } + wantPG := []HostPort{{Host: "operator-pg", Port: 6000}} + if !reflect.DeepEqual(taskManager.Hosts, wantPG) { + t.Errorf("%s hosts = %v, want the operator's %v", serviceTaskManagerDB, taskManager.Hosts, wantPG) + } +} + +// TestResolveEndpointExternalWithoutAddress is the case discovery cannot cover: +// the database is demonstrably not in the deployment and nothing records where +// it is. The run must stop naming the override rather than guess a host. +func TestResolveEndpointExternalWithoutAddress(t *testing.T) { + tests := []struct { + service string + wantFlag string + }{ + {service: serviceNeo4j, wantFlag: "--neo4j-address"}, + {service: serviceTaskManagerDB, wantFlag: "--postgres-address"}, + } + + for _, tt := range tests { + t.Run(tt.service, func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{tt.service: EndpointLocationExternal}, &discovered) + + _, err := iops.resolveEndpointWith(queries, tt.service) + if err == nil { + t.Fatalf("resolving the %s endpoint succeeded with no address anywhere, want error", tt.service) + } + if !strings.Contains(err.Error(), tt.wantFlag) { + t.Errorf("err = %v, want it to name %s", err, tt.wantFlag) + } + }) + } +} + +// TestResolveEndpointSeparatesUnconfiguredFromUnreadable is FR-004 against +// FR-019, which are two conditions with two remedies and were one message. +// +// Both arms have the location positively established and no address to reach +// the database by. The difference is whether the deployment answered: an +// operator who never set the address needs to set it, and an operator whose +// `pods/exec` was denied needs that back — and reads a run that asserted their +// settings "were read for it, and none of them records an address", which is a +// claim about their configuration made on evidence this run never obtained. That +// same conflation is what caused this feature's original reset, one question +// earlier (see KubernetesBackend.undeterminedLocation). +func TestResolveEndpointSeparatesUnconfiguredFromUnreadable(t *testing.T) { + denied := errors.New(`pods "infrahub-server-0" is forbidden: User "system:serviceaccount:infrahub:backup" cannot create resource "pods/exec"`) + + for _, service := range []string{serviceNeo4j, serviceTaskManagerDB} { + t.Run(service+": the settings were read and record no address", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{service: EndpointLocationExternal}, &discovered) + + _, err := iops.resolveEndpointWith(queries, service) + if !errors.Is(err, errEndpointUndiscoverable) { + t.Fatalf("err = %v, want it to wrap %v: discovery answered and nothing records an address (FR-004)", err, errEndpointUndiscoverable) + } + if errors.Is(err, errDeploymentSettingsUnreadable) { + t.Error("a deployment that answered was reported as one that could not be read") + } + }) + + t.Run(service+": the settings could not be read at all", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{service: EndpointLocationExternal}, &discovered) + queries.discover = func(string) error { return denied } + + _, err := iops.resolveEndpointWith(queries, service) + if !errors.Is(err, errDeploymentSettingsUnreadable) { + t.Fatalf("err = %v, want it to wrap %v: the deployment was never asked, so whether it records an address is unknown (FR-019)", err, errDeploymentSettingsUnreadable) + } + if errors.Is(err, errEndpointUndiscoverable) { + t.Error("an RBAC denial was reported as a deployment that records no address; the two have different remedies") + } + + // The cluster's own reason, and both actions — the flag that + // unblocks the run now, and the access the tool was relying on. + if !strings.Contains(err.Error(), "pods/exec") { + t.Errorf("err = %v, want the cluster's own reason carried through", err) + } + if !strings.Contains(err.Error(), "not a report that the setting is unset") { + t.Errorf("err = %v, want it to say outright what it is not claiming", err) + } + }) + + t.Run(service+": a supplied address survives a discovery failure", func(t *testing.T) { + // The existing tolerance, which the split must not cost: an + // operator who supplied the one thing that cannot be defaulted gets + // their run, and the failure stays a warning. + iops := &InfrahubOps{config: &Configuration{ExternalDB: ExternalDBConfig{ + Neo4jAddress: "neo4j.example:7687", + PostgresAddress: "pg.example:5432", + }}} + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{service: EndpointLocationExternal}, &discovered) + queries.discover = func(string) error { return denied } + + endpoint, err := iops.resolveEndpointWith(queries, service) + if err != nil { + t.Fatalf("resolveEndpointWith() = %v, want the supplied address to be enough", err) + } + if len(endpoint.Hosts) == 0 { + t.Error("endpoint has no hosts, want the operator's address") + } + }) + } +} + +// TestResolveEndpointUnknownService keeps resolution to the two documented +// database service names (FR-017): a typo must not resolve into an endpoint the +// run then tries to reach. +func TestResolveEndpointUnknownService(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{"cache": EndpointLocationExternal}, &discovered) + + if _, err := iops.resolveEndpointWith(queries, "cache"); err == nil { + t.Fatal("resolveEndpointWith accepted a service that is not a database, want error") + } +} + +// TestEnsureDiscoveryAsksOnce covers the widened gate's own guard. Both helpers +// exec into a deployment component, so an ops with no detected environment +// cannot complete one — which is what makes this a real assertion that a +// database already read for is not read for again. +func TestEnsureDiscoveryAsksOnce(t *testing.T) { + t.Run("Neo4j", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + iops.discoveredNeo4j.read = true + + if err := iops.ensureNeo4jDiscovery(); err != nil { + t.Errorf("ensureNeo4jDiscovery asked the deployment a second time: %v", err) + } + }) + + t.Run("task-manager", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}} + iops.discoveredPostgres.read = true + + if err := iops.ensurePostgresDiscovery(); err != nil { + t.Errorf("ensurePostgresDiscovery asked the deployment a second time: %v", err) + } + }) +} + +// TestParseDatabaseVersion covers the version shapes these two servers and their +// images produce, and the ones that cannot be ordered at all. The scheme is +// asserted alongside the fields because it, not the digits, is what orders two +// versions from different schemes; the field count is asserted because a version +// that states no patch level must not acquire a zero one. +func TestParseDatabaseVersion(t *testing.T) { + tests := []struct { + name string + version string + wantScheme versionScheme + wantFields []int + wantPrerelease bool + wantErr bool + }{ + { + name: "Neo4j 5.x release", + version: "5.26.1", + wantScheme: versionSchemeSequential, + wantFields: []int{5, 26, 1}, + }, + { + name: "Neo4j calendar release", + version: "2025.10.1", + wantScheme: versionSchemeCalendar, + wantFields: []int{2025, 10, 1}, + }, + { + name: "calendar month keeps its leading zero as a number", + version: "2025.06", + wantScheme: versionSchemeCalendar, + wantFields: []int{2025, 6}, + }, + { + name: "the pinned Neo4j image tag is a release, not a pre-release", + version: "2025.10.1-enterprise", + wantScheme: versionSchemeCalendar, + wantFields: []int{2025, 10, 1}, + }, + { + name: "a community tag is likewise a release", + version: "5.26.1-community", + wantScheme: versionSchemeSequential, + wantFields: []int{5, 26, 1}, + }, + { + name: "the pinned PostgreSQL image tag states only its major", + version: "18-alpine", + wantScheme: versionSchemeSequential, + wantFields: []int{18}, + }, + { + name: "PostgreSQL server_version carries its packaging detail", + version: "18.1 (Debian 18.1-1.pgdg120+1)", + wantScheme: versionSchemeSequential, + wantFields: []int{18, 1}, + }, + { + name: "a numbered alpha is a pre-release", + version: "5.26.0-alpha09", + wantScheme: versionSchemeSequential, + wantFields: []int{5, 26, 0}, + wantPrerelease: true, + }, + { + name: "a release candidate is a pre-release", + version: "2026.01.0-rc1", + wantScheme: versionSchemeCalendar, + wantFields: []int{2026, 1, 0}, + wantPrerelease: true, + }, + { + name: "the marker is matched however it is cased", + version: "5.26.0-Beta.2", + wantScheme: versionSchemeSequential, + wantFields: []int{5, 26, 0}, + wantPrerelease: true, + }, + { + name: "build metadata is not a pre-release", + version: "18.1+build.7", + wantScheme: versionSchemeSequential, + wantFields: []int{18, 1}, + }, + { + name: "surrounding whitespace is not part of the version", + version: " 5.26.1 ", + wantScheme: versionSchemeSequential, + wantFields: []int{5, 26, 1}, + }, + { + name: "an empty version cannot be ordered", + version: "", + wantErr: true, + }, + { + name: "whitespace alone cannot be ordered", + version: " ", + wantErr: true, + }, + { + name: "a non-numeric version cannot be ordered", + version: "unknown", + wantErr: true, + }, + { + name: "an empty component cannot be ordered", + version: "5..1", + wantErr: true, + }, + { + name: "a qualifier with no numbers before it cannot be ordered", + version: "-enterprise", + wantErr: true, + }, + { + name: "a non-numeric component cannot be ordered", + version: "5.26.x", + wantErr: true, + }, + { + name: "an unparsed utility banner cannot be ordered", + version: "pg_dump (PostgreSQL) 18.1", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseDatabaseVersion(tt.version) + if tt.wantErr { + if err == nil { + t.Fatalf("parseDatabaseVersion(%q) = %+v, want an error", tt.version, got) + } + + return + } + if err != nil { + t.Fatalf("parseDatabaseVersion(%q) returned %v, want no error", tt.version, err) + } + + if got.scheme != tt.wantScheme { + t.Errorf("scheme = %d, want %d", got.scheme, tt.wantScheme) + } + if !reflect.DeepEqual(got.fields, tt.wantFields) { + t.Errorf("fields = %v, want %v", got.fields, tt.wantFields) + } + if got.prerelease != tt.wantPrerelease { + t.Errorf("prerelease = %t, want %t", got.prerelease, tt.wantPrerelease) + } + if got.Original != strings.TrimSpace(tt.version) { + t.Errorf("Original = %q, want the version as it was read", got.Original) + } + }) + } +} + +// TestDatabaseVersionCompare is the ordering FR-006 rests on. Every case is run +// in both directions, because a comparator that answers "older" both ways round +// would satisfy half a table and still abort a working run. +func TestDatabaseVersionCompare(t *testing.T) { + tests := []struct { + name string + // want is the ordering of left against right: -1 older, 0 neither + // older, 1 newer. + left string + right string + want int + }{ + // Within the 5.x scheme. + {name: "patch levels order numerically", left: "5.26.0", right: "5.26.1", want: -1}, + {name: "minor versions order numerically, not lexically", left: "5.26.0", right: "5.9.0", want: 1}, + {name: "majors order numerically", left: "4.4.30", right: "5.26.1", want: -1}, + {name: "identical versions are neither older", left: "5.26.1", right: "5.26.1", want: 0}, + + // Within the calendar scheme. + {name: "calendar months order within a year", left: "2025.06.0", right: "2025.10.1", want: -1}, + {name: "calendar years order across a year boundary", left: "2025.12.0", right: "2026.01.0", want: -1}, + {name: "calendar patch levels order numerically", left: "2025.10.1", right: "2025.10.0", want: 1}, + + // Across the two schemes: the misordering FR-006 names. + {name: "a calendar release is newer than the 5.26 LTS", left: "5.26.1", right: "2025.06.0", want: -1}, + {name: "and newer whichever side it is written on", left: "2026.07.0", right: "5.26.1", want: 1}, + {name: "the pair a lexical comparison orders backwards", left: "5.26", right: "2025.06", want: -1}, + {name: "a bare calendar year outranks a bare 5.x major", left: "2025", right: "5", want: 1}, + + // Precision: an unstated field is not a zero. + {name: "a major-only utility matches its own series", left: "18", right: "18.1", want: 0}, + {name: "a major-only utility is still older than the next series", left: "18", right: "19.2", want: -1}, + {name: "a minor-only version matches its own patch series", left: "5.26", right: "5.26.4", want: 0}, + {name: "the pinned image tag matches the server it reads", left: "18-alpine", right: "18.1", want: 0}, + + // Pre-releases, and the build qualifiers that only look like them. + {name: "an alpha precedes the release of the same numbers", left: "2025.10.1-alpha", right: "2025.10.1", want: -1}, + {name: "a release candidate precedes its release", left: "5.26.0-rc1", right: "5.26.0", want: -1}, + {name: "an edition tag does not precede anything", left: "2025.10.1-enterprise", right: "2025.10.1", want: 0}, + {name: "two pre-releases of the same numbers are not ordered", left: "5.26.0-alpha09", right: "5.26.0-beta", want: 0}, + {name: "a pre-release still loses to newer numbers", left: "2026.01.0-rc1", right: "2025.10.1", want: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + left, err := parseDatabaseVersion(tt.left) + if err != nil { + t.Fatalf("parseDatabaseVersion(%q) returned %v", tt.left, err) + } + right, err := parseDatabaseVersion(tt.right) + if err != nil { + t.Fatalf("parseDatabaseVersion(%q) returned %v", tt.right, err) + } + + if got := sign(left.compare(right)); got != tt.want { + t.Errorf("compare(%q, %q) = %d, want %d", tt.left, tt.right, got, tt.want) + } + if got := sign(right.compare(left)); got != -tt.want { + t.Errorf("compare(%q, %q) = %d, want %d", tt.right, tt.left, got, -tt.want) + } + }) + } +} + +// sign reduces a comparison result to -1, 0 or 1, so a case states the ordering +// it means rather than the magnitude a comparator happens to return. +func sign(result int) int { + switch { + case result < 0: + return -1 + case result > 0: + return 1 + default: + return 0 + } +} + +// TestCheckVersionCompatibility is FR-006's asymmetry: an older utility aborts +// naming both versions, a newer one warns and continues. The two directions are +// asserted separately and neither is derived from the other, because collapsing +// them into one symmetric check is exactly the mistake the requirement forbids. +func TestCheckVersionCompatibility(t *testing.T) { + tests := []struct { + name string + service string + utility string + server string + wantErr bool + wantWarn bool + }{ + { + name: "an older utility aborts", + service: serviceNeo4j, + utility: "5.26.1", + server: "2025.10.1", + wantErr: true, + }, + { + name: "an older utility within the same scheme aborts too", + service: serviceTaskManagerDB, + utility: "17.5", + server: "18.1", + wantErr: true, + }, + { + name: "a newer utility warns and continues", + service: serviceNeo4j, + utility: "2025.10.1-enterprise", + server: "5.26.1", + wantWarn: true, + }, + { + name: "a matching pair passes quietly", + service: serviceNeo4j, + utility: "2025.10.1-enterprise", + server: "2025.10.1", + }, + { + name: "the pinned PostgreSQL tag passes against its own series", + service: serviceTaskManagerDB, + utility: "18-alpine", + server: "18.1 (Debian 18.1-1.pgdg120+1)", + }, + { + // pg_dump's own rule is major-only, so a provider that patches its + // managed server ahead of the image is not an incompatibility — and + // aborting on it would name an image flag with nothing to move to, + // since the official images publish no 18.2-alpine. + name: "a provider-patched PostgreSQL server passes against an earlier patch", + service: serviceTaskManagerDB, + utility: "18.1", + server: "18.2", + }, + { + name: "a newer PostgreSQL major still warns", + service: serviceTaskManagerDB, + utility: "19.0", + server: "18.6", + wantWarn: true, + }, + { + // The Neo4j arm is deliberately not loosened alongside it: no + // published rule says its backup client tolerates an older patch, + // and inventing one would put a guess inside an abort condition. + name: "the Neo4j arm still compares the patch", + service: serviceNeo4j, + utility: "5.26.1", + server: "5.26.2", + wantErr: true, + }, + { + name: "an unreadable server version aborts rather than passing", + service: serviceNeo4j, + utility: "2025.10.1-enterprise", + server: "unknown", + wantErr: true, + }, + { + name: "an unreadable utility version aborts rather than passing", + service: serviceTaskManagerDB, + utility: "", + server: "18.1", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + logged := captureLogrus(t, func() { + err = checkVersionCompatibility(tt.service, tt.utility, tt.server) + }) + + if tt.wantErr { + if err == nil { + t.Fatalf("checkVersionCompatibility(%q, %q, %q) = nil, want an error", tt.service, tt.utility, tt.server) + } + // Both versions have to be in the message: the operator cannot + // tell which end to move from only one of them. + for _, want := range []string{tt.utility, tt.server} { + if want == "" { + continue + } + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %q, want it to name %q", err, want) + } + } + + return + } + if err != nil { + t.Fatalf("checkVersionCompatibility(%q, %q, %q) = %v, want no error", tt.service, tt.utility, tt.server, err) + } + + warned := strings.Contains(logged, "level=warning") + if warned != tt.wantWarn { + t.Errorf("warned = %t, want %t (log = %q)", warned, tt.wantWarn, logged) + } + }) + } +} + +// TestCheckVersionCompatibilityAbortNamesTheFlag is the other half of a useful +// abort: the message says which image to move. Both databases run the one +// image (FR-031), so both name the one flag, and the database that actually +// failed is named beside it. +func TestCheckVersionCompatibilityAbortNamesTheFlag(t *testing.T) { + tests := []struct { + service string + wantFlag string + }{ + {service: serviceNeo4j, wantFlag: "--external-db-image"}, + {service: serviceTaskManagerDB, wantFlag: "--external-db-image"}, + } + + for _, tt := range tests { + t.Run(tt.service, func(t *testing.T) { + err := checkVersionCompatibility(tt.service, "5.26.1", "2025.10.1") + if err == nil { + t.Fatal("checkVersionCompatibility() = nil, want an error") + } + if !strings.Contains(err.Error(), tt.wantFlag) { + t.Errorf("error = %q, want it to name %q", err, tt.wantFlag) + } + if !strings.Contains(err.Error(), tt.service) { + t.Errorf("error = %q, want it to name the %q database", err, tt.service) + } + }) + } +} + +// TestFailureMessageContract is contracts/cli-surface.md's failure-message +// table, asserted row by row: every message names the condition, the specific +// resource, and the operator action (SC-008). +// +// It is one table over every row rather than an assertion beside each +// constructor, because the requirement is uniform and because a row with no +// case here is exactly how this contract goes stale. Each case drives the +// constructor a production path calls — never a message assembled by the test — +// so a condition whose message exists but which nothing reaches would fail +// here only if it were also deleted; the caller-reachability of each is what +// `grep` over non-test callers answers, and the doc comment on each constructor +// names its caller. +// +// `wantAbsent` is as load-bearing as the rest for the two rows the contract +// states negatively. Reporting a denied query as an absent database, or a +// Community refusal as a missing service, is the defect this feature exists to +// remove, and a message can carry all three elements and still say the wrong +// one of those. +func TestFailureMessageContract(t *testing.T) { + endpoint := &DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "neo4j.example.com", Port: 7687}}, + Database: "infrahub", + } + refused := errors.New(`pods is forbidden: User "system:serviceaccount:infrahub:backup" cannot create resource "pods" in API group "" in the namespace "infrahub"`) + + tests := []struct { + // row is the condition as contracts/cli-surface.md names it. + row string + err error + + // The three elements the contract requires of every message. Split so a + // failure says which of the three is missing rather than that a long + // string did not match. + wantCondition []string + wantResource []string + wantAction []string + + // wantAbsent is what the message must not say, for the rows the + // contract states as "explicitly not …". + wantAbsent []string + }{ + { + row: "no container and no discoverable endpoint (Neo4j)", + err: undiscoverableEndpoint(serviceNeo4j, componentInfrahubServer, + []string{neo4jAddressEnvVar, neo4jPortEnvVar}, neo4jAddressFlag), + wantCondition: []string{"does not run in this deployment", "could be discovered"}, + wantResource: []string{neo4jAddressEnvVar, neo4jPortEnvVar, componentInfrahubServer, "were read for it"}, + wantAction: []string{"--" + neo4jAddressFlag, "INFRAHUB_NEO4J_ADDRESS", "--k8s-namespace"}, + }, + { + // The condition above with its central claim removed: the settings + // were not read, so what they record is unknown. Two conditions, + // two remedies — and they were one message. + row: "no container and the deployment's settings could not be read", + err: unreadableDeploymentSettings(serviceNeo4j, componentInfrahubServer, + []string{neo4jAddressEnvVar, neo4jPortEnvVar}, neo4jAddressFlag, refused), + wantCondition: []string{"does not run in this deployment", "could not be read"}, + wantResource: []string{neo4jAddressEnvVar, componentInfrahubServer, "is forbidden"}, + wantAction: []string{"--" + neo4jAddressFlag, "INFRAHUB_NEO4J_ADDRESS", "restore this run's access"}, + wantAbsent: []string{"were read for it", "none of them records an address"}, + }, + { + row: "the restored database's state cannot be read at all", + err: neo4jRestoreUnreadable( + &externalRestore{externalSource: &externalSource{Endpoint: endpoint}, Bound: time.Hour}, + refused, 90*time.Second), + wantCondition: []string{"unable to read the database's state", "1m30s"}, + wantResource: []string{"infrahub", "neo4j.example.com:7687", "is forbidden"}, + wantAction: []string{"Check that pod and that permission", "read the database's own state directly"}, + // It must not read as a verdict on the database: the read never + // reached it, and the seed may be loading successfully. + wantAbsent: []string{"the database is not usable", "neo4j.log"}, + }, + { + row: "no container and no discoverable endpoint (task manager)", + err: undiscoverableEndpoint(serviceTaskManagerDB, componentTaskManager, + prefectConnectionEnvVars, postgresAddressFlag), + wantCondition: []string{"does not run in this deployment", "could be discovered"}, + wantResource: []string{ + prefectConnectionEnvVars[0], prefectConnectionEnvVars[1], + componentTaskManager, "were read for it", + }, + wantAction: []string{"--" + postgresAddressFlag, "INFRAHUB_POSTGRES_ADDRESS", "--k8s-namespace"}, + }, + { + row: "deployment query denied", + err: newTestKubernetesBackend().undeterminedLocation(serviceNeo4j, refused), + wantCondition: []string{"failed to determine whether", "could not be queried", "is forbidden"}, + wantResource: []string{serviceNeo4j, "namespace infrahub", "deployments and statefulsets in namespace infrahub"}, + wantAction: []string{"get and list on pods, deployments and statefulsets", "--k8s-namespace"}, + // FR-002. The contract states this row negatively because the two + // readings have opposite remedies: an operator told the database is + // absent configures an external one they do not have. + wantAbsent: []string{"the database is absent:"}, + }, + { + row: "missing workload-creation permission", + err: workloadPermissionError(refused, "create", "pods", "infrahub"), + wantCondition: []string{"refused this tool the transient workload", "is forbidden"}, + wantResource: []string{"create on pods in namespace infrahub"}, + wantAction: []string{transientWorkloadRBAC, "grant it", "from a host that can reach the database directly"}, + }, + { + row: "external Community edition", + err: externalCommunityCaptureRefusal(endpoint), + wantCondition: []string{"Community Edition backup mechanism requires direct access to the database's own storage"}, + wantResource: []string{"neo4j.example.com:7687"}, + wantAction: []string{"Neo4j Enterprise Edition feature"}, + // FR-008. "service not found" is the message that sent a customer + // looking for a container that was never meant to be there. + wantAbsent: []string{"service not found", "no such service"}, + }, + { + row: "version incompatibility", + err: checkVersionCompatibility(serviceNeo4j, "5.26.1", "2025.10.1"), + wantCondition: []string{"is older than the server"}, + wantResource: []string{"5.26.1", "2025.10.1"}, + wantAction: []string{externalDBImageOption, "at least as new as the server"}, + }, + { + row: "missing authorisation for external restore", + err: externalRestoreUnauthorised(serviceNeo4j), + wantCondition: []string{"requires an explicit authorisation", "does not run in this deployment"}, + wantResource: []string{serviceNeo4j, "no Infrahub service was stopped and no data was changed"}, + wantAction: []string{"--" + AllowExternalRestoreFlag, AllowExternalRestoreEnvVar}, + }, + { + row: "server cannot read the artifact", + err: unreadableSeedArtifact("s3://backups/seed/infrahub.backup", errors.New("NoSuchKey")), + wantCondition: []string{"could not be read back", "NoSuchKey"}, + wantResource: []string{"s3://backups/seed/infrahub.backup"}, + wantAction: []string{"readable both from here and from the database host"}, + }, + { + row: "restore did not come online", + err: neo4jRestoreFailure( + &externalRestore{externalSource: &externalSource{Endpoint: endpoint}, Bound: time.Hour}, + "offline", false), + wantCondition: []string{"was accepted but the database is not usable"}, + wantResource: []string{"infrahub", "neo4j.example.com:7687", "Its reported state is offline"}, + wantAction: []string{"neo4j.log", "debug.log"}, + }, + { + row: "infrahub-collect --include-backup against an external database", + err: (&InfrahubOps{externalCaptureForbidden: true}).refuseForbiddenExternalCapture("infrahub"), + wantCondition: []string{"does not run in this deployment"}, + wantResource: []string{"transient pod and secret in namespace infrahub"}, + wantAction: []string{"infrahub-backup create", "--include-backup"}, + }, + { + // FR-031. The build version is set for the rest of this test, which + // no other row reads. + row: "development build with no --external-db-image", + err: func() error { + withBuildVersion(t, "") + + return refuseUnpublishedExternalDBImage(&Configuration{}) + }(), + wantCondition: []string{"does not run in this deployment", "development build"}, + wantResource: []string{"no published image", "transient workload"}, + wantAction: []string{"--external-db-image", "released infrahub-backup"}, + }, + } + + for _, tt := range tests { + t.Run(tt.row, func(t *testing.T) { + if tt.err == nil { + t.Fatalf("the %q condition produced no error, so there is no message to read", tt.row) + } + message := tt.err.Error() + + for element, wants := range map[string][]string{ + "condition": tt.wantCondition, + "resource": tt.wantResource, + "operator action": tt.wantAction, + } { + if len(wants) == 0 { + t.Errorf("no %s is asserted for this row, so the contract is not checked for it", element) + } + for _, want := range wants { + if !strings.Contains(message, want) { + t.Errorf("the %s element is missing %q:\n%s", element, want, message) + } + } + } + + for _, absent := range tt.wantAbsent { + if strings.Contains(strings.ToLower(message), strings.ToLower(absent)) { + t.Errorf("the message says %q, which the contract states this condition must not:\n%s", absent, message) + } + } + }) + } +} + +// TestDiagnosticRefusalsAreDistinguishable is the conflation the contract draws +// two separate rows for: an unanswered deployment query and an endpoint nothing +// records are different conditions with different remedies, and each must be +// identifiable as itself. +// +// It asserts on the sentinel rather than the wording, because that is what a +// caller can branch on — and because a reworded message that merged the two +// would otherwise still pass the table above. +func TestDiagnosticRefusalsAreDistinguishable(t *testing.T) { + undiscoverable := undiscoverableEndpoint(serviceNeo4j, componentInfrahubServer, + []string{neo4jAddressEnvVar}, neo4jAddressFlag) + undetermined := newTestKubernetesBackend().undeterminedLocation(serviceNeo4j, + errors.New(`pods is forbidden: cannot list resource "pods"`)) + forbidden := workloadPermissionError( + errors.New(`pods is forbidden: cannot create resource "pods"`), "create", "pods", "infrahub") + + if !errors.Is(undiscoverable, errEndpointUndiscoverable) { + t.Errorf("the undiscoverable-endpoint refusal does not carry its own sentinel: %v", undiscoverable) + } + if errors.Is(undiscoverable, errLocationUndetermined) || errors.Is(undiscoverable, errTransientWorkloadForbidden) { + t.Errorf("an endpoint nobody records reads as a cluster failure: %v", undiscoverable) + } + + if !errors.Is(undetermined, errLocationUndetermined) { + t.Errorf("the undetermined-location refusal does not carry its own sentinel: %v", undetermined) + } + if errors.Is(undetermined, errEndpointUndiscoverable) { + t.Errorf("a denied deployment query reads as an undiscoverable endpoint, which is the conflation FR-002 and FR-004 are two requirements to prevent: %v", undetermined) + } + + if !errors.Is(forbidden, errTransientWorkloadForbidden) { + t.Errorf("the missing-workload-permission refusal does not carry its own sentinel: %v", forbidden) + } + if errors.Is(forbidden, errEndpointUndiscoverable) || errors.Is(forbidden, errLocationUndetermined) { + t.Errorf("an RBAC failure creating the workload reads as a database nobody could find, which is the exact conflation FR-019 is separate from FR-004 to prevent: %v", forbidden) + } +} + +// TestWorkloadPermissionErrorLeavesOtherFailuresAlone keeps FR-019's message on +// the one condition it belongs to. A bound that expired, or a connection that +// dropped, is not a missing permission, and telling the operator to grant a +// Role for it would send them to fix something that is not broken. +func TestWorkloadPermissionErrorLeavesOtherFailuresAlone(t *testing.T) { + for _, cause := range []error{ + errors.New("Unable to connect to the server: dial tcp: i/o timeout"), + errors.New("error: timed out waiting for the condition"), + errors.New(`error: You must be logged in to the server (Unauthorized)`), + } { + got := workloadPermissionError(cause, "create", "pods", "infrahub") + if got != cause { + t.Errorf("workloadPermissionError(%v) rewrote a failure that was not a refusal: %v", cause, got) + } + if errors.Is(got, errTransientWorkloadForbidden) { + t.Errorf("%v was reported as a missing permission", cause) + } + } +} + +// TestReportDatabaseLocations is US2 scenario 5 at the command an operator +// reaches for first: `environment detect` has to say, per database, whether it +// runs in the deployment and where an external one was resolved to. +// +// It asserts through the same constructor every other path reaches through +// resolveDatabaseTargets, and on the operation it uses it with: databaseInspect is what keeps the report +// from creating anything, which is what `infrahub-collect` depends on +// (ADR-0003), and what keeps a failure from naming a backup nobody asked for. +func TestReportDatabaseLocations(t *testing.T) { + t.Run("a mixed topology reports each database on its own evidence", func(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := stubQueries(map[string]EndpointLocation{ + serviceNeo4j: EndpointLocationExternal, + serviceTaskManagerDB: EndpointLocationInternal, + }, &discovered) + + var reportErr error + logged := captureLogrus(t, func() { + reportErr = iops.reportDatabaseLocationsWith(queries) + }) + if reportErr != nil { + t.Fatalf("reportDatabaseLocationsWith() error = %v; the report must not fail the command", reportErr) + } + t.Logf("environment detect reported:\n%s", logged) + + // Only the external database is discovered for: an internal one has a + // container, so reporting it costs no new query (FR-015). + if !reflect.DeepEqual(discovered, []string{serviceNeo4j}) { + t.Errorf("discovery ran for %v, want only the external %s", discovered, serviceNeo4j) + } + + // What the operator actually reads. Asserted on the output rather than + // on the resolution, because US2 scenario 5 is a claim about the report + // and a resolution nothing prints satisfies none of it. + for _, want := range []string{ + // Per database, and by name: a report that says "one database is + // external" leaves the operator to work out which. + "Database " + serviceNeo4j + ": does not run in this deployment", + "Database " + serviceTaskManagerDB + ": runs in this deployment", + // Where the external one was resolved to. + "core-1:7687", "core-2:7688", + } { + if !strings.Contains(logged, want) { + t.Errorf("the report does not state %q:\n%s", want, logged) + } + } + }) + + t.Run("an unanswered location query is reported, not treated as external", func(t *testing.T) { + iops := mixedTopologyOps() + discovered := []string{} + queries := deploymentQueries{ + locate: func(string) (EndpointLocation, error) { + return "", errors.New(`pods is forbidden: cannot list resource "pods"`) + }, + discover: func(service string) error { + discovered = append(discovered, service) + + return nil + }, + } + + var reportErr error + logged := captureLogrus(t, func() { + reportErr = iops.reportDatabaseLocationsWith(queries) + }) + if reportErr != nil { + t.Fatalf("reportDatabaseLocationsWith() error = %v; a diagnostic must still report what it could", reportErr) + } + t.Logf("environment detect reported:\n%s", logged) + + // FR-002 at the report: nothing may proceed as though the database were + // outside the deployment on the strength of a query that failed. + if len(discovered) != 0 { + t.Errorf("discovery ran for %v after the location query failed, want none", discovered) + } + if strings.Contains(logged, "does not run in this deployment") { + t.Errorf("an unanswered query was reported as a database outside the deployment (FR-002):\n%s", logged) + } + for _, service := range []string{serviceNeo4j, serviceTaskManagerDB} { + if want := "Database " + service + ": could not establish where it lives"; !strings.Contains(logged, want) { + t.Errorf("the report is silent about the query it could not answer for %s:\n%s", service, logged) + } + } + }) + + t.Run("one unanswered location query does not hide the other database", func(t *testing.T) { + // A run stops at the first database it cannot locate, because nothing + // may be stopped once one location is unknown. The report must not: + // a failed query for one database says nothing about the other, and + // the operator diagnosing it needs what the deployment could answer. + iops := mixedTopologyOps() + queries := deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + if service == serviceNeo4j { + return "", errors.New(`pods is forbidden: cannot list resource "pods"`) + } + + return EndpointLocationInternal, nil + }, + discover: func(string) error { return nil }, + } + + var reportErr error + logged := captureLogrus(t, func() { + reportErr = iops.reportDatabaseLocationsWith(queries) + }) + if reportErr != nil { + t.Fatalf("reportDatabaseLocationsWith() error = %v; a diagnostic must still report what it could", reportErr) + } + + for _, want := range []string{ + "Database " + serviceNeo4j + ": could not establish where it lives", + "Database " + serviceTaskManagerDB + ": runs in this deployment", + } { + if !strings.Contains(logged, want) { + t.Errorf("the report does not state %q:\n%s", want, logged) + } + } + }) + + t.Run("an inspection needs no restore authorisation", func(t *testing.T) { + // Deliberately the zero authorisation, which is what every binary's + // `environment detect` holds. Were an inspection to consult FR-009, + // detect would refuse to describe the very deployment an operator is + // diagnosing because they cannot restore into it. + target, err := databaseTargetWith( + stubQueries(map[string]EndpointLocation{serviceNeo4j: EndpointLocationExternal}, &[]string{}), + databaseInspect, serviceNeo4j, ExternalRestoreAuth{}) + if err != nil { + t.Fatalf("databaseTargetWith(inspect) = %v, want an external database to be inspectable unauthorised", err) + } + if target.Location != EndpointLocationExternal { + t.Errorf("target.Location = %q, want %q", target.Location, EndpointLocationExternal) + } + }) + + t.Run("a failure names the report rather than a backup", func(t *testing.T) { + _, err := databaseTargetWith( + deploymentQueries{locate: func(string) (EndpointLocation, error) { + return "", errors.New(`pods is forbidden: cannot list resource "pods"`) + }}, + databaseInspect, serviceNeo4j, ExternalRestoreAuth{}) + if err == nil { + t.Fatal("databaseTargetWith(inspect) = nil, want the unanswered location query reported (FR-002)") + } + if strings.Contains(err.Error(), "cannot start the backup") { + t.Errorf("the report's failure names a backup the operator did not ask for: %v", err) + } + if !strings.Contains(err.Error(), databaseInspect.describe()) { + t.Errorf("err = %v, want it to name the %q operation", err, databaseInspect.describe()) + } + }) +} + +// TestCypherColumnLookupIsIndifferentToCase pins the contract every reader of a +// `--format plain` result now shares: a column is found under the spelling the +// caller asks with, whatever case either side uses. +// +// It is a test of the helper rather than of a parser because no production +// column constant is mixed-case on the member path today, so no parser test can +// reach the trap. The trap is the next constant: three readers answered "which +// position is this column at?" three ways, and one of them — parseNeo4jMemberRoles +// — indexed the header lowercased and then looked its constants up raw. That is +// correct only while every constant happens to be lowercase. A member column +// spelled the way neo4jStatusColumn already is, `currentStatus`, would not have +// been found, and the parser answers a missing column by returning nil: every +// member's role gone from SourceRoles, FR-005's provenance silently empty, and +// no error raised anywhere. +// +// Both directions are asserted, because both sides of the lookup have to +// normalise or the two disagree: the header carries the server's spelling, the +// query carries the caller's. +func TestCypherColumnLookupIsIndifferentToCase(t *testing.T) { + // A header as `--format plain` prints one: the server's own spellings, and + // padding around them. + columns := indexCypherColumns([]string{" currentStatus ", "ADDRESS", "writer"}) + + t.Run("a caller asking with a mixed-case name", func(t *testing.T) { + for _, asked := range []string{"currentStatus", "currentstatus", "CURRENTSTATUS", " currentStatus "} { + index, ok := columns.at(asked) + if !ok || index != 0 { + t.Errorf("at(%q) = (%d, %t), want (0, true): a column constant spelled with a capital must still be found", asked, index, ok) + } + } + }) + + t.Run("a server spelling the header differently from the constant", func(t *testing.T) { + index, ok := columns.at("address") + if !ok || index != 1 { + t.Errorf("at(%q) = (%d, %t), want (1, true): the header carries the server's spelling, not the caller's", "address", index, ok) + } + }) + + t.Run("a column the result does not carry", func(t *testing.T) { + if index, ok := columns.at("statusMessage"); ok { + t.Errorf("at(%q) = (%d, true), want it absent", "statusMessage", index) + } + }) + + // The same normalisation the header check has to use, or a header can name + // a column that the lookup then cannot find — which is the shape of the + // original divergence. + if !indexCypherColumns([]string{" currentStatus ", "ADDRESS"}).has("currentstatus", "Address") { + t.Error("has() = false, want the header recognised under either side's spelling") + } +} diff --git a/src/internal/app/external_image_test.go b/src/internal/app/external_image_test.go new file mode 100644 index 00000000..941b0dab --- /dev/null +++ b/src/internal/app/external_image_test.go @@ -0,0 +1,235 @@ +package app + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// withBuildVersion sets the build version a binary's main would have set from +// its ldflags, for the duration of one test. The default workload image is read +// from it when the configuration is built (FR-031), so a test that needs a +// released build's default, or a development build's absence of one, says +// which it is rather than inheriting whatever the test binary was built with. +func withBuildVersion(t *testing.T, v string) { + t.Helper() + + previous := version + SetVersion(v) + t.Cleanup(func() { SetVersion(previous) }) +} + +// TestPublishedImageTag is the definition of "development build" FR-031 turns +// on, as a table: a release sets main.version from its git tag ("v2.3.0") and +// the release workflow pushes the image under that tag without its "v" +// ("2.3.0"), so exactly the release-shaped versions have a tag, and it is that +// one. +func TestPublishedImageTag(t *testing.T) { + tests := []struct { + version string + wantTag string + wantOK bool + }{ + {version: "v2.3.0", wantTag: "2.3.0", wantOK: true}, + // `make build VERSION=2.3.0` names the same release. + {version: "2.3.0", wantTag: "2.3.0", wantOK: true}, + // A published pre-release is a release too: the workflow runs on it. + {version: "v2.4.0-rc.1", wantTag: "2.4.0-rc.1", wantOK: true}, + // A plain `go build` or `make build` leaves the version empty. + {version: "", wantOK: false}, + // A commit hash, as BuildRevision falls back to. + {version: "ea1d73b", wantOK: false}, + {version: "ea1d73b4c0ffee-dirty", wantOK: false}, + // `git describe` output and build metadata: not a tag anything was + // pushed under. + {version: "v2.3.0-4-gea1d73b", wantOK: false}, + {version: "v2.3.0-4-gea1d73b-dirty", wantOK: false}, + {version: "v2.3.0-dirty", wantOK: false}, + {version: "v2.3.0+dirty", wantOK: false}, + // A describe suffix after a pre-release names a commit after it. + {version: "v2.4.0-rc.1-3-gabc1234", wantOK: false}, + {version: "v2.4.0-rc.1-3-gabc1234-dirty", wantOK: false}, + {version: "2.4.0-rc.1-3-gabc1234", wantOK: false}, + {version: "v2.4.0-rc.1-dirty", wantOK: false}, + {version: "v2.4.0-beta.2", wantTag: "2.4.0-beta.2", wantOK: true}, + // semver reads "v2.3" as "v2.3.0", but no image was pushed as "2.3". + {version: "v2.3", wantOK: false}, + {version: "dev", wantOK: false}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%q", tt.version), func(t *testing.T) { + tag, ok := publishedImageTag(tt.version) + if ok != tt.wantOK || tag != tt.wantTag { + t.Errorf("publishedImageTag(%q) = (%q, %v), want (%q, %v)", tt.version, tag, ok, tt.wantTag, tt.wantOK) + } + }) + } +} + +// TestDefaultExternalDBImageFollowsTheBuildVersion is T165: the workload runs +// this tool's own image at the running binary's version, from the registry the +// release workflow publishes to, and a development build has no default at all +// rather than a guessed one. +func TestDefaultExternalDBImageFollowsTheBuildVersion(t *testing.T) { + t.Run("a release runs its own image", func(t *testing.T) { + withBuildVersion(t, "v2.3.0") + + const want = "registry.opsmill.io/opsmill/infrahub-backup:2.3.0" + if got := defaultExternalDBImage(); got != want { + t.Errorf("defaultExternalDBImage() = %q, want %q", got, want) + } + if got := NewInfrahubOps().Config().ExternalDB.Image; got != want { + t.Errorf("a new configuration's Image = %q, want %q", got, want) + } + }) + + t.Run("a development build has no default", func(t *testing.T) { + withBuildVersion(t, "") + + if got := defaultExternalDBImage(); got != "" { + t.Errorf("defaultExternalDBImage() = %q on a development build, want empty: no image was published for it", got) + } + if got := NewInfrahubOps().Config().ExternalDB.Image; got != "" { + t.Errorf("a new configuration's Image = %q on a development build, want empty", got) + } + }) +} + +// writeRecordingKubectl installs a kubectl that appends its arguments to log +// and fails, and returns the log's path. Failing is deliberate: a stub that +// succeeded would let a run that should have been refused go on to read its +// own invented answers. +func writeRecordingKubectl(t *testing.T) string { + t.Helper() + + bin := t.TempDir() + log := filepath.Join(t.TempDir(), "kubectl-calls") + script := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' \"$*\" >> %q\nexit 1\n", log) + stub := filepath.Join(bin, "kubectl") + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatalf("writing the kubectl stub = %v, want nil", err) + } + + // A positive control: the assertion below is "the log is empty", which a + // stub that could not write its log would satisfy vacuously. + if err := exec.Command(stub, "version").Run(); err == nil { + t.Fatal("the kubectl stub exited 0, want it to fail") + } + recorded, err := os.ReadFile(log) + if err != nil || strings.TrimSpace(string(recorded)) != "version" { + t.Fatalf("the kubectl stub did not record its own call (log %q, err %v); the test could not observe a call", recorded, err) + } + if err := os.Remove(log); err != nil { + t.Fatalf("resetting the kubectl log = %v, want nil", err) + } + + t.Setenv("PATH", bin) + + return log +} + +// TestDevelopmentBuildIsRefusedBeforeAnyKubectlCall is FR-031's refusal, +// asserted where it has to land: after the gate has found an external +// database, and before the run has spent a single kubectl call — not the +// reaper's list and delete, not a create. Both directions are driven through +// their real gate, because the refusal is only worth having if neither a +// backup nor a restore can reach a workload without passing it. +func TestDevelopmentBuildIsRefusedBeforeAnyKubectlCall(t *testing.T) { + neo4jExternal := deploymentQueries{ + locate: func(service string) (EndpointLocation, error) { + if service == serviceNeo4j { + return EndpointLocationExternal, nil + } + + return EndpointLocationInternal, nil + }, + discover: func(string) error { return nil }, + } + + runs := []struct { + name string + run func(iops *InfrahubOps) error + }{ + { + name: "backup", + run: func(iops *InfrahubOps) error { + return iops.prepareDatabaseCaptureWith(neo4jExternal, true) + }, + }, + { + name: "restore", + run: func(iops *InfrahubOps) error { + // Authorised, so the only thing left to refuse it is the image. + iops.config.ExternalDB.Restore = ExternalRestoreAuth{Source: ExternalRestoreAuthFlag} + + return iops.prepareDatabaseRestoreWith(neo4jExternal, true) + }, + }, + } + + for _, tt := range runs { + t.Run(tt.name, func(t *testing.T) { + withBuildVersion(t, "") + log := writeRecordingKubectl(t) + + iops := NewInfrahubOps() + iops.backend = newTestKubernetesBackend() + + err := tt.run(iops) + if err == nil { + t.Fatal("the gate returned nil on a development build with no --external-db-image, want the FR-031 refusal") + } + for _, want := range []string{"--external-db-image", "development build", "Nothing was created or stopped"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to contain %q", err, want) + } + } + + if calls, readErr := os.ReadFile(log); readErr == nil && len(calls) > 0 { + t.Errorf("kubectl was called before the refusal:\n%s\nwant no call at all: the refusal must precede the reaper and every create", calls) + } + }) + } +} + +// TestExternalDBImageFlagLiftsTheDevelopmentBuildRefusal is the other half of +// FR-031's "unless that flag is set", and the release-build case: with an +// image to run there is nothing to refuse, and an image explicitly emptied on +// a release is refused for what it is rather than called a development build. +func TestExternalDBImageFlagLiftsTheDevelopmentBuildRefusal(t *testing.T) { + t.Run("a development build given the flag passes", func(t *testing.T) { + withBuildVersion(t, "") + cfg := &Configuration{ExternalDB: ExternalDBConfig{Image: "registry.example.net/infrahub-backup:dev"}} + + if err := refuseUnpublishedExternalDBImage(cfg); err != nil { + t.Errorf("refuseUnpublishedExternalDBImage() = %v, want nil: the operator chose an image", err) + } + }) + + t.Run("a release passes on its default", func(t *testing.T) { + withBuildVersion(t, "v2.3.0") + + if err := refuseUnpublishedExternalDBImage(NewInfrahubOps().Config()); err != nil { + t.Errorf("refuseUnpublishedExternalDBImage() = %v, want nil on a release's own default", err) + } + }) + + t.Run("a release given an empty flag names the flag, not a development build", func(t *testing.T) { + withBuildVersion(t, "v2.3.0") + + err := refuseUnpublishedExternalDBImage(&Configuration{}) + if err == nil { + t.Fatal("refuseUnpublishedExternalDBImage() = nil with no image, want a refusal") + } + if !strings.Contains(err.Error(), "--external-db-image") || !strings.Contains(err.Error(), "empty value") { + t.Errorf("err = %v, want it to say the flag was given an empty value", err) + } + if strings.Contains(err.Error(), "development build") { + t.Errorf("err = %v, want no mention of a development build on a release", err) + } + }) +} diff --git a/src/internal/app/external_invariants_test.go b/src/internal/app/external_invariants_test.go new file mode 100644 index 00000000..be668a56 --- /dev/null +++ b/src/internal/app/external_invariants_test.go @@ -0,0 +1,117 @@ +package app + +import ( + "strings" + "testing" +) + +// TestAPreparationRefusesATargetThatDidNotComeFromTheGate is the construction +// witness on databaseTarget. +// +// The target is what the two preparers hold instead of re-deriving where a +// database lives and whether this run may write to it — and re-deriving would +// be a second answer to the question the gate exists to settle once. That makes +// the preparers' trust in their argument load-bearing, and it was held by +// convention: with four exported fields and nothing else, the literal below was +// a value any caller in the package could write, and it carries a zero Restore. +// Passed to the restore preparation it would have performed an unauthorised +// external restore, refused nowhere in the tool. +// +// No production caller ever wrote one, which is exactly the shape of the two +// defects this branch has already had: an invariant true until someone quietly +// broke it. So the verdict is now a field only databaseTargetWith sets. +func TestAPreparationRefusesATargetThatDidNotComeFromTheGate(t *testing.T) { + // The value the review named, verbatim: external, a restore, no + // authorisation. + handBuilt := databaseTarget{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Operation: databaseRestore, + } + + t.Run("a restore", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + + err := iops.prepareExternalRestoreWith((&recordingCaptureOps{}).ops(), handBuilt) + if err == nil { + t.Fatal("prepareExternalRestoreWith() = nil for a target that never passed the gate, want the refusal: this would be an unauthorised external restore (FR-009)") + } + if !strings.Contains(err.Error(), "did not come from the gate") { + t.Errorf("err = %v, want it to say the target carries no verdict", err) + } + // Principle II's claim, on the refusal that comes before anything runs. + if !strings.Contains(err.Error(), "No data was changed") { + t.Errorf("err = %v, want it to state that nothing was changed", err) + } + }) + + t.Run("a capture", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + + capture := handBuilt + capture.Operation = databaseCapture + + err := iops.prepareExternalSourceWith((&recordingCaptureOps{}).ops(), capture) + if err == nil { + t.Fatal("prepareExternalSourceWith() = nil for a target that never passed the gate, want the refusal: FR-002 forbids acting on a location that was never established") + } + if !strings.Contains(err.Error(), "did not come from the gate") { + t.Errorf("err = %v, want it to say the target carries no verdict", err) + } + }) + + // The other half: a target the gate did produce is accepted, so the witness + // refuses the right thing rather than everything. Both preparations get + // past the check and fail later, at the probe this double cannot answer — + // which is what says the refusal above was the witness and not the + // deployment. + t.Run("a gated target is not refused for want of a witness", func(t *testing.T) { + iops, _ := newExternalOps(t, "db-a") + + err := iops.prepareExternalSourceWith((&recordingCaptureOps{}).ops(), externalCaptureTarget(t, serviceNeo4j)) + if err != nil && strings.Contains(err.Error(), "did not come from the gate") { + t.Errorf("err = %v, want a target from databaseTargetWith to carry its verdict", err) + } + }) +} + +// TestExternalRestoreAuthorisationIsItsChannel is FR-009 and FR-010 held by the +// type rather than by every constructor being careful. +// +// ExternalRestoreAuth carried an `Allowed bool` beside its Source, and the two +// answered one question between them: the refusal read only the bool, the audit +// line read only the source. `{Allowed: true}` was therefore representable — it +// authorised a destructive write into infrastructure the deployment does not +// manage and then recorded "no channel this run can name", losing the one fact +// FR-010 turns on, which is whether an operator was present. Every constructor +// in cli.go set both consistently; none had to. +// +// With the bool gone the state cannot be written down, and this pins what +// replaces it: the authorisation *is* the channel, and every value that +// authorises can name one. +func TestExternalRestoreAuthorisationIsItsChannel(t *testing.T) { + for _, source := range []ExternalRestoreAuthSource{ + ExternalRestoreAuthAbsent, + ExternalRestoreAuthFlag, + ExternalRestoreAuthConfig, + } { + auth := ExternalRestoreAuth{Source: source} + + if want := source != ExternalRestoreAuthAbsent; auth.allowed() != want { + t.Errorf("ExternalRestoreAuth{Source: %q}.allowed() = %t, want %t", source, auth.allowed(), want) + } + + if auth.allowed() && strings.Contains(auth.describe(), "no channel") { + t.Errorf("ExternalRestoreAuth{Source: %q} authorises a restore it cannot attribute (%q), which is the FR-010 fact the record exists for", source, auth.describe()) + } + if !auth.allowed() && !strings.Contains(auth.describe(), "no channel") { + t.Errorf("ExternalRestoreAuth{Source: %q}.describe() = %q, want it to name no channel", source, auth.describe()) + } + } + + // The zero value is the unauthorised one, which is what makes a restore + // path that never consults the authorisation unable to proceed by default. + if (ExternalRestoreAuth{}).allowed() { + t.Error("the zero ExternalRestoreAuth authorises a restore, want it to authorise nothing") + } +} diff --git a/src/internal/app/external_local.go b/src/internal/app/external_local.go new file mode 100644 index 00000000..40219018 --- /dev/null +++ b/src/internal/app/external_local.go @@ -0,0 +1,789 @@ +package app + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// Direct mode (FR-032, ADR 0017). +// +// When infrahub-backup itself runs as a pod in the target namespace, and its +// own image carries the database tools, the transient Job is a second pod doing +// what this one could do: the tools are on this process's PATH and the +// database is reachable from this pod's network. So the run executes them as +// child processes of its own and creates no Job and no Secret. +// +// The capture and restore sequences do not know which of the two they are +// using. Both hand around an execution target — the string the Job path calls +// a pod — and every operation they perform on one goes through four seams: +// the bounded exec (execBounded), the streaming exec (ExecStreamPipe), the copy +// across the workload's boundary (copyAcrossTransientWorkload) and the dump +// fit check (refuseUnfittableRestoreDump). Each seam asks localWorkloadFor +// first, and a target this file registered is served here. That is what keeps +// FR-032's "every other requirement applies unchanged" structural: the version +// gate, the edition gate, the scratch sizing, the incomplete-capture removal +// and the restore authorisation are the same lines of code either way. +// +// The sequences address the Job's scratch volume by its mount path, +// transientScratchPath. A local workload maps that path onto a run-specific +// directory, the way the Job's volume mount maps it onto an emptyDir: every +// argument, environment value and copy path is rewritten before it reaches the +// child (see translate). The directory is created under os.TempDir — the same +// place the run stages its own archive, which the operator moves with TMPDIR — +// so the copy out of it is a rename on one filesystem rather than a second +// copy of the database. + +const ( + // serviceAccountNamespaceFile is where the kubelet writes the namespace of + // the pod a container runs in. Its absence is the plain answer to "is this + // process running as a pod at all". + serviceAccountNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + + // localTargetPrefix starts every execution target a local workload hands + // out. A colon cannot occur in a pod name, so no target this file issues can + // be mistaken for a real pod, and no real pod can be mistaken for one of + // these. + localTargetPrefix = "local:" +) + +// localToolsFor is every program the commands for one database run, by name. +// Direct mode is taken only when all of them are found on PATH, because a tool +// missing halfway through a capture is a failed capture, not a degraded one. +// +// The database clients are what the image adds; the rest are what the +// sequences' shell commands call (directory preparation, the CA write, the +// tar stream), which a minimal image could lack just as easily. +func localToolsFor(service string) []string { + switch service { + case serviceNeo4j: + return []string{"neo4j-admin", "cypher-shell", "keytool", "sh", "tar", "rm", "mkdir", "ls", "base64"} + case serviceTaskManagerDB: + return []string{"pg_dump", "pg_restore", "psql", "sh", "rm", "base64"} + default: + return nil + } +} + +// directModeHost is everything direct mode reads from the host it runs on, as +// fields, so detection and the runner can be exercised against a fake host in +// tests without mutating process-wide state. +type directModeHost struct { + // namespaceFile is serviceAccountNamespaceFile outside tests. + namespaceFile string + // lookPath is exec.LookPath outside tests. + lookPath func(file string) (string, error) + // tempDir is the parent of every run-specific scratch directory. + tempDir func() string + // freeSpace reports the bytes available to an unprivileged writer under + // path. + freeSpace func(path string) (uint64, error) + // command builds the child process. Tests wrap it to record what reached + // the child; every child is still a real process. + command func(ctx context.Context, name string, args ...string) *exec.Cmd +} + +func defaultDirectModeHost() *directModeHost { + return &directModeHost{ + namespaceFile: serviceAccountNamespaceFile, + lookPath: exec.LookPath, + tempDir: os.TempDir, + freeSpace: localFreeSpace, + command: exec.CommandContext, + } +} + +// directHost is the host direct mode reads, the real one unless a test +// supplied another. +func (iops *InfrahubOps) directHost() *directModeHost { + if iops.directModeHost == nil { + iops.directModeHost = defaultDirectModeHost() + } + + return iops.directModeHost +} + +// directModeDecision is the answer to "run the tools here?" and why. +type directModeDecision struct { + direct bool + reason string +} + +// detectDirectMode applies FR-032's two conditions: this process runs as a pod +// in the namespace the run targets, and every tool the databases need is on +// PATH. A third condition is practical rather than stated: the scratch parent +// must be a path the sequences' unquoted shell commands can carry, which any +// default temporary directory is. +func (h *directModeHost) detectDirectMode(namespace string, services []string) directModeDecision { + if len(services) == 0 { + return directModeDecision{reason: "no database was named to run tools for"} + } + + content, err := os.ReadFile(h.namespaceFile) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return directModeDecision{reason: fmt.Sprintf("this process is not running as a Kubernetes pod (no %s)", h.namespaceFile)} + } + + return directModeDecision{reason: fmt.Sprintf("the pod's namespace could not be read from %s: %v", h.namespaceFile, err)} + } + + own := strings.TrimSpace(string(content)) + if own != namespace { + return directModeDecision{reason: fmt.Sprintf("this pod runs in namespace %q, not in the target namespace %q", own, namespace)} + } + + missing := []string{} + for _, service := range services { + tools := localToolsFor(service) + if tools == nil { + return directModeDecision{reason: fmt.Sprintf("%s is not a database this tool runs clients for", service)} + } + + for _, tool := range tools { + if _, err := h.lookPath(tool); err != nil { + missing = append(missing, tool) + } + } + } + + if len(missing) > 0 { + sort.Strings(missing) + + return directModeDecision{reason: fmt.Sprintf("this pod runs in namespace %q but %s not found on PATH", namespace, describeMissingTools(dedupeSorted(missing)))} + } + + if parent := h.tempDir(); !isShellSafePath(parent) { + return directModeDecision{reason: fmt.Sprintf("the temporary directory %q contains characters the database commands cannot carry unquoted; set TMPDIR to a plain path", parent)} + } + + return directModeDecision{direct: true, reason: fmt.Sprintf("this pod runs in the target namespace %q and finds %s on PATH", namespace, strings.Join(services, " and ")+" tools")} +} + +func describeMissingTools(tools []string) string { + if len(tools) == 1 { + return tools[0] + " is" + } + + return strings.Join(tools, ", ") + " are" +} + +func dedupeSorted(sorted []string) []string { + out := sorted[:0] + for i, value := range sorted { + if i == 0 || value != sorted[i-1] { + out = append(out, value) + } + } + + return out +} + +// isShellSafePath reports whether path can appear unquoted in the sh -c +// commands the sequences build (`rm -rf /scratch/...`), once translated. +func isShellSafePath(path string) bool { + if path == "" { + return false + } + + for i := 0; i < len(path); i++ { + if !isPathByte(path[i]) { + return false + } + } + + return true +} + +// isPathByte is a byte that can continue a path token. '+' is included so that +// base64 text, which the CA write carries in an argument, can never present +// the scratch mount path as a token of its own (see translate). +func isPathByte(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || + c == '_' || c == '.' || c == '-' || c == '/' || c == '+' +} + +// decideDirectMode settles, once per database, whether this run executes the +// database tools in its own process, and logs the answer at info. +// +// It is per database and remembered, so the probe, the capture and the restore +// of one database all run in the same place: a later call for a database +// already decided reads the memo. It reports direct only when every database +// asked about is direct, because the caller builds one set of operations for +// all of them. +func (iops *InfrahubOps) decideDirectMode(namespace string, services []string) bool { + if iops.directModeDecisions == nil { + iops.directModeDecisions = map[string]bool{} + } + + undecided := []string{} + for _, service := range services { + if _, decided := iops.directModeDecisions[service]; !decided { + undecided = append(undecided, service) + } + } + + if len(undecided) > 0 { + decision := iops.directHost().detectDirectMode(namespace, undecided) + for _, service := range undecided { + iops.directModeDecisions[service] = decision.direct + } + + if decision.direct { + logrus.Infof("Running the external database tools in this process, with no transient Job: %s", decision.reason) + } else { + logrus.Infof("Running the external database tools in a transient Job, not in this process: %s", decision.reason) + } + } + + if len(services) == 0 { + return false + } + + for _, service := range services { + if !iops.directModeDecisions[service] { + return false + } + } + + return true +} + +// --------------------------------------------------------------------------- +// The local workload +// --------------------------------------------------------------------------- + +// localWorkload is the stand-in for one transient Job when the tools run in +// this process: a run-specific scratch directory, the credentials the Job +// would have received from a Secret, and the host it runs children on. +type localWorkload struct { + service string + role workloadRole + scratch string + // credentials reach the child only through its environment (FR-014): they + // are appended to cmd.Env and never to an argument. + credentials map[string]string + host *directModeHost +} + +// translate maps the Job's scratch mount path onto this workload's directory +// wherever it appears as a path of its own: a whole argument, the start of one +// (`/scratch/infrahubops`), or a token inside one (`--temp-path=/scratch/x`, +// `rm -rf /scratch/a /scratch/b`, `'/scratch/ca.pem'`). An occurrence that is +// part of a longer path (`/data/scratch`, `/scratchpad`) is left alone. +func (w *localWorkload) translate(value string) string { + if !strings.Contains(value, transientScratchPath) { + return value + } + + // Both boundary checks read the original string, never the remainder + // after an earlier match: in `/data/scratch/scratch` the second + // occurrence follows `/scratch`, which makes it part of a longer path. + var out strings.Builder + start := 0 + for { + found := strings.Index(value[start:], transientScratchPath) + if found < 0 { + out.WriteString(value[start:]) + + return out.String() + } + + index := start + found + end := index + len(transientScratchPath) + startsToken := index == 0 || !isPathByte(value[index-1]) + endsToken := end == len(value) || value[end] == '/' || !isPathByte(value[end]) + + out.WriteString(value[start:index]) + if startsToken && endsToken { + out.WriteString(w.scratch) + } else { + out.WriteString(transientScratchPath) + } + + start = end + } +} + +// within resolves a path a sequence named in the Job's scratch volume to this +// workload's directory, and refuses any other path: a copy across the +// workload's boundary has no business outside its scratch space, and on this +// host there is nothing else to protect it. +func (w *localWorkload) within(path string) (string, error) { + translated := filepath.Clean(w.translate(path)) + if translated != w.scratch && !strings.HasPrefix(translated, w.scratch+string(filepath.Separator)) { + return "", fmt.Errorf("refusing to copy %s: it is not in the scratch space of the %s %s run in this process", path, w.service, w.role) + } + + return translated, nil +} + +// imageToolEnvironment is the NEO4J_* variables the image itself sets so that +// neo4j-admin and cypher-shell find their jars, configuration and log +// directory. The Job's container has them too, so the child keeps them. +var imageToolEnvironment = map[string]bool{ + "NEO4J_HOME": true, + "NEO4J_CONF": true, + "NEO4J_LOGS": true, +} + +// clientReadsOnItsOwn reports whether a variable is one the database clients +// or their JVM read without being told to: libpq's PG* (PGSERVICE, +// PGPASSFILE, PGSSLMODE...), the Neo4j tools' NEO4J_* (NEO4J_USERNAME, +// NEO4J_ADDRESS...) other than the ones the image sets, and the JVM's option +// variables. The Job's container receives none of them from this process: the +// manifest sets only the credentials, and the sequence sets the rest. +func clientReadsOnItsOwn(key string) bool { + switch { + case imageToolEnvironment[key]: + return false + case strings.HasPrefix(key, "PG"), strings.HasPrefix(key, "NEO4J_"): + return true + } + + switch key { + case "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS", "JAVA_OPTS": + return true + } + + return false +} + +// environment is the child's environment, built to match what the Job's +// container would have: this process's own minus every variable a client reads +// on its own (clientReadsOnItsOwn), then the credentials, then what the +// sequence asked for. Later entries win, which os/exec guarantees for +// duplicate keys. +func (w *localWorkload) environment(extra map[string]string) []string { + parent := os.Environ() + env := make([]string, 0, len(parent)+len(w.credentials)+len(extra)) + for _, entry := range parent { + key, _, _ := strings.Cut(entry, "=") + if clientReadsOnItsOwn(key) { + continue + } + env = append(env, entry) + } + + for _, key := range sortedKeys(w.credentials) { + env = append(env, key+"="+w.credentials[key]) + } + + for _, key := range sortedKeys(extra) { + env = append(env, key+"="+w.translate(extra[key])) + } + + return env +} + +func sortedKeys(values map[string]string) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + + return keys +} + +// prepare builds the child for one command, with its argument vector +// translated and its environment carrying the credentials. +func (w *localWorkload) prepare(ctx context.Context, command []string, opts *ExecOptions) (*exec.Cmd, error) { + if len(command) == 0 { + return nil, fmt.Errorf("no command was given to run for the %s %s", w.service, w.role) + } + + args := make([]string, len(command)) + for i, arg := range command { + args[i] = w.translate(arg) + } + + var extra map[string]string + if opts != nil { + extra = opts.Env + } + + cmd := w.host.command(ctx, args[0], args[1:]...) + cmd.Env = w.environment(extra) + cmd.Dir = w.scratch + // Many commands are `sh -c` pipelines: cancelling has to stop the tools + // the shell started, not only the shell. + killProcessGroupOnCancel(cmd) + cmd.WaitDelay = commandWaitDelay + + return cmd, nil +} + +// run is the bounded exec: stdout and stderr apart, and a *timeoutError when +// the bound expired, which is what execBounded's callers read either way. +func (w *localWorkload) run(ctx context.Context, bound time.Duration, command []string, opts *ExecOptions) (string, string, error) { + cctx, cancel := context.WithTimeout(ctx, bound) + defer cancel() + + cmd, err := w.prepare(cctx, command, opts) + if err != nil { + return "", "", err + } + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + out, errOut := strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()) + if err != nil && errors.Is(cctx.Err(), context.DeadlineExceeded) { + return out, errOut, &timeoutError{timeout: bound} + } + + return out, errOut, err +} + +// stream is the streaming exec: the caller reads stdout and then calls wait. +// It carries no bound of its own, like the Job path's stream: its callers hold +// an idle deadline over the reader. +func (w *localWorkload) stream(command []string, opts *ExecOptions) (io.ReadCloser, func() error, error) { + ctx, cancel := context.WithCancel(context.Background()) + + cmd, err := w.prepare(ctx, command, opts) + if err != nil { + cancel() + + return nil, nil, err + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + cancel() + + return nil, nil, err + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + cancel() + + return nil, nil, err + } + + wait := func() error { + defer cancel() + + return withStderr(cmd.Wait(), stderr.String()) + } + + return cancelOnClose{ReadCloser: stdout, cancel: cancel}, wait, nil +} + +// cancelOnClose is a child's stdout whose Close also stops the child. The +// reader is closed before the child is waited for, either because the stream +// ended or because its idle timeout or deadline expired; in the second case a +// child blocked in database I/O never notices the closed pipe, and waiting for +// it would hold the run past the bound that just expired. +type cancelOnClose struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (c cancelOnClose) Close() error { + c.cancel() + + return c.ReadCloser.Close() +} + +// copyOut takes an artifact out of the scratch directory to the run's own +// staging, the counterpart of `kubectl cp pod:src dest`: dest becomes the file +// or directory src was. A rename where both are on one filesystem, which they +// are by default; a copy otherwise. +func (w *localWorkload) copyOut(ctx context.Context, src, dest string) error { + from, err := w.within(src) + if err != nil { + return err + } + + if _, err := os.Stat(dest); errors.Is(err, fs.ErrNotExist) { + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return err + } + + if err := os.Rename(from, dest); err == nil { + return nil + } + } + + return copyTreeContext(ctx, from, dest) +} + +// copyIn places a file from the run's own staging in the scratch directory. +func (w *localWorkload) copyIn(ctx context.Context, src, dest string) error { + to, err := w.within(dest) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil { + return err + } + + return copyFileContext(ctx, src, to) +} + +// requireFree refuses an operation that needs more scratch space than the +// directory's filesystem has free. what names the content, need is the bytes +// it needs, and remedy is what the operator can change. +func (w *localWorkload) requireFree(what string, need int64, remedy string) error { + free, err := w.host.freeSpace(w.scratch) + if err != nil { + return fmt.Errorf("cannot check whether %s fits the free space under %s: %w", what, w.scratch, err) + } + + if need > 0 && uint64(need) > free { + return fmt.Errorf("%s needs %s, and the filesystem holding %s has %s free: %s. Nothing was changed", + what, formatBytes(need), filepath.Dir(w.scratch), formatBytes(int64(min(free, uint64(1)<<62))), remedy) + } + + return nil +} + +// release removes the scratch directory and everything in it. It is called on +// every exit path: by the sequence when it gives the workload back, and by the +// run's deferred releaseTransientWorkloads for anything still registered. +func (w *localWorkload) release() { + if err := os.RemoveAll(w.scratch); err != nil { + logrus.Warnf("Could not remove the scratch directory %s of the %s %s run in this process: %v", w.scratch, w.service, w.role, err) + } +} + +// copyTreeContext copies a file or a directory tree to dest, which becomes +// src, and stops with ctx's error once ctx is done. +func copyTreeContext(ctx context.Context, src, dest string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + + if !info.IsDir() { + return copyFileContext(ctx, src, dest) + } + + if err := os.MkdirAll(dest, info.Mode()); err != nil { + return err + } + + entries, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + if err := copyTreeContext(ctx, filepath.Join(src, entry.Name()), filepath.Join(dest, entry.Name())); err != nil { + return err + } + } + + return nil +} + +// copyFileContext is copyFile, stopping with ctx's error between reads once +// ctx is done. +func copyFileContext(ctx context.Context, src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + srcInfo, err := srcFile.Stat() + if err != nil { + return err + } + + dstFile, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, srcInfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, contextReader{ctx: ctx, reader: srcFile}) + + return err +} + +// contextReader is a reader that fails with ctx's error once ctx is done. +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + + return r.reader.Read(p) +} + +// boundedLocalCopy runs a copy in this process under bound, as the Job path's +// copy runs under it. The copy stops between reads once the bound expires; +// a copy stalled inside one read or write cannot be interrupted, so the run is +// told of the timeout without waiting for it. +func boundedLocalCopy(bound time.Duration, run func(ctx context.Context) error) error { + ctx, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + + done := make(chan error, 1) + go func() { done <- run(ctx) }() + + select { + case err := <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// --------------------------------------------------------------------------- +// Binding the sequences to this process +// --------------------------------------------------------------------------- + +// localWorkloadFor is the local workload an execution target names, or nil for +// a real pod or no target at all. It is the one question each seam asks. +func (iops *InfrahubOps) localWorkloadFor(target string) *localWorkload { + if !strings.HasPrefix(target, localTargetPrefix) { + return nil + } + + return iops.localWorkloads[target] +} + +// localWorkloadForOptions is localWorkloadFor for the exec seams. +func (iops *InfrahubOps) localWorkloadForOptions(opts *ExecOptions) *localWorkload { + if opts == nil { + return nil + } + + return iops.localWorkloadFor(strings.TrimSpace(opts.Pod)) +} + +// startLocalWorkload creates the scratch directory for one role and registers +// it, returning the execution target the sequences will name and how to give +// it back. +func (iops *InfrahubOps) startLocalWorkload(service string, role workloadRole) (string, *localWorkload, func(), error) { + credentials, err := transientCredentials(iops.config, service) + if err != nil { + return "", nil, nil, err + } + + host := iops.directHost() + + // The scratch path is both the child's working directory and a path in its + // arguments, so it has to be absolute: from a relative TMPDIR, an argument + // naming scratch/x would resolve to scratch/scratch/x once the child is in + // it. + parent, err := filepath.Abs(host.tempDir()) + if err != nil { + return "", nil, nil, fmt.Errorf("cannot resolve the directory for the scratch space of the %s %s run in this process: %w", service, role, err) + } + + scratch, err := os.MkdirTemp(parent, fmt.Sprintf("%s-%s-%s-", transientObjectPrefix, role, service)) + if err != nil { + return "", nil, nil, fmt.Errorf("cannot create the scratch directory for the %s %s run in this process: %w", service, role, err) + } + + workload := &localWorkload{service: service, role: role, scratch: scratch, credentials: credentials, host: host} + + iops.localWorkloadSeq++ + target := fmt.Sprintf("%s%s-%s-%d", localTargetPrefix, role, service, iops.localWorkloadSeq) + + if iops.localWorkloads == nil { + iops.localWorkloads = map[string]*localWorkload{} + } + iops.localWorkloads[target] = workload + + return target, workload, func() { iops.releaseLocalWorkload(target) }, nil +} + +// releaseLocalWorkload gives back one local workload. Safe to call twice. +func (iops *InfrahubOps) releaseLocalWorkload(target string) { + workload, held := iops.localWorkloads[target] + if !held { + return + } + + delete(iops.localWorkloads, target) + delete(iops.externalCATrust, target) + workload.release() +} + +// releaseLocalWorkloads gives back every local workload this run still holds: +// the backstop on every exit path, as ReleaseTransientWorkloads is for Jobs. +func (iops *InfrahubOps) releaseLocalWorkloads() { + targets := make([]string, 0, len(iops.localWorkloads)) + for target := range iops.localWorkloads { + targets = append(targets, target) + } + + for _, target := range targets { + iops.releaseLocalWorkload(target) + } +} + +// localCaptureOps are externalCaptureOps served by this process. They create +// nothing in the namespace, so none of the Job path's steps run before them: +// no run identifier is minted, no stray is reaped, no Secret or Job exists. +func (iops *InfrahubOps) localCaptureOps() externalCaptureOps { + return externalCaptureOps{ + startProbe: func(service string, _ int) (string, func(), error) { + target, _, release, err := iops.startLocalWorkload(service, workloadRoleProbe) + + return target, release, err + }, + startCapture: func(service string, storeSizeBytes int64) (string, func(), error) { + // FR-022 unchanged: the size is derived from the store, or the + // override, or the run stops naming the override. What changes is + // what it is compared with: the free space this pod has, where the + // Job path would size a volume. + size, err := resolveScratchSize(service, iops.config.ExternalDB.ScratchSize, storeSizeBytes) + if err != nil { + return "", nil, err + } + + need, err := parseStorageQuantity(size) + if err != nil { + return "", nil, fmt.Errorf("cannot read the scratch size %s for the %s capture: %w", size, service, err) + } + + target, workload, release, err := iops.startLocalWorkload(service, workloadRoleCapture) + if err != nil { + return "", nil, err + } + + if err := workload.requireFree("the "+service+" capture's scratch space", need, + "free space there, point TMPDIR at a larger volume, or supply --external-db-scratch-size if the derived size is wrong"); err != nil { + release() + + return "", nil, err + } + + return target, release, nil + }, + startRestore: func(service string) (string, func(), error) { + target, _, release, err := iops.startLocalWorkload(service, workloadRoleRestore) + + return target, release, err + }, + probe: iops.probeExternalDatabase, + } +} diff --git a/src/internal/app/external_local_statfs.go b/src/internal/app/external_local_statfs.go new file mode 100644 index 00000000..c047f933 --- /dev/null +++ b/src/internal/app/external_local_statfs.go @@ -0,0 +1,17 @@ +//go:build linux || darwin + +package app + +import "syscall" + +// localFreeSpace is the space an unprivileged writer can still use on the +// filesystem holding path: available blocks, not free blocks, because the +// reserve for root is not this process's to spend. +func localFreeSpace(path string) (uint64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, err + } + + return uint64(stat.Bavail) * uint64(stat.Bsize), nil //nolint:unconvert // Bsize is uint32 on darwin and int64 on linux +} diff --git a/src/internal/app/external_local_statfs_other.go b/src/internal/app/external_local_statfs_other.go new file mode 100644 index 00000000..b637b23e --- /dev/null +++ b/src/internal/app/external_local_statfs_other.go @@ -0,0 +1,15 @@ +//go:build !linux && !darwin + +package app + +import ( + "fmt" + "runtime" +) + +// localFreeSpace has no implementation off Linux and macOS. Direct mode is +// reached only inside a Kubernetes pod, which is Linux, so this answer is the +// refusal of a check rather than a path anyone takes. +func localFreeSpace(string) (uint64, error) { + return 0, fmt.Errorf("reading free disk space is not supported on %s", runtime.GOOS) +} diff --git a/src/internal/app/external_local_test.go b/src/internal/app/external_local_test.go new file mode 100644 index 00000000..a6028cb1 --- /dev/null +++ b/src/internal/app/external_local_test.go @@ -0,0 +1,718 @@ +package app + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// directTestHost is a fake host for direct mode: a namespace file in a +// temporary directory, a PATH lookup that answers from a set, a scratch parent +// of the test's own, and a configurable amount of free space. Children are real +// processes; command records what reached each one. +type directTestHost struct { + *directModeHost + + mu sync.Mutex + argvs [][]string +} + +func newDirectTestHost(t *testing.T, namespace string, missing ...string) *directTestHost { + t.Helper() + + nsFile := filepath.Join(t.TempDir(), "namespace") + if namespace != "" { + if err := os.WriteFile(nsFile, []byte(namespace+"\n"), 0o600); err != nil { + t.Fatalf("writing the namespace file = %v", err) + } + } + + absent := map[string]bool{} + for _, tool := range missing { + absent[tool] = true + } + + scratchParent := t.TempDir() + h := &directTestHost{} + h.directModeHost = &directModeHost{ + namespaceFile: nsFile, + lookPath: func(file string) (string, error) { + if absent[file] { + return "", exec.ErrNotFound + } + + return "/usr/bin/" + file, nil + }, + tempDir: func() string { return scratchParent }, + freeSpace: func(string) (uint64, error) { return 1 << 40, nil }, + command: func(ctx context.Context, name string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, args...) + h.mu.Lock() + h.argvs = append(h.argvs, append([]string{name}, args...)) + h.mu.Unlock() + + return cmd + }, + } + + return h +} + +// scratchEntries lists what is left under the scratch parent. +func (h *directTestHost) scratchEntries(t *testing.T) []string { + t.Helper() + + entries, err := os.ReadDir(h.tempDir()) + if err != nil { + t.Fatalf("listing the scratch parent = %v", err) + } + + names := []string{} + for _, entry := range entries { + names = append(names, entry.Name()) + } + + return names +} + +// writeFakeTools puts scripts named after the database tools in a directory at +// the front of PATH, alongside a kubectl that records every call and fails. +// Each fake tool appends its argv and the credential variables it sees to two +// logs, so a test can tell where the credentials travelled. +func writeFakeTools(t *testing.T, tools map[string]string) (argvLog, envLog, kubectlLog string) { + t.Helper() + + bin := t.TempDir() + logs := t.TempDir() + argvLog = filepath.Join(logs, "argv") + envLog = filepath.Join(logs, "env") + kubectlLog = filepath.Join(logs, "kubectl") + + write := func(name, body string) { + script := fmt.Sprintf("#!/bin/sh\nprintf '%%s %%s\\n' %q \"$*\" >> %q\nprintf 'PGUSER=%%s PGPASSWORD=%%s NEO4J_USERNAME=%%s NEO4J_PASSWORD=%%s\\n' \"$PGUSER\" \"$PGPASSWORD\" \"$NEO4J_USERNAME\" \"$NEO4J_PASSWORD\" >> %q\n%s\n", + name, argvLog, envLog, body) + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0o755); err != nil { + t.Fatalf("writing fake %s = %v", name, err) + } + } + + for name, body := range tools { + write(name, body) + } + + kubectl := fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' \"$*\" >> %q\nexit 1\n", kubectlLog) + if err := os.WriteFile(filepath.Join(bin, "kubectl"), []byte(kubectl), 0o755); err != nil { + t.Fatalf("writing the kubectl stub = %v", err) + } + + t.Setenv("PATH", bin+string(os.PathListSeparator)+"/usr/bin"+string(os.PathListSeparator)+"/bin") + + return argvLog, envLog, kubectlLog +} + +func readLog(t *testing.T, path string) string { + t.Helper() + + content, err := os.ReadFile(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("reading %s = %v", path, err) + } + + return string(content) +} + +// newDirectModeOps is an InfrahubOps on a Kubernetes backend in namespace +// "infrahub", with credentials for both databases and the given host. +func newDirectModeOps(host *directTestHost) *InfrahubOps { + iops := NewInfrahubOps() + iops.backend = newTestKubernetesBackend() + iops.directModeHost = host.directModeHost + iops.config.PostgresUsername = "prefect-user" + iops.config.PostgresPassword = "pg-s3cr3t-value" + iops.config.Neo4jUsername = "neo4j-user" + iops.config.Neo4jPassword = "neo4j-s3cr3t-value" + + return iops +} + +// TestDetectDirectMode is FR-032's detection: the namespace file must name the +// target namespace and every tool must be on PATH, and each refusal says why. +func TestDetectDirectMode(t *testing.T) { + tests := []struct { + name string + namespace string + missing []string + services []string + wantDirect bool + wantReason string + }{ + {name: "a pod in the target namespace with every tool", namespace: "infrahub", services: []string{serviceNeo4j, serviceTaskManagerDB}, wantDirect: true, wantReason: "target namespace"}, + {name: "no namespace file: not a pod", namespace: "", services: []string{serviceNeo4j}, wantReason: "not running as a Kubernetes pod"}, + {name: "a pod in another namespace", namespace: "monitoring", services: []string{serviceNeo4j}, wantReason: `namespace "monitoring", not in the target namespace "infrahub"`}, + {name: "a missing tool", namespace: "infrahub", missing: []string{"cypher-shell"}, services: []string{serviceNeo4j}, wantReason: "cypher-shell is not found on PATH"}, + {name: "two missing tools", namespace: "infrahub", missing: []string{"pg_dump", "psql"}, services: []string{serviceTaskManagerDB}, wantReason: "pg_dump, psql are not found on PATH"}, + {name: "a tool only the other database needs is not required", namespace: "infrahub", missing: []string{"neo4j-admin"}, services: []string{serviceTaskManagerDB}, wantDirect: true}, + {name: "no database named", namespace: "infrahub", wantReason: "no database"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + host := newDirectTestHost(t, tt.namespace, tt.missing...) + + decision := host.detectDirectMode("infrahub", tt.services) + if decision.direct != tt.wantDirect { + t.Errorf("direct = %v, want %v (reason %q)", decision.direct, tt.wantDirect, decision.reason) + } + if !strings.Contains(decision.reason, tt.wantReason) { + t.Errorf("reason = %q, want it to contain %q", decision.reason, tt.wantReason) + } + }) + } + + t.Run("a scratch parent the shell commands cannot carry", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + host.tempDir = func() string { return "/tmp/with space" } + + decision := host.detectDirectMode("infrahub", []string{serviceNeo4j}) + if decision.direct || !strings.Contains(decision.reason, "TMPDIR") { + t.Errorf("decision = %+v, want not direct, naming TMPDIR", decision) + } + }) +} + +// TestLocalWorkloadTranslate maps the Job's scratch mount onto the run's +// directory wherever it is a path of its own, and nowhere else. +func TestLocalWorkloadTranslate(t *testing.T) { + w := &localWorkload{scratch: "/tmp/xdb-1"} + + tests := map[string]string{ + "/scratch": "/tmp/xdb-1", + "/scratch/infrahubops": "/tmp/xdb-1/infrahubops", + "--temp-path=/scratch/staging": "--temp-path=/tmp/xdb-1/staging", + "rm -rf /scratch/a /scratch/b": "rm -rf /tmp/xdb-1/a /tmp/xdb-1/b", + "-C /scratch /scratch/x": "-C /tmp/xdb-1 /tmp/xdb-1/x", + "base64 -d > '/scratch/ca.pem'": "base64 -d > '/tmp/xdb-1/ca.pem'", + "-Djavax.net.ssl.trustStore=/scratch/x": "-Djavax.net.ssl.trustStore=/tmp/xdb-1/x", + "/data/scratch/x": "/data/scratch/x", + "/data/scratch/scratch": "/data/scratch/scratch", + "/data/scratch/scratch/x": "/data/scratch/scratch/x", + "/scratch/scratch": "/tmp/xdb-1/scratch", + "/data/scratch /scratch": "/data/scratch /tmp/xdb-1", + "/scratchpad": "/scratchpad", + "abc+/scratch+def": "abc+/scratch+def", + "no path at all": "no path at all", + } + + for in, want := range tests { + if got := w.translate(in); got != want { + t.Errorf("translate(%q) = %q, want %q", in, got, want) + } + } +} + +// TestLocalWorkloadEnvironmentDropsWhatClientsReadOnTheirOwn holds the child's +// environment to what the Job's container would have: a PGSERVICE or +// PGPASSFILE in this process's environment would point libpq at a connection +// or a password the run never chose, so neither reaches the child, while the +// image's own tool variables, the credentials and the sequence's variables do. +func TestLocalWorkloadEnvironmentDropsWhatClientsReadOnTheirOwn(t *testing.T) { + t.Setenv("PGSERVICE", "inherited-service") + t.Setenv("PGPASSFILE", "/inherited/.pgpass") + t.Setenv("NEO4J_USERNAME", "inherited-user") + t.Setenv("JAVA_TOOL_OPTIONS", "-Dinherited=true") + t.Setenv("NEO4J_HOME", "/opt/neo4j") + t.Setenv("NEO4J_CONF", "/opt/neo4j/conf") + t.Setenv("NEO4J_LOGS", "/opt/neo4j/logs") + t.Setenv("JAVA_HOME", "/usr/lib/jvm/java-21-openjdk") + + w := &localWorkload{ + scratch: "/tmp/xdb-1", + credentials: map[string]string{"PGUSER": "prefect-user", "PGPASSWORD": "pg-s3cr3t-value"}, + } + env := w.environment(map[string]string{"PGSSLMODE": "require", "JAVA_OPTS": "-Dx=/scratch/ts"}) + + values := map[string][]string{} + for _, entry := range env { + key, value, _ := strings.Cut(entry, "=") + values[key] = append(values[key], value) + } + + for _, key := range []string{"PGSERVICE", "PGPASSFILE", "NEO4J_USERNAME", "JAVA_TOOL_OPTIONS"} { + if got, ok := values[key]; ok { + t.Errorf("%s reached the child as %q, want it dropped", key, got) + } + } + + want := map[string]string{ + "PATH": os.Getenv("PATH"), + "NEO4J_HOME": "/opt/neo4j", + "NEO4J_CONF": "/opt/neo4j/conf", + "NEO4J_LOGS": "/opt/neo4j/logs", + "JAVA_HOME": "/usr/lib/jvm/java-21-openjdk", + "PGUSER": "prefect-user", + "PGPASSWORD": "pg-s3cr3t-value", + "PGSSLMODE": "require", + "JAVA_OPTS": "-Dx=/tmp/xdb-1/ts", + } + for key, value := range want { + if got := values[key]; len(got) != 1 || got[0] != value { + t.Errorf("%s = %q, want exactly [%q]", key, got, value) + } + } +} + +// TestDirectModeCreatesNothingInTheNamespace is T171: when direct mode is +// taken the gate hands back operations served by this process, and not one +// kubectl call is made — no reaper listing, no Job, no Secret. A development +// build with no --external-db-image is not refused, because no image is run. +func TestDirectModeCreatesNothingInTheNamespace(t *testing.T) { + withBuildVersion(t, "") + _, _, kubectlLog := writeFakeTools(t, map[string]string{}) + + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + + ops, err := iops.externalCaptureOps(serviceTaskManagerDB) + if err != nil { + t.Fatalf("externalCaptureOps() = %v, want direct-mode operations", err) + } + + target, release, err := ops.startProbe(serviceTaskManagerDB, 1) + if err != nil { + t.Fatalf("startProbe() = %v", err) + } + if !strings.HasPrefix(target, localTargetPrefix) { + t.Errorf("target = %q, want a local target", target) + } + release() + + if iops.externalRun != "" || iops.transientStraysReaped { + t.Errorf("run id %q, reaped %v: want neither — nothing is created, so nothing is labelled or swept", iops.externalRun, iops.transientStraysReaped) + } + + // Positive control for the log: the stub records when it is called. + if calls := readLog(t, kubectlLog); calls != "" { + t.Errorf("kubectl was called in direct mode:\n%s\nwant no call at all", calls) + } + if err := exec.Command("kubectl", "version").Run(); err == nil { + t.Fatal("the kubectl stub exited 0, want it to fail") + } + if !strings.Contains(readLog(t, kubectlLog), "version") { + t.Fatal("the kubectl stub did not record its own call; the assertion above observed nothing") + } +} + +// TestDirectModeOutsideTheNamespaceKeepsTheJobPath is the other half: a run +// that is not a pod in the namespace goes on to the Job path, whose first +// steps are the image refusal (a development build here) — so direct mode was +// not taken. +func TestDirectModeOutsideTheNamespaceKeepsTheJobPath(t *testing.T) { + withBuildVersion(t, "") + writeFakeTools(t, map[string]string{}) + + host := newDirectTestHost(t, "") + iops := newDirectModeOps(host) + + _, err := iops.externalCaptureOps(serviceTaskManagerDB) + if err == nil || !strings.Contains(err.Error(), "--external-db-image") { + t.Fatalf("externalCaptureOps() = %v, want the Job path's development-build refusal", err) + } +} + +// TestDirectModeIsStillRefusedForCollect is ADR 0003: infrahub-collect is +// turned back before the direct-mode decision, so it captures in neither mode. +func TestDirectModeIsStillRefusedForCollect(t *testing.T) { + writeFakeTools(t, map[string]string{}) + + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + iops.forbidExternalCapture() + + if _, err := iops.externalCaptureOps(serviceNeo4j); err == nil || !strings.Contains(err.Error(), "infrahub-collect") { + t.Fatalf("externalCaptureOps() = %v, want the collect refusal", err) + } + if len(iops.directModeDecisions) != 0 { + t.Errorf("direct mode was decided (%v) for a run that may not capture", iops.directModeDecisions) + } +} + +// TestDirectModeProbeRunsLocallyWithCredentialsInTheEnvironment is T170 and +// FR-014 through the real probe sequence: the PostgreSQL probe runs pg_dump and +// psql as children of this process, they read the credentials from their +// environment, and no argument any child received carries a credential. +func TestDirectModeProbeRunsLocallyWithCredentialsInTheEnvironment(t *testing.T) { + argvLog, envLog, kubectlLog := writeFakeTools(t, map[string]string{ + "pg_dump": `echo "pg_dump (PostgreSQL) 18.1"`, + "psql": `case "$*" in *server_version*) echo "18.1" ;; *) echo "1048576" ;; esac`, + }) + + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + + ops, err := iops.externalCaptureOps(serviceTaskManagerDB) + if err != nil { + t.Fatalf("externalCaptureOps() = %v", err) + } + + target, release, err := ops.startProbe(serviceTaskManagerDB, 1) + if err != nil { + t.Fatalf("startProbe() = %v", err) + } + defer release() + + endpoint := &DatabaseEndpoint{Service: serviceTaskManagerDB, Hosts: []HostPort{{Host: "db.example.net", Port: 5432}}, Database: "prefect"} + facts, err := ops.probe(target, endpoint, externalTLSDecision{Service: serviceTaskManagerDB, Encrypt: true, OptedOut: true}) + if err != nil { + t.Fatalf("probe() = %v", err) + } + if facts.UtilityVersion != "18.1" || facts.ServerVersion != "18.1" || facts.StoreSizeBytes != 1048576 { + t.Errorf("facts = %+v, want utility 18.1, server 18.1, size 1048576", facts) + } + + for _, secret := range []string{"pg-s3cr3t-value", "neo4j-s3cr3t-value"} { + if argv := readLog(t, argvLog); strings.Contains(argv, secret) { + t.Errorf("a child's argv carried a credential:\n%s", argv) + } + for _, argv := range host.argvs { + if strings.Contains(strings.Join(argv, " "), secret) { + t.Errorf("argv %q carries a credential", argv) + } + } + } + if env := readLog(t, envLog); !strings.Contains(env, "PGUSER=prefect-user PGPASSWORD=pg-s3cr3t-value") { + t.Errorf("the children's environment = %q, want the PostgreSQL credentials there", env) + } + if len(host.argvs) < 3 { + t.Errorf("ran %d children, want the utility version, the server version and the size", len(host.argvs)) + } + if calls := readLog(t, kubectlLog); calls != "" { + t.Errorf("kubectl was called:\n%s", calls) + } +} + +// TestDirectModeScratchIsRemovedOnEveryExitPath covers a capture that succeeds +// (artifact copied out, scratch removed on release), one that fails (scratch +// removed on release), one whose scratch does not fit (never left behind), and +// the run's backstop for a workload nobody released. +func TestDirectModeScratchIsRemovedOnEveryExitPath(t *testing.T) { + writeFakeTools(t, map[string]string{}) + + t.Run("success: the artifact is copied out and the scratch removed", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + ops, err := iops.externalCaptureOps(serviceTaskManagerDB) + if err != nil { + t.Fatalf("externalCaptureOps() = %v", err) + } + + target, release, err := ops.startCapture(serviceTaskManagerDB, 1<<20) + if err != nil { + t.Fatalf("startCapture() = %v", err) + } + + if _, err := iops.execBoundedAgainst(externalDBControlBound(iops.config), "writing", "test", serviceTaskManagerDB, + []string{"sh", "-c", "printf dump-bytes > " + externalPostgresDumpFile}, &ExecOptions{Pod: target}); err != nil { + t.Fatalf("writing into scratch = %v", err) + } + + dest := filepath.Join(t.TempDir(), "backup", "prefect.dump") + if err := iops.copyFromTransientWorkload(target, externalPostgresDumpFile, dest); err != nil { + t.Fatalf("copyFromTransientWorkload() = %v", err) + } + if content, err := os.ReadFile(dest); err != nil || string(content) != "dump-bytes" { + t.Errorf("copied artifact = %q, %v; want %q", content, err, "dump-bytes") + } + + release() + if left := host.scratchEntries(t); len(left) != 0 { + t.Errorf("scratch left after release: %v", left) + } + }) + + t.Run("failure: a failed command, then release, leaves nothing", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + target, release, err := ops.startCapture(serviceTaskManagerDB, 1<<20) + if err != nil { + t.Fatalf("startCapture() = %v", err) + } + + _, err = iops.execBoundedAgainst(externalDBControlBound(iops.config), "failing", "test", serviceTaskManagerDB, + []string{"sh", "-c", "echo partial > /scratch/partial; echo boom >&2; exit 3"}, &ExecOptions{Pod: target}) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Fatalf("err = %v, want the failure with its stderr", err) + } + + release() + if left := host.scratchEntries(t); len(left) != 0 { + t.Errorf("scratch left after a failure: %v", left) + } + }) + + t.Run("a capture that does not fit is refused and leaves nothing", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + host.freeSpace = func(string) (uint64, error) { return 1 << 30, nil } + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + _, _, err := ops.startCapture(serviceTaskManagerDB, 100<<30) + if err == nil || !strings.Contains(err.Error(), "TMPDIR") || !strings.Contains(err.Error(), "--external-db-scratch-size") { + t.Fatalf("err = %v, want a refusal naming TMPDIR and the override", err) + } + if left := host.scratchEntries(t); len(left) != 0 { + t.Errorf("scratch left after the refusal: %v", left) + } + }) + + t.Run("an unknown store size still names the override (FR-022)", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + if _, _, err := ops.startCapture(serviceTaskManagerDB, 0); err == nil || !strings.Contains(err.Error(), "--external-db-scratch-size") { + t.Fatalf("err = %v, want the FR-022 refusal", err) + } + }) + + t.Run("the run's backstop removes a workload nobody released", func(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + if _, _, err := ops.startRestore(serviceTaskManagerDB); err != nil { + t.Fatalf("startRestore() = %v", err) + } + if len(host.scratchEntries(t)) != 1 { + t.Fatalf("want one scratch directory while the restore holds it") + } + + iops.releaseTransientWorkloads() + if left := host.scratchEntries(t); len(left) != 0 { + t.Errorf("scratch left after the backstop: %v", left) + } + }) +} + +// TestDirectModeScratchFromARelativeTempDir is a relative TMPDIR: the scratch +// path is the child's working directory and also appears in its arguments, so +// a relative one would resolve beneath itself once the child is in it. +func TestDirectModeScratchFromARelativeTempDir(t *testing.T) { + writeFakeTools(t, map[string]string{}) + t.Chdir(t.TempDir()) + if err := os.Mkdir("tmp", 0o700); err != nil { + t.Fatalf("creating the relative scratch parent = %v", err) + } + + host := newDirectTestHost(t, "infrahub") + host.tempDir = func() string { return "tmp" } + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + target, release, err := ops.startCapture(serviceTaskManagerDB, 1<<20) + if err != nil { + t.Fatalf("startCapture() = %v", err) + } + defer release() + + if scratch := iops.localWorkloadFor(target).scratch; !filepath.IsAbs(scratch) { + t.Errorf("scratch = %q, want an absolute path", scratch) + } + + if _, err := iops.execBoundedAgainst(externalDBControlBound(iops.config), "writing", "test", serviceTaskManagerDB, + []string{"sh", "-c", "printf dump-bytes > " + externalPostgresDumpFile}, &ExecOptions{Pod: target}); err != nil { + t.Fatalf("writing into scratch = %v", err) + } + + dest := filepath.Join(t.TempDir(), "prefect.dump") + if err := iops.copyFromTransientWorkload(target, externalPostgresDumpFile, dest); err != nil { + t.Fatalf("copyFromTransientWorkload() = %v", err) + } + if content, err := os.ReadFile(dest); err != nil || string(content) != "dump-bytes" { + t.Errorf("copied artifact = %q, %v; want %q", content, err, "dump-bytes") + } +} + +// TestDirectModeStreamAbortStopsTheChild is a stream whose deadline expires +// while the child is blocked without writing: closing its stdout does not +// stop it, so Close has to, together with whatever the shell started. +func TestDirectModeStreamAbortStopsTheChild(t *testing.T) { + host := newDirectTestHost(t, "infrahub") + w := &localWorkload{service: serviceTaskManagerDB, role: workloadRoleCapture, scratch: t.TempDir(), host: host.directModeHost} + + stdout, wait, err := w.stream([]string{"sh", "-c", "sleep 30; echo late"}, nil) + if err != nil { + t.Fatalf("stream() = %v", err) + } + + reader := &execReadCloser{reader: stdout, wait: wait, deadline: time.Now().Add(200 * time.Millisecond)} + if _, err := reader.Read(make([]byte, 16)); !errors.Is(err, errStreamDeadline) { + t.Fatalf("Read() = %v, want the stream deadline", err) + } + + start := time.Now() + closed := make(chan error, 1) + go func() { closed <- reader.Close() }() + + select { + case <-closed: + if elapsed := time.Since(start); elapsed > commandWaitDelay+time.Second { + t.Errorf("Close() took %v after the deadline, want the child stopped at once", elapsed) + } + case <-time.After(10 * time.Second): + t.Fatal("Close() still blocked 10s after the stream deadline expired") + } +} + +// TestDirectModeRestoreDumpFitReadsLocalFreeSpace is the restore's fit check in +// direct mode: the free space of the scratch filesystem, not the Job's sized +// volume, and the copy into scratch when it fits. +func TestDirectModeRestoreDumpFitReadsLocalFreeSpace(t *testing.T) { + writeFakeTools(t, map[string]string{}) + + host := newDirectTestHost(t, "infrahub") + host.freeSpace = func(string) (uint64, error) { return 1 << 20, nil } + iops := newDirectModeOps(host) + ops, _ := iops.externalCaptureOps(serviceTaskManagerDB) + + target, release, err := ops.startRestore(serviceTaskManagerDB) + if err != nil { + t.Fatalf("startRestore() = %v", err) + } + defer release() + + if err := iops.refuseUnfittableRestoreDump(target, 2<<20); err == nil || !strings.Contains(err.Error(), "TMPDIR") { + t.Errorf("refuseUnfittableRestoreDump() = %v, want a refusal naming TMPDIR", err) + } + if err := iops.refuseUnfittableRestoreDump(target, 1<<10); err != nil { + t.Errorf("refuseUnfittableRestoreDump() = %v, want nil for a dump that fits", err) + } + + dump := filepath.Join(t.TempDir(), "prefect.dump") + if err := os.WriteFile(dump, []byte("restore-bytes"), 0o600); err != nil { + t.Fatal(err) + } + if err := iops.copyToTransientWorkload(target, dump, externalPostgresRestoreFile); err != nil { + t.Fatalf("copyToTransientWorkload() = %v", err) + } + staged := filepath.Join(iops.localWorkloadFor(target).scratch, filepath.Base(externalPostgresRestoreFile)) + if content, err := os.ReadFile(staged); err != nil || string(content) != "restore-bytes" { + t.Errorf("staged dump = %q, %v; want it in the run's scratch", content, err) + } + + if err := iops.copyToTransientWorkload(target, dump, "/etc/passwd"); err == nil { + t.Error("a copy outside the scratch space was accepted, want a refusal") + } +} + +// TestDirectModeCATrustLandsInScratch is the CA path in direct mode: the +// deployment's certificate is written into the run's scratch directory and +// imported from there by the local keytool, with the trust store path the JVM +// is pointed at translated the same way. +func TestDirectModeCATrustLandsInScratch(t *testing.T) { + argvLog, _, _ := writeFakeTools(t, map[string]string{ + // keytool proves the PEM is where -file says, then creates the store. + "keytool": `file=""; store=""; while [ $# -gt 0 ]; do case "$1" in -file) file="$2"; shift ;; -keystore) store="$2"; shift ;; esac; shift; done; grep -q "BEGIN CERTIFICATE" "$file" && : > "$store"`, + }) + + host := newDirectTestHost(t, "infrahub") + iops := newDirectModeOps(host) + iops.config.ExternalDB.Neo4jTLS.CAFile = "/etc/infrahub/ca.pem" + iops.deploymentCAFiles = map[string]string{"infrahub-server:/etc/infrahub/ca.pem": pemCertificateMarker + "\nMIIB\n-----END CERTIFICATE-----\n"} + + ops, err := iops.externalCaptureOps(serviceNeo4j) + if err != nil { + t.Fatalf("externalCaptureOps() = %v", err) + } + target, release, err := ops.startRestore(serviceNeo4j) + if err != nil { + t.Fatalf("startRestore() = %v", err) + } + scratch := iops.localWorkloadFor(target).scratch + + env := iops.installExternalCATrust(target, externalTLSDecision{Service: serviceNeo4j, Encrypt: true}) + if env == nil { + t.Fatalf("installExternalCATrust() = nil, want the JVM trust environment (keytool log:\n%s)", readLog(t, argvLog)) + } + if _, err := os.Stat(filepath.Join(scratch, filepath.Base(externalCATrustStorePath))); err != nil { + t.Errorf("trust store not in scratch: %v", err) + } + + // The environment as the child receives it points at the scratch copy. + childEnv := strings.Join(iops.localWorkloadFor(target).environment(env), "\n") + if !strings.Contains(childEnv, "-Djavax.net.ssl.trustStore="+filepath.Join(scratch, filepath.Base(externalCATrustStorePath))) { + t.Errorf("JAVA_OPTS reaching the child does not name the scratch trust store:\n%s", childEnv) + } + + release() + if _, err := os.Stat(scratch); !os.IsNotExist(err) { + t.Errorf("scratch %s still exists after release (err %v)", scratch, err) + } +} + +// TestLocalFreeSpaceReadsTheHost is the real statfs read, on the host the test +// runs on: a temporary directory has some space free. +func TestLocalFreeSpaceReadsTheHost(t *testing.T) { + free, err := localFreeSpace(t.TempDir()) + if err != nil { + t.Fatalf("localFreeSpace() = %v", err) + } + if free == 0 { + t.Error("localFreeSpace() = 0, want the free space of the test's temporary directory") + } +} + +// TestDirectModeCopyIsBounded is a direct-mode copy under the bound every +// external operation has: one stalled in a read returns at the bound, and one +// that keeps making progress stops between reads. +func TestDirectModeCopyIsBounded(t *testing.T) { + t.Run("a stalled copy returns at the bound", func(t *testing.T) { + stalled, unblock := io.Pipe() + defer unblock.Close() + + start := time.Now() + err := boundedLocalCopy(100*time.Millisecond, func(context.Context) error { + _, err := io.Copy(io.Discard, stalled) + + return err + }) + if !errors.Is(err, context.DeadlineExceeded) || !isCommandTimeout(err) { + t.Fatalf("err = %v, want the bound's deadline", err) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("returned after %v, want at the 100ms bound", elapsed) + } + }) + + t.Run("a copy making progress stops once the bound expires", func(t *testing.T) { + src := filepath.Join(t.TempDir(), "endless") + if err := os.Symlink("/dev/zero", src); err != nil { + t.Skipf("no /dev/zero: %v", err) + } + dest := filepath.Join(t.TempDir(), "copy") + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + done := make(chan error, 1) + go func() { done <- copyTreeContext(ctx, src, dest) }() + + select { + case err := <-done: + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v, want the deadline", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the copy was still running 5s after its context expired") + } + }) +} diff --git a/src/internal/app/external_requirements_test.go b/src/internal/app/external_requirements_test.go new file mode 100644 index 00000000..36de8e37 --- /dev/null +++ b/src/internal/app/external_requirements_test.go @@ -0,0 +1,282 @@ +package app + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Three requirements had no task naming them and so no test asserting them. +// They are grouped here because what they have in common is that each is a +// promise about a failure — an artefact that cannot be reached, a runtime that +// is not installed, an artefact written by a version that did not know about a +// field — and a promise about a failure is the kind that goes unnoticed when it +// stops being kept. + +// TestRestoreRefusesAnUnreachableArtefactBeforeAnyDestructiveStep is FR-020. +// +// The requirement is an ordering, not a message: where the artefact store +// cannot be reached, the run must refuse *before* any destructive step, and +// name what it could not reach. Getting that ordering wrong is not a worse +// error message — it is a deployment scaled to zero, its transient data wiped, +// and then a restore that never had an artefact to restore from. +func TestRestoreRefusesAnUnreachableArtefactBeforeAnyDestructiveStep(t *testing.T) { + tests := []struct { + name string + storage BackendType + restore func(iops *InfrahubOps) error + }{ + { + name: "a named archive that is not there", + storage: BackendTarball, + restore: func(iops *InfrahubOps) error { + missing := filepath.Join(iops.config.BackupDir, "infrahub_backup_20260101_000000.tar.gz") + + return iops.RestoreBackup(missing, false, false, 0, "", true, false) + }, + }, + { + // --latest against a directory holding nothing: the same absence, + // reached through the entry point that chooses the artefact itself. + name: "no artefact at all under --latest", + storage: BackendTarball, + restore: func(iops *InfrahubOps) error { + return iops.RestoreLatestBackup(false, false, false, 0, "", true, false) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backend := newGatedBackend() + iops := newGatedOps(t, backend, tt.storage) + + err := tt.restore(iops) + if err == nil { + t.Fatal("the restore succeeded with no artefact to restore from, want a refusal") + } + + // Naming the prerequisite: the operator has to be able to tell an + // unreachable artefact from a deployment that would not cooperate. + if !strings.Contains(strings.ToLower(err.Error()), "backup") { + t.Errorf("err = %v, want it to name the artefact it could not reach", err) + } + + // The half that matters. Nothing may have been stopped, started or + // run against the deployment — FR-020's "before any destructive + // step", and constitution Principle II. + if len(backend.stopped) > 0 { + t.Errorf("services stopped = %v, want none: the restore took the deployment down before discovering it had nothing to restore", backend.stopped) + } + if len(backend.started) > 0 { + t.Errorf("services started = %v, want none", backend.started) + } + for _, call := range backend.execs { + t.Errorf("command %q ran against the deployment before the artefact was established", call) + } + }) + } +} + +// TestEnvironmentDetectionNamesTheMissingRuntime is FR-018, and constitution +// Principle III's other half: the binaries are copied onto hosts where the +// runtime they shell out to may simply not be installed, and they must say so +// rather than report the deployment as absent. +// +// "no Infrahub environment detected" is a claim about the customer's cluster. +// When every backend was skipped because its own CLI is missing, nothing ever +// asked the cluster anything, and that message sends the operator to look in +// the wrong place entirely. +// +// It drives the real backends against an empty PATH rather than a substitute +// for them, because the behaviour under test is what happens when the tool is +// genuinely not installed — and a double that reports "not installed" would be +// asserting the test's own belief about how absence is detected. +func TestEnvironmentDetectionNamesTheMissingRuntime(t *testing.T) { + t.Run("neither runtime installed", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + iops := &InfrahubOps{config: &Configuration{}, executor: NewCommandExecutor()} + + _, err := iops.ensureBackend() + if err == nil { + t.Fatal("ensureBackend() = nil error with no runtime on PATH, want a refusal") + } + + for _, want := range []string{"docker", "kubectl", "PATH"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to name %q — what is missing and where it was expected", err, want) + } + } + }) + + t.Run("a runtime that answered is not reported as missing", func(t *testing.T) { + // kubectl present and working, but no Infrahub anywhere. That is a + // statement about the cluster, and it must not be dressed up as a + // missing tool — the operator would go and install what they already + // have. + bin := t.TempDir() + writeStubCommand(t, filepath.Join(bin, "kubectl")) + t.Setenv("PATH", bin) + + iops := &InfrahubOps{config: &Configuration{}, executor: NewCommandExecutor()} + + _, err := iops.ensureBackend() + if err == nil { + t.Fatal("ensureBackend() = nil error, want a refusal") + } + if strings.Contains(err.Error(), "PATH") { + t.Errorf("err = %v, want no claim about a missing tool: kubectl was there and answered", err) + } + }) +} + +// writeStubCommand writes an executable that succeeds and prints nothing, which +// for `kubectl version --client` is a working client and for the namespace +// listing is an empty result. +func writeStubCommand(t *testing.T, path string) { + t.Helper() + + if err := os.WriteFile(path, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("writing the stub command = %v, want nil", err) + } +} + +// TestMetadataCompatibilityHoldsInBothDirections is FR-016 and FR-024. +// +// The guarantee is that an operator never has to do anything for an artefact +// written by a different version of the tool to be restorable, and it has two +// directions. The new-reads-old direction is the one this repository exercises +// by accident every time an older artefact is restored. The old-reads-new +// direction is the one nothing exercises, because the tool that would read it +// is the one that shipped last quarter — so it is pinned here, against a +// declared copy of that older reader's shape (releasedBackupMetadata), which +// knows none of the provenance fields this feature added. +func TestMetadataCompatibilityHoldsInBothDirections(t *testing.T) { + t.Run("the declared metadata version does not move", func(t *testing.T) { + // FR-024 states it directly: additions MUST NOT change the artefact's + // declared metadata version. A previously released tool that compared + // this value would refuse an artefact it can in fact read, so a change + // here is a compatibility break rather than a version bump. + if metadataVersion != 2025111200 { + t.Errorf("metadataVersion = %d, want it unchanged: moving it breaks the artefacts released tools can read (FR-024)", metadataVersion) + } + }) + + t.Run("an old tool reads a new artefact without refusing it", func(t *testing.T) { + var metadata releasedBackupMetadata + if err := json.Unmarshal(newProvenanceArtefact(), &metadata); err != nil { + t.Fatalf("an older reader refused a newer artefact: %v", err) + } + + // Not refusing is half of it. Acting correctly on what it *can* see is + // the other half, and the more easily lost: a reader that silently got + // the component list wrong would restore the wrong set of databases. + if metadata.MetadataVersion != metadataVersion { + t.Errorf("MetadataVersion = %d, want %d", metadata.MetadataVersion, metadataVersion) + } + if metadata.BackupID != "20260101_120000" { + t.Errorf("BackupID = %q", metadata.BackupID) + } + if metadata.Neo4jEdition != "enterprise" { + t.Errorf("Neo4jEdition = %q, want enterprise: the restore's edition decision reads this", metadata.Neo4jEdition) + } + if got := metadata.Components; len(got) != 2 || got[0] != "database" || got[1] != serviceTaskManagerDB { + t.Errorf("Components = %v, want both databases: this is what decides which are restored", got) + } + if metadata.Checksums["neo4j.dump"] != "abc123" { + t.Errorf("Checksums = %v, want the artefact's own", metadata.Checksums) + } + }) + + t.Run("a new tool reads an old artefact that states none of the optional fields", func(t *testing.T) { + // FR-016: an artefact from a previously released version must remain + // restorable without operator action. The fields added since are all + // omitempty, so an older artefact simply does not carry them, and each + // must read as the conservative value rather than as a reason to stop. + oldArtefact := []byte(`{ + "metadata_version": 2025111200, + "backup_id": "20250101_120000", + "created_at": "2025-01-01T12:00:00Z", + "tool_version": "1.0.0", + "infrahub_version": "1.0.0", + "components": ["database"] + }`) + + var metadata BackupMetadata + if err := json.Unmarshal(oldArtefact, &metadata); err != nil { + t.Fatalf("the current reader refused an older artefact: %v", err) + } + + if metadata.Neo4jEdition != "" { + t.Errorf("Neo4jEdition = %q, want it unstated", metadata.Neo4jEdition) + } + if metadata.Encrypted || metadata.Redacted { + t.Errorf("Encrypted/Redacted = %t/%t, want both false: an absent field is not an assertion", metadata.Encrypted, metadata.Redacted) + } + if metadata.Checksums != nil { + t.Errorf("Checksums = %v, want nil rather than an empty map that reads as 'verified nothing'", metadata.Checksums) + } + + // An unstated edition is not a refusal: ResolveRestoreEdition uses the + // edition the target reports, which is what keeps such an artefact + // restorable without operator action. A probe that did not answer is a + // different case: the restore method cannot be chosen, so the restore + // stops before anything is stopped (TestResolveRestoreEdition). + edition, err := NewNeo4jEditionInfo(neo4jEditionCommunity, nil).ResolveRestoreEdition(metadata.Neo4jEdition) + if err != nil { + t.Fatalf("an older artefact's unstated edition stopped the restore: %v", err) + } + if edition != neo4jEditionCommunity { + t.Errorf("ResolveRestoreEdition() = %q, want the edition the target reported", edition) + } + }) +} + +// TestAnExplicitTargetPinsItsBackend is T156. A run that names its deployment +// must act on that deployment or fail: with --k8s-namespace and the namespace +// check refused, detection used to fall through to Docker and pick up an +// unrelated Compose project on the same host (observed in CI), and a restore +// would then have written to it. +func TestAnExplicitTargetPinsItsBackend(t *testing.T) { + bin := t.TempDir() + dockerCalls := filepath.Join(t.TempDir(), "docker-calls") + + // kubectl is installed but the namespace check is refused. + kubectl := "#!/bin/sh\nif [ \"$1\" = version ]; then exit 0; fi\n" + + "echo 'Error from server (Forbidden): pods is forbidden' >&2\nexit 1\n" + // docker records every call and would report a running Compose project. + docker := "#!/bin/sh\necho \"$*\" >> " + dockerCalls + "\necho infrahub-other\nexit 0\n" + for name, body := range map[string]string{"kubectl": kubectl, "docker": docker} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil { + t.Fatalf("writing the %s stub = %v, want nil", name, err) + } + } + t.Setenv("PATH", bin) + + t.Run("--k8s-namespace fails rather than detecting Docker", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{K8sNamespace: "infrahub"}, executor: NewCommandExecutor()} + + backend, err := iops.ensureBackend() + if err == nil { + t.Fatalf("ensureBackend() = %s backend, want the namespace check's failure", backend.Name()) + } + if !strings.Contains(err.Error(), "forbidden") { + t.Errorf("err = %v, want kubectl's own reason carried through", err) + } + if calls, _ := os.ReadFile(dockerCalls); len(calls) != 0 { + t.Errorf("docker was called (%q), want no Docker detection when a namespace is named", calls) + } + }) + + t.Run("no target still detects", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}, executor: NewCommandExecutor()} + _, _ = iops.ensureBackend() + + if calls, _ := os.ReadFile(dockerCalls); len(calls) == 0 { + t.Error("docker was never called, want auto-detection to try it when no target is named") + } + }) +} diff --git a/src/internal/app/external_restore.go b/src/internal/app/external_restore.go new file mode 100644 index 00000000..512a6437 --- /dev/null +++ b/src/internal/app/external_restore.go @@ -0,0 +1,1367 @@ +package app + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// Restoring into a database that lives outside the deployment inverts the data +// path a capture uses, and every difference on this page follows from that one +// fact. +// +// A capture pulls: the transient workload runs the vendor's backup client +// against the server and the bytes come back through the pod. A Neo4j restore +// pushes nothing. There is no remote equivalent of `neo4j-admin database +// restore` — that command needs the server's own store directory — so the only +// documented way to put a backup artifact into a server this tool cannot reach +// the filesystem of is to ask the *server* to fetch it: +// +// CREATE OR REPLACE DATABASE OPTIONS { existingData: 'use', seedURI: 's3://…' } +// +// Three consequences run through everything below (research R2): +// +// - **The database host does the reading, so the artifact has to be somewhere +// it can read from.** That is a customer-side prerequisite this tool cannot +// arrange and must not assume: the run stages the artifact in the object +// store the operator already configured, proves an independent reader can +// address it, and names the remaining prerequisite plainly (FR-007, +// FR-020). What it cannot do is verify the route from the database host, +// which is why the confirmation below exists rather than being optional. +// - **Accepting the seed request proves nothing.** The download and its +// validation happen when the database starts, not when the statement +// returns, and a seed that fails leaves the database offline with a +// `statusMessage` rather than failing the statement. Reporting success on +// the statement's return would be precisely the silent data loss +// constitution Principle II exists to prevent, so the run polls until the +// database is online and fails otherwise (FR-021). +// - **Only the cloud providers are built in.** s3, gs and azb need no +// server-side provider configuration; anything else requires +// `dbms.databases.seed_from_uri_providers` on every server, which this tool +// cannot see. So the staged URI is an object-store one, and a scheme that +// is not is refused before the destructive step rather than after it. +// +// PostgreSQL keeps the shape it already had. `pg_restore` is a client, so the +// external path differs from the internal one in the three places the dump +// path differs: the host it connects to, where its credentials come from, and +// whether it verifies the server's certificate. +// +// The workload is created once, at the gate, and held for the whole run. That +// is not the capture's two-phase shape and it is deliberate: see +// workloadRoleRestore. + +// externalRestore is what a run established about one database outside the +// deployment that it is about to write to, together with the workload the +// writing happens in. +// +// It embeds the same externalSource a capture records — the endpoint, this +// run's certificate decision, and what the probe read — because the questions +// that have to be settled before data moves are the same ones, and FR-006's +// version gate is answered from the same two fields on either path. +type externalRestore struct { + *externalSource + + // Pod is the transient workload's pod, named on every exec and every copy + // this restore makes. It is named rather than registered, for the reason + // ExecOptions.Pod exists: a restore calls StartServices, which replaces the + // resolver's cache wholesale. + Pod string + + // Bound is the timeout on a single restore operation (FR-025). It is the + // operator's --external-db-timeout, because seeding a large database is + // legitimately slow — and because this is the path where a stall happens + // with Infrahub scaled to zero. + Bound time.Duration + + // ClientEnv is what the clients in this pod need beyond their credentials: + // the trust anchor for a Bolt session, or the TLS mode and root + // certificate for libpq. It is resolved once, when the workload is created, + // because resolving it writes material into that pod — and because it is + // read out of infrahub-server and task-manager, which this run scales to + // zero before the destructive step. + ClientEnv map[string]string + + // Seed is the artifact the database server will fetch, established by the + // pre-flight before anything destructive happened (FR-007). Its zero value + // means no pre-flight ran, which the seed refuses to proceed on rather than + // treating as "no seed needed": the whole point of the pre-flight is that + // the destructive step cannot be reached without it. + Seed restoreSeed +} + +// There is deliberately no Release here, and no release function carried on the +// value, which is the one place this type differs from externalCapture rather +// than mirroring it. +// +// A capture gives its workload back as soon as the capture is done, so it holds +// the means to. A restore's workload is needed until the run ends — the seed and +// then the wait for the database to come online are the last things a restore +// does — so the deferred releaseTransientWorkloads of whichever entry point was +// called is its only exit path (FR-011), and a second one carried here would be +// a field nothing reads. Preparation's own failure arms release the workload +// directly, before it has been recorded as the run's. +// +// "Whichever entry point" rather than "the run's single one": `restore --latest +// --s3` runs the gate before its download and then delegates to RestoreBackup, +// which runs it again, so two entry points on that path each defer one. Both +// have to, because the paths between them — a failed download, a second +// database that fails after the first was recorded — return without ever +// reaching the inner one. + +// execOptions names this restore's workload as the target of one command, with +// the client environment the pod's tooling needs. +// +// The credentials are never here, for the reason externalCapture.execOptions +// gives: ExecOptions.Env becomes an `env KEY=VALUE` prefix on the exec command +// line, which is the exposure FR-014 forbids. The clients read their +// credentials from the secret the workload owns. +func (r *externalRestore) execOptions() *ExecOptions { + return &ExecOptions{Pod: r.Pod, Env: r.ClientEnv} +} + +// boltURI is the client URI the seed and the confirmation are sent to: the +// member that answered the probe, which is the one endpoint this run has +// evidence is reachable. +// +// One member rather than the whole list, unlike a capture's `--from`. The seed +// is a destructive statement against the system database, and walking a list +// with it would risk issuing it twice — once against a member whose answer was +// lost, once against the next. +func (r *externalRestore) boltURI() (string, error) { + answered := r.Facts.Answered + if answered.Host == "" { + return "", fmt.Errorf("cannot address the external %s database: no member answered the probe, so there is no endpoint to send the restore to", r.Endpoint.Service) + } + + return r.TLS.boltScheme(r.Endpoint.effectiveProtocol()) + "://" + answered.String(), nil +} + +// --------------------------------------------------------------------------- +// Preparing the restore (the gate) +// --------------------------------------------------------------------------- + +// prepareExternalRestores prepares every database this restore will write to +// that lives outside the deployment: it resolves the endpoint, decides the +// certificate question, creates the workload the restore will run in, probes +// the server through it, and applies the refusals that must land before +// anything is stopped. +// +// It returns before doing anything at all when no database is external, which +// is what keeps FR-015 structural rather than promised. +// +// It takes the targets the gate resolved rather than service names, for the +// reason prepareExternalSources does: where each database lives is exactly what +// a target already carries, and asking again would be a second answer to a +// question the gate has already answered — with only one of the two being the +// gate. +func (iops *InfrahubOps) prepareExternalRestores(targets databaseTargets) error { + external := iops.unpreparedExternalRestores(targets) + if len(external) == 0 { + return nil + } + + // The one line that records the destructive authorisation actually being + // exercised, and it names the channel rather than only the fact. + // + // Which channel matters on its own (FR-010): the flag is an operator at a + // terminal, and the configuration key is the only way an unattended + // restore can carry it — nothing here ever prompts. An operator reading a + // scheduled job's log needs to see which of the two authorised writing to + // infrastructure this tool does not manage, because "somebody set an + // environment variable on the job" and "somebody typed it just now" are + // very different findings after an unexpected restore. + logrus.Warnf("This restore writes to %d database(s) the deployment does not manage (%v), authorised by %s", + len(external), external.services(), external[0].Restore.describe()) + + ops, err := iops.externalCaptureOps(external.services()...) + if err != nil { + return err + } + + for _, target := range external { + if err := iops.prepareExternalRestoreWith(ops, target); err != nil { + return err + } + } + + return nil +} + +// unpreparedExternalRestores are the external databases this run has not +// already prepared a restore for. +// +// It exists because the gate is not called once per run. `restore --latest +// --s3` calls it before pulling a multi-gigabyte object out of a bucket that a +// refusal would make pointless, and then RestoreBackup calls it again — which +// was free while the gate only asked a question. It is not free now: a second +// pass would create a second workload for the same database under the same run +// identifier, which is the same pod name, so the create would collide and the +// run would fail reporting a concurrent operator. The second pass must +// therefore do nothing at all, not merely nothing destructive, which is why +// this is decided before a run identifier or a cluster call is spent. +func (iops *InfrahubOps) unpreparedExternalRestores(targets databaseTargets) databaseTargets { + pending := databaseTargets{} + for _, target := range targets.external() { + if iops.externalRestores[target.Service] == nil { + pending = append(pending, target) + } + } + + return pending +} + +// prepareExternalRestoreWith prepares one external database's restore. +// +// The order is the requirement. The workload is created and probed before the +// version gate and the edition refusal, because the probe is the only thing +// that can answer them; and all of it happens before a single Infrahub service +// has been stopped, because FR-006 requires its answer before any data is +// written and a run that aborts here has changed nothing. +// +// It takes the target rather than a service name so the one thing it may only +// be called for — a database the gate established lives outside the deployment, +// with FR-009's authorisation already checked on that target — is carried by +// its argument rather than checked by its caller. That is only sound while a +// target cannot be built without passing the gate, which is what the first +// line below requires and what databaseTarget.gated makes checkable. +func (iops *InfrahubOps) prepareExternalRestoreWith(ops externalCaptureOps, target databaseTarget) error { + service := target.Service + + endpoint, tls, err := iops.resolveExternalTarget(target, "restore into") + if err != nil { + return err + } + + pod, release, err := ops.startRestore(service) + if err != nil { + return err + } + + // Until the workload is recorded it belongs to this function, and every way + // out of here that is not the record below gives it back. A deferred guard + // rather than an unwind at each return, so a branch added later cannot leak + // the pod (FR-011). + // + // Installed on the line after the workload exists, before anything is done + // with it: a guard that only covers the statements written below it on the + // day it was added is the same guard the unwinds were. + recorded := false + defer func() { + if !recorded { + release() + } + }() + + // Before the probe, because the probe's own Bolt statements need the trust + // anchor this installs, and because it is read out of the deployment's + // running components — which are up now and will not be when the restore + // itself runs. + clientEnv := iops.restoreClientEnv(service, pod, tls) + + facts, err := ops.probe(pod, endpoint, tls) + if err != nil { + return fmt.Errorf("failed to read what the external %s restore needs from %s: %w", service, endpoint.endpointTarget(), err) + } + + if service == serviceNeo4j { + if err := refuseExternalCommunityRestore(endpoint, facts.Edition); err != nil { + return err + } + } + + // FR-006, at the only point where it can hold: the utility has been read + // from the image that will drive the restore, the server has been read over + // the wire, and no data has been written and no service stopped. + if err := checkVersionCompatibility(service, facts.UtilityVersion, facts.ServerVersion); err != nil { + return err + } + + // Recorded, and from here on the workload is the run's: the deferred + // releaseTransientWorkloads each restore entry point makes is what gives it + // back, on the failing paths as well as the succeeding one (FR-011). + iops.recordExternalRestore(service, &externalRestore{ + externalSource: &externalSource{Endpoint: endpoint, TLS: tls, Facts: facts}, + Pod: pod, + Bound: externalDBBound(iops.config), + ClientEnv: clientEnv, + }) + recorded = true + + return nil +} + +// restoreClientEnv is the environment the clients in a freshly created restore +// workload need beyond their credentials. +// +// Both arms are populated here, unlike the capture's. A capture of Neo4j speaks +// the backup protocol, which authenticates through the server's own backup SSL +// policy and has nothing to do with INFRAHUB_DB_TLS_CA_FILE; a restore speaks +// Bolt, so the deployment's own trust anchor is exactly what its client needs +// (see external_ca.go). +func (iops *InfrahubOps) restoreClientEnv(service, pod string, tls externalTLSDecision) map[string]string { + if service == serviceTaskManagerDB { + return iops.postgresClientEnv(pod, tls) + } + + return iops.installExternalCATrust(pod, tls) +} + +// recordExternalRestore stores what this run prepared for one database. +func (iops *InfrahubOps) recordExternalRestore(service string, restore *externalRestore) { + if iops.externalRestores == nil { + iops.externalRestores = map[string]*externalRestore{} + } + + iops.externalRestores[service] = restore +} + +// externalRestoreFor is what the restore paths dispatch on: what this run +// prepared for a database that lives outside the deployment, and nil for one +// that lives inside it. +// +// A nil source against a database located *external* is neither branch, and it +// is an error naming what was not done. That asymmetry is the whole point, and +// it is the lesson externalDatabaseFor records: branching on "did +// preparation record something?" reads a nil as internal, and the run then +// execs the vendor utility inside a container the database does not live in — +// silently, because nothing further along asks the question again. +// +// It issues no query a deployment did not already make. The gate memoises every +// location it establishes, and on Docker Compose the answer is internal by +// construction, so a run that went through the gate reads a memo and an +// internal deployment's path is unchanged (FR-015). +func (iops *InfrahubOps) externalRestoreFor(service string) (*externalRestore, error) { + if restore := iops.externalRestores[service]; restore != nil { + return restore, nil + } + + return nil, iops.refuseUnpreparedExternal(service, + "nothing was prepared to restore into it, so there is no container to write it in; "+ + "this run reached a restore path without establishing what the restore needs") +} + +// --------------------------------------------------------------------------- +// The edition refusal (FR-008's restore side) +// --------------------------------------------------------------------------- + +// refuseExternalCommunityRestore refuses a restore into an external Community +// server, saying why the mechanism is unavailable. +// +// It is the mirror of refuseExternalCommunityCapture and it is separate because +// it refuses for a different reason and offers a different remedy. A Community +// *capture* is impossible because the dump reads the store files directly. A +// Community *restore* is impossible because the mechanism is a Cypher database +// creation seeded from a URI, and Community hosts exactly one database and +// cannot create another — so there is no statement to issue, not merely no file +// to read. +// +// An edition the server did not report is not read as Community, for the reason +// its sibling gives: guessing the destructive branch from a probe that did not +// answer is what Principle II forbids. +func refuseExternalCommunityRestore(endpoint *DatabaseEndpoint, edition string) error { + if !isCommunityEdition(edition) { + return nil + } + + return fmt.Errorf( + "cannot restore into %s: %w, and the remote alternative — directing the server to seed a database from the backup artifact — creates a database, which Community Edition cannot do. "+ + "Seeding from a URI is a Neo4j Enterprise Edition feature. "+ + "Nothing is missing from the deployment — the %s service has no container here because the database is managed outside it. "+ + "No Infrahub service was stopped and no data was changed", + endpoint.endpointTarget(), errExternalCommunityRestore, serviceNeo4j, + ) +} + +// errExternalCommunityRestore is the refusal above as a value callers and tests +// can identify rather than a message they have to match. +var errExternalCommunityRestore = errors.New("the Community Edition restore mechanism cannot create the database a seed would populate") + +// refuseExternalCommunityDumpRestore is where a Plakar restore that would load +// a Neo4j dump into a database outside the deployment is refused: at the gate, +// before anything is stopped, wiped or restarted. +// +// The refusal used to be raised from inside restoreNeo4jCommunityStream, which +// is where the impossibility actually bites but which both Plakar paths reach +// only after stopAppContainers, confirmAppContainersQuiesced and +// restartDependencies. So the run took the deployment down to discover something +// it already knew — the artifact's edition is read from the snapshot's own tags +// long before — and then said "No Infrahub service was stopped", which by then +// was untrue. Every other refusal on this feature is raised before the first +// destructive step (see external_backup_gate.go), and this is now one of them. +// +// loadsDump is the caller's own answer to "will this restore load a Neo4j +// dump?", which the two entry points establish differently: a group has a Neo4j +// component and a metadata edition, a single snapshot has a component tag and an +// edition tag. Taking the answer rather than re-deriving it is what keeps the +// refusal from disagreeing with the branch it guards. +// It reads what the gate prepared rather than going through +// externalRestoreFor: both callers run after prepareDatabaseRestore, so a +// service absent from externalRestores is an in-deployment one — which is what +// every other path on this feature branches on. externalRestoreFor would ask +// the cluster where the database lives a second time, a listing an ordinary +// internal deployment would newly pay for on every Plakar restore (FR-015). Its +// own location query stays where it is needed: in the load, which can be +// reached without a gate. +func (iops *InfrahubOps) refuseExternalCommunityDumpRestore(loadsDump bool) error { + if !loadsDump { + return nil + } + + restore := iops.externalRestores[serviceNeo4j] + if restore == nil { + return nil + } + + return refuseExternalDumpRestore(restore.Endpoint) +} + +// refuseExternalDumpRestore refuses to load a Community dump into a database +// outside the deployment. +// +// It exists because the Plakar paths stream a dump straight into the database's +// container without going through restoreNeo4j, so the dispatch that sends an +// external database to the seed path is not on that route. Loading a dump means +// running `neo4j-admin database load` against the server's own store +// directory — which is what makes it work on an in-deployment database and what +// makes it impossible here — so the refusal names the artifact rather than the +// server: an Enterprise online backup can seed an external server, and a dump +// cannot. +func refuseExternalDumpRestore(endpoint *DatabaseEndpoint) error { + return fmt.Errorf(externalDumpRestoreRefusal+ + "No Infrahub service was stopped and no data was changed", endpoint.endpointTarget()) +} + +// externalDumpRestoreRefusal is the half of the refusal that states a fact +// about the artifact rather than about the run, so the two callers below cannot +// drift on it. What they must not share is the closing clause: whether anything +// has been stopped is a fact about where the refusal was reached, and asserting +// the wrong one is what went wrong when this refusal lived in only one place. +const externalDumpRestoreRefusal = "cannot restore this snapshot into %s: it carries a Neo4j dump, which is loaded by writing into the server's own store directory, and this run has no access to the storage of a server it does not host. " + + "The artifact an external server can be restored from is an Enterprise online backup, which it fetches itself. " + +// refuseExternalDumpRestoreAfterQuiescing is the same refusal reached from +// inside the stream load, which the gate is meant to make unreachable. +// +// It is kept rather than deleted because restoreNeo4jCommunityStream is where +// the container is actually needed, so it is the last place the guarantee can be +// held, and because a Plakar path added later could reach it without passing a +// gate. What it does not do is repeat the gate's "nothing was stopped" claim: by +// here something has been, and a refusal that asserts otherwise is exactly what +// went wrong when this one lived only here. +func refuseExternalDumpRestoreAfterQuiescing(endpoint *DatabaseEndpoint) error { + return fmt.Errorf(externalDumpRestoreRefusal+ + "This run had already quiesced the deployment before reaching this point, so its services are returned to their prior scale; no database was written to", + endpoint.endpointTarget()) +} + +// --------------------------------------------------------------------------- +// Staging the seed the server will fetch (FR-007, FR-020) +// --------------------------------------------------------------------------- + +// seedKeySegment is the key segment the staged artifact is written under, so it +// sits beside the archives rather than among them. +// +// Retention only ever recognises an object whose key is exactly the one +// buildS3Key would produce for a backup's own base name, so an object one +// segment deeper is invisible to it. That is what keeps a staged seed out of a +// retention decision without retention having to learn about seeds. +const seedKeySegment = "seed" + +// neo4jSeedArtifactSuffix is the extension the online backup gives its +// artifact. The archive carries exactly one of them for the database it was +// taken from. +const neo4jSeedArtifactSuffix = ".backup" + +// restoreSeed is the artifact a database server will fetch, and the evidence +// this run has that it is there. +type restoreSeed struct { + // URI is what the seed statement will name. It addresses an object store, + // because those are the providers a server supports without configuration. + URI string + + // Key is the object's key within the bucket, kept so the staged copy can be + // removed once the restore has actually landed. + Key string + + // Bucket is where it was staged, for the same reason. + Bucket string + + // Bytes is the size an independent reader saw at that URI, which is the + // pre-flight's actual finding rather than the size the upload reported + // writing. + Bytes int64 +} + +// stageNeo4jSeed puts the restore artifact where the database server can fetch +// it and establishes that it is readable there, before anything destructive +// happens (FR-007). +// +// What it proves and what it cannot prove are worth stating exactly, because +// the difference is the residual risk this whole path carries. It proves that +// an artifact exists in the archive, that an object store is configured, that +// the object was written, and that a *reader* — a client built from the URI +// that will be published, not the one that wrote it — finds it at that URI. It +// cannot prove there is a route from the database host to the object store, or +// that the server has the provider tooling and credentials the fetch needs: +// neither is visible from here, and no documented interface answers either +// question. Both are the customer-side prerequisites FR-020 requires be named, +// and the confirmation in waitForNeo4jRestore is what turns a prerequisite that +// does not hold into a failed run rather than a database reported restored. +func (iops *InfrahubOps) stageNeo4jSeed(workDir string) (restoreSeed, error) { + artifact, err := neo4jSeedArtifact(workDir) + if err != nil { + return restoreSeed{}, err + } + + configured := iops.objectStore() + if err := configured.ValidateConfig(); err != nil { + return restoreSeed{}, fmt.Errorf( + "cannot restore into an external Neo4j database: the server fetches the backup artifact itself, so the artifact has to be staged in an object store the database host can read, and none is configured (%w). "+ + "Configure the bucket this run may stage into with --s3-bucket (plus --s3-prefix, --s3-endpoint and --s3-region as your provider needs), and make sure the Neo4j servers can read from it. "+ + "No Infrahub service was stopped and no data was changed", err) + } + + client, err := seedStoreClient(configured, configured.Bucket, seedPrefix(configured.Prefix, filepath.Base(artifact))) + if err != nil { + return restoreSeed{}, fmt.Errorf("failed to reach the object store the restore artifact must be staged in: %w", err) + } + + logrus.Infof("Staging %s in %s so the Neo4j servers can fetch it themselves", filepath.Base(artifact), client.locationName()) + + ctx, cancel := context.WithTimeout(context.Background(), externalDBBound(iops.config)) + defer cancel() + + uri, err := client.Upload(ctx, artifact) + if err != nil { + // The same three elements as unreadableSeedArtifact: staging happens before + // anything is stopped, so a store this run cannot write to is refused with + // the deployment untouched, and the operator needs to read that first. + return restoreSeed{}, fmt.Errorf( + "failed to stage the restore artifact where the Neo4j servers can fetch it: %w. "+ + "The bucket has to exist and be writable from here, and readable from the database host, because the Neo4j servers fetch the artifact themselves. "+ + "No Infrahub service was stopped and no data was changed", err) + } + + seed, err := verifySeedReadable(ctx, configured, uri) + if err != nil { + return restoreSeed{}, err + } + + logrus.Infof("The restore artifact is readable at %s (%s). The Neo4j servers fetch it themselves, so each of them needs a route to that object store and the provider credentials for it — a prerequisite this run cannot verify from here; a fetch that fails leaves the database offline, which the confirmation after the restore reports", + seed.URI, formatBytes(seed.Bytes)) + + return seed, nil +} + +// objectStore is the operator's S3 configuration, tolerating a Configuration +// that was never given one. That shape is not theoretical — a run configured +// entirely by flags that name no bucket has it — and what such a run must read +// is the refusal in stageNeo4jSeed, not a panic on a nil dereference. +func (iops *InfrahubOps) objectStore() *S3Config { + if iops.config == nil || iops.config.S3 == nil { + return &S3Config{} + } + + return iops.config.S3 +} + +// seedPrefix is the staged artifact's key prefix: a folder of its own, named +// after the artifact, under the seed segment below wherever the archives are +// written (or under that segment alone where no prefix is configured). +// +// The folder of its own is what makes a restore independent of every earlier +// one. A Neo4j server seeding from a `.backup` URI reads the other artifacts +// beside it as one backup chain, and refuses the seed when they do not form one +// ("is not part of a valid backup chain"): observed on 2025.10.1 against MinIO, +// where the same artifact seeded alone and failed next to another. A failed +// restore leaves its staged copy on purpose (discardStagedSeed), so with a shared +// folder that copy broke every later restore into the same bucket, and the +// database stayed offline until the confirmation timed out. Naming the folder +// after the artifact keeps one artifact per folder without a run identifier: +// restoring the same archive twice writes the same object again. +func seedPrefix(prefix, artifact string) string { + folder := seedKeySegment + "/" + strings.TrimSuffix(artifact, neo4jSeedArtifactSuffix) + + trimmed := strings.Trim(strings.TrimSpace(prefix), "/") + if trimmed == "" { + return folder + } + + return trimmed + "/" + folder +} + +// verifySeedReadable reads the staged object back through a client built from +// the URI that will be published, which is the closest this host can get to the +// read the database server will perform. +// +// A fresh client rather than the uploader is the whole point of the check. The +// uploader reports what it believes it wrote; this addresses the object the way +// a reader addresses it, so a prefix that does not compose, a bucket the +// endpoint resolves differently, or an object that never landed is a failure +// here rather than a database left offline after the destructive step. +func verifySeedReadable(ctx context.Context, cfg *S3Config, uri string) (restoreSeed, error) { + bucket, key, ok := ParseS3URI(uri) + if !ok { + return restoreSeed{}, fmt.Errorf("cannot verify the staged restore artifact: %q is not an object-store URI, and only the object-store providers are ones a Neo4j server can fetch from without server-side configuration", uri) + } + + reader, err := seedStoreClient(cfg, bucket, "") + if err != nil { + return restoreSeed{}, fmt.Errorf("failed to build a reader for the staged restore artifact at %s: %w", uri, err) + } + + size, err := reader.Stat(ctx, key) + if err != nil { + return restoreSeed{}, unreadableSeedArtifact(uri, err) + } + + return restoreSeed{URI: uri, Key: key, Bucket: bucket, Bytes: size}, nil +} + +// seedStoreClient is a client on the object store the seed is staged in: the +// configured endpoint and region, addressing one bucket under one prefix. The +// upload, the read-back and the removal each build their own, on purpose (see +// verifySeedReadable), so the recipe is stated once here. +func seedStoreClient(configured *S3Config, bucket, prefix string) (*S3Client, error) { + return NewS3Client(&S3Config{Bucket: bucket, Prefix: prefix, Endpoint: configured.Endpoint, Region: configured.Region}) +} + +// errSeedArtifactUnreadable is FR-007's refusal, as a value callers and tests +// can identify rather than a message they have to match. +var errSeedArtifactUnreadable = errors.New("the restore artifact could not be read back from where it was staged") + +// unreadableSeedArtifact is what an operator reads when the pre-flight staged +// the artifact and then could not read it back. +// +// The failure-message contract requires it to name the URI attempted and the +// server-side prerequisite, which are what separate this from an ordinary +// object-store error: the Neo4j server fetches this artifact itself, so the +// remedy is a route and a policy on the *database host* rather than anything on +// the host running this tool. It states that nothing was stopped and nothing +// changed, because on the restore path that is the fact an operator needs +// first. +// +// It is a named constructor rather than an inline message so that the three +// elements can be asserted where they are produced — the same reason +// externalCommunityCaptureRefusal and neo4jRestoreFailure are named. +func unreadableSeedArtifact(uri string, cause error) error { + return fmt.Errorf( + "%w: %s cannot be read back: %w. "+ + "The Neo4j servers fetch this artifact themselves, so a URI this run cannot read is one they certainly cannot — the bucket has to exist, hold the object, and be readable both from here and from the database host. "+ + "No Infrahub service was stopped and no data was changed", + errSeedArtifactUnreadable, uri, cause) +} + +// discardStagedSeed removes the staged copy of the artifact. +// +// Only ever after the restore has been confirmed online. A seed that is still +// being fetched — which is every moment between the statement returning and the +// database coming up — must not have its source deleted underneath it, and a +// restore that failed is one an operator is about to look into, so the object +// stays and its URI is in the log. +func (iops *InfrahubOps) discardStagedSeed(seed restoreSeed) { + client, err := seedStoreClient(iops.objectStore(), seed.Bucket, "") + if err != nil { + logrus.Warnf("Could not remove the staged restore artifact at %s: %v", seed.URI, err) + + return + } + + if err := client.Delete(context.Background(), seed.Key); err != nil { + logrus.Warnf("Could not remove the staged restore artifact at %s; it is a copy of the archive's own contents and is safe to delete by hand: %v", seed.URI, err) + + return + } + + logrus.Debugf("Removed the staged restore artifact at %s", seed.URI) +} + +// neo4jSeedArtifact is the one artifact in the extracted archive a server can +// be seeded from. +// +// Exactly one, and named rather than assumed. The online backup produces a +// single `-.backup` file and the archive carries it raw +// (see contracts/artifact-and-metadata.md), so zero of them means this archive +// is not an Enterprise one and several means the archive holds captures of two +// databases — and picking one of those by sort order would seed a database from +// another database's data. +func neo4jSeedArtifact(workDir string) (string, error) { + databaseDir := filepath.Join(workDir, "backup", "database") + + matches, err := filepath.Glob(filepath.Join(databaseDir, "*"+neo4jSeedArtifactSuffix)) + if err != nil { + return "", fmt.Errorf("failed to look for the Neo4j backup artifact in %s: %w", databaseDir, err) + } + + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf( + "cannot restore into an external Neo4j database: the archive holds no %s artifact in %s, which is what an Enterprise online backup produces and what a server can be seeded from. "+ + "An archive taken from a Community deployment carries a dump instead, and a dump can only be loaded by a server whose store files this run can reach. "+ + "No Infrahub service was stopped and no data was changed", neo4jSeedArtifactSuffix, databaseDir) + default: + return "", fmt.Errorf( + "cannot restore into an external Neo4j database: %s holds %d %s artifacts, so which database this run would seed from is ambiguous. "+ + "No Infrahub service was stopped and no data was changed", databaseDir, len(matches), neo4jSeedArtifactSuffix) + } +} + +// --------------------------------------------------------------------------- +// The seed statement (research R2) +// --------------------------------------------------------------------------- + +// neo4jSeedReplaceFloor is the first server version whose CREATE OR REPLACE +// DATABASE accepts a seed URI. At or above it the destructive step is one +// atomic replacement; below it the database has to be dropped first, which +// opens a window in which it does not exist at all. +const neo4jSeedReplaceFloor = "2025.01" + +// neo4jSeedStatements is the Cypher that seeds a database from a URI, in the +// form the server version supports. +// +// Two forms rather than one, and the newer one is not merely tidier. `CREATE OR +// REPLACE DATABASE … OPTIONS { seedURI: … }` is a single replacement, so a +// failure to fetch the seed leaves a database that is offline with a reason +// attached. The older servers need `DROP DATABASE … IF EXISTS` first, and +// between that statement and the create the database does not exist — so a run +// interrupted in between leaves the deployment with no database at all, which +// is worth an operator knowing before they start. +// +// A server version that cannot be ordered takes the older form. It works on +// every version, where the newer one does not, so an unreadable version costs +// the atomicity rather than the restore. In practice it does not arise: +// checkVersionCompatibility has already aborted on a version neither side could +// read. +func neo4jSeedStatements(database, seedURI, serverVersion string) ([]string, error) { + name, err := safeDatabaseIdentifier(database) + if err != nil { + return nil, fmt.Errorf("cannot build the statement that seeds the external %s database: %w", serviceNeo4j, err) + } + + uri, err := safeSeedURI(seedURI) + if err != nil { + return nil, err + } + + options := fmt.Sprintf("OPTIONS {existingData: 'use', seedURI: '%s'}", uri) + + if atomicSeedReplaceSupported(serverVersion) { + return []string{fmt.Sprintf("CREATE OR REPLACE DATABASE %s %s", name, options)}, nil + } + + logrus.Warnf("This server (%s) is older than %s, so the database is dropped before it is created from the seed; between those two statements %s does not exist", + serverVersion, neo4jSeedReplaceFloor, name) + + return []string{ + fmt.Sprintf("DROP DATABASE %s IF EXISTS", name), + fmt.Sprintf("CREATE DATABASE %s %s", name, options), + }, nil +} + +// atomicSeedReplaceSupported reports whether this server's CREATE OR REPLACE +// DATABASE accepts a seed URI. +func atomicSeedReplaceSupported(serverVersion string) bool { + server, err := parseDatabaseVersion(serverVersion) + if err != nil { + logrus.Warnf("Could not read the server version %q, so the restore takes the drop-then-create form that every version accepts: %v", serverVersion, err) + + return false + } + + floor, err := parseDatabaseVersion(neo4jSeedReplaceFloor) + if err != nil { + return false + } + + return server.compare(floor) >= 0 +} + +// safeSeedURI refuses a URI that carries anything but a URI before it is +// interpolated into a Cypher string literal. +// +// The value is this run's own construction rather than an operator's text, so +// this is a belt rather than braces — but it is the one value on this path that +// reaches a statement inside quotes, and a quote or a newline in it would close +// the literal and continue the statement. It also refuses a scheme that is not +// an object store's, which is a correctness check rather than a safety one: the +// other providers need `dbms.databases.seed_from_uri_providers` configured on +// every server, and a run that cannot see that configuration must not depend +// on it. +func safeSeedURI(uri string) (string, error) { + trimmed := strings.TrimSpace(uri) + if trimmed == "" { + return "", fmt.Errorf("cannot seed the external %s database: no artifact URI was established", serviceNeo4j) + } + + if strings.ContainsAny(trimmed, "'\"`\\\n\r\t ") { + return "", fmt.Errorf("cannot seed the external %s database from %q: the URI carries a character a URI may not", serviceNeo4j, trimmed) + } + + scheme, _, found := strings.Cut(trimmed, "://") + if !found { + return "", fmt.Errorf( + "cannot seed the external %s database from %q: it names no scheme, and a server can only fetch an artifact it has a provider for. The schemes every server supports as shipped are %s", + serviceNeo4j, trimmed, strings.Join(cloudSeedSchemes, ", ")) + } + + if !isCloudSeedScheme(scheme) { + return "", fmt.Errorf( + "cannot seed the external %s database from %q: a server fetches %s URIs only where dbms.databases.seed_from_uri_providers has been configured for that scheme, which this run cannot see. "+ + "The schemes every server supports as shipped are %s", + serviceNeo4j, trimmed, scheme, strings.Join(cloudSeedSchemes, ", ")) + } + + return trimmed, nil +} + +// cloudSeedSchemes are the seed providers a Neo4j server carries as shipped: +// Amazon S3, Google Cloud Storage and Azure Cloud Storage. Every other scheme +// — http, https, ftp, file — requires per-server configuration (research R2). +var cloudSeedSchemes = []string{"s3", "gs", "azb"} + +// isCloudSeedScheme reports whether a scheme is one of those. +func isCloudSeedScheme(scheme string) bool { + return slices.ContainsFunc(cloudSeedSchemes, func(supported string) bool { + return strings.EqualFold(strings.TrimSpace(scheme), supported) + }) +} + +// --------------------------------------------------------------------------- +// Confirming the restore actually landed (FR-021) +// --------------------------------------------------------------------------- + +const ( + // neo4jStatusColumn and neo4jStatusMessageColumn are what SHOW DATABASE + // reports about a database's own state, and the two fields the seed's real + // outcome appears in. + neo4jStatusColumn = "currentStatus" + neo4jStatusMessageColumn = "statusMessage" + + // neo4jOnlineStatus is the only status that means the seed was fetched, + // validated and loaded. + neo4jOnlineStatus = "online" + neo4jOfflineStatus = "offline" + + // neo4jSeedPollInterval is how often the database's status is read while it + // starts. It is a wait between reads rather than a budget: the window is + // the operation bound, and this only decides how many statements are spent + // inside it. + neo4jSeedPollInterval = 5 * time.Second + + // neo4jStatusUnreadableGrace is how long the wait tolerates being unable to + // ask the database anything at all before it gives up on the question + // rather than on the database. + // + // It is short on purpose, and it is not a second copy of the operator's + // bound. The condition it covers is a server refusing a session for a + // moment after replacing a database — seconds — whereas the bound covers a + // store that legitimately takes hours to load. Anything past this is the + // run's own path to the database failing, which no amount of further + // waiting improves, and which must not be reported as the database's state + // (see neo4jRestoreUnreadable). + neo4jStatusUnreadableGrace = time.Minute +) + +// neo4jSeedFailureMarker is the statusMessage a server reports for a seed it +// could not download or validate. It is matched so the run can fail as soon as +// the server says so, rather than polling out a window whose answer is already +// known. +const neo4jSeedFailureMarker = "unable to start database" + +// neo4jDatabaseStatus is one member's report of the database's state. +type neo4jDatabaseStatus struct { + Status string + Message string +} + +// online reports whether this member has the database up. +func (s neo4jDatabaseStatus) online() bool { + return strings.EqualFold(strings.TrimSpace(s.Status), neo4jOnlineStatus) +} + +// failed reports whether this member has given up on the seed. A server that +// could not fetch or validate the artifact says so in the status message, and +// waiting out the window after that only delays a failure the server has +// already reported. +// +// Two shapes are read as that report: the marker below, and an `offline` state +// that carries a message. The second was observed on 2025.10.1 for a seed the +// server rejected ("The backup artifact (...) to restore is not part of a valid +// backup chain."): the database went offline with that message within seconds, +// and without this the run waited out the whole --external-db-timeout, two hours +// by default, for a database that was never going to start. An `offline` state +// with no message is not read as a failure, because a database being replaced +// can pass through it. +func (s neo4jDatabaseStatus) failed() bool { + if strings.Contains(strings.ToLower(s.Message), neo4jSeedFailureMarker) { + return true + } + + return strings.EqualFold(strings.TrimSpace(s.Status), neo4jOfflineStatus) && strings.TrimSpace(s.Message) != "" +} + +// describe renders one member's report for the failure an operator reads. +func (s neo4jDatabaseStatus) describe() string { + status := strings.TrimSpace(s.Status) + if status == "" { + status = "an unreported status" + } + if message := strings.TrimSpace(s.Message); message != "" { + return fmt.Sprintf("%s (%s)", status, message) + } + + return status +} + +// neo4jDatabaseStatusCommand asks the server for the state of the database that +// was just seeded. +// +// It runs against the system database because the database being asked about +// may not be up — which is the entire point of asking. +func neo4jDatabaseStatusCommand(uri, database string) ([]string, error) { + name, err := safeDatabaseIdentifier(database) + if err != nil { + return nil, fmt.Errorf("cannot query the state of the external %s database: %w", serviceNeo4j, err) + } + + statement := fmt.Sprintf("SHOW DATABASE %s YIELD %s, %s", name, neo4jStatusColumn, neo4jStatusMessageColumn) + + return cypherShellCommand(uri, neo4jSystemDatabase, statement), nil +} + +// parseNeo4jDatabaseStatuses reads every member's report out of one +// SHOW DATABASE result. +// +// Every member, not the first row. A clustered database is reported once per +// member hosting it, and a seed that loaded on one server and failed on another +// is a half-restored database — which reading row zero would report as a +// success or a failure depending on which member the listing happened to put +// first. +// +// It reads its columns by name and skips whatever the server printed above the +// header, which is the rule every parser on this path follows: a notice line +// read as data parses far enough to be acted on rather than far enough to fail. +func parseNeo4jDatabaseStatuses(output string) []neo4jDatabaseStatus { + columns, rows := cypherResult(output, neo4jStatusColumn) + if len(rows) == 0 { + return nil + } + + // The header was located by the status column, so it is present. + statusAt, _ := columns.at(neo4jStatusColumn) + messageAt, hasMessage := columns.at(neo4jStatusMessageColumn) + + statuses := make([]neo4jDatabaseStatus, 0, len(rows)) + for _, line := range rows { + fields := splitPlainRow(line) + if statusAt >= len(fields) { + continue + } + + status := neo4jDatabaseStatus{Status: fields[statusAt]} + if hasMessage && messageAt < len(fields) { + status.Message = fields[messageAt] + } + + statuses = append(statuses, status) + } + + return statuses +} + +// seedVerdict is what one reading of SHOW DATABASE settles. +type seedVerdict struct { + // Online is true only when every member hosting the database reports it up. + Online bool + + // Failed is true when a member has reported it cannot start the database, + // which is how a seed that could not be fetched or validated surfaces. + Failed bool + + // Detail is what the members reported, for the operator-facing message. + Detail string +} + +// classifySeedStatuses turns one SHOW DATABASE result into a verdict. +// +// A result with no rows is neither online nor failed: a database being replaced +// is briefly absent from the listing, and reading that as a failure would fail +// a restore that is working. +func classifySeedStatuses(statuses []neo4jDatabaseStatus) seedVerdict { + if len(statuses) == 0 { + return seedVerdict{Detail: "the server reported no state for it yet"} + } + + verdict := seedVerdict{Online: true} + described := make([]string, 0, len(statuses)) + for _, status := range statuses { + if !status.online() { + verdict.Online = false + } + if status.failed() { + verdict.Failed = true + } + described = append(described, status.describe()) + } + + verdict.Detail = strings.Join(described, "; ") + + return verdict +} + +// waitForNeo4jRestore polls the server until the restored database is online, +// and fails otherwise (FR-021). +// +// This is not a nicety on top of the seed. Download and validation of a seed +// happen only as the database starts, so the statement that requested it +// returns success whatever the artifact turns out to be: a truncated upload, an +// artifact the server has no route to, a bucket it has no credentials for, and +// a perfectly good backup are indistinguishable at that moment. Reporting +// success there would be a restore that silently loaded nothing, which is the +// failure constitution Principle II exists to rule out. +// +// The window is the operation bound, and each statement inside it is bounded on +// its own, so a server that stops answering is reported rather than waited out +// (FR-025). +func (iops *InfrahubOps) waitForNeo4jRestore(restore *externalRestore, uri string) error { + return iops.waitForNeo4jRestoreWithin(restore, uri, neo4jStatusUnreadableGrace) +} + +// waitForNeo4jRestoreWithin is the wait under an explicit unreadable grace, +// following confirmAppContainersQuiescedWithin's idiom: it makes a run that can +// no longer ask the database anything assertable without waiting out the real +// grace. A grace of zero means the first unanswered read is the verdict. +func (iops *InfrahubOps) waitForNeo4jRestoreWithin(restore *externalRestore, uri string, grace time.Duration) error { + command, err := neo4jDatabaseStatusCommand(uri, restore.Endpoint.Database) + if err != nil { + return err + } + + logrus.Infof("Waiting for %s to come online; the seed is downloaded and validated as the database starts, so this is where a seed the server could not read is reported", + restore.Endpoint.Database) + + deadline := time.Now().Add(restore.Bound) + verdict := seedVerdict{Detail: "the server was never asked"} + + // Whether the run can still ask the question, and since when it could not. + // + // The operator's bound belongs to the database: a large store legitimately + // takes hours to load, which is why it defaults to two of them. It does not + // belong to the *question*. A read that never reached the server says + // nothing about the database, and spending the whole bound on it with + // Infrahub scaled to zero — then reporting "the database is not usable; its + // reported state is the server could not be asked" — attributes a cluster + // failure to the customer's database and sends the operator to the wrong + // logs. So the two conditions are separated here, and each is bounded by + // what it actually is. + var ( + unreadableSince time.Time + unreadableErr error + ) + + for attempt := 1; ; attempt++ { + output, err := iops.execBoundedAgainst( + externalDBProbeBound(iops.config), "reading the restored database's state", + restore.Endpoint.endpointTarget(), serviceNeo4j, command, restore.execOptions(), + ) + if err != nil { + // A server that has just replaced a database can refuse a session + // for a moment, so a failed read is a retry rather than a verdict — + // for as long as that moment lasts. Past the grace, the cause is not + // a database settling: the transient workload was evicted, + // `pods/exec` was revoked, the API server is gone. + verdict = seedVerdict{Detail: fmt.Sprintf("the server could not be asked for its state: %v", err)} + if unreadableSince.IsZero() { + unreadableSince = time.Now() + } + unreadableErr = err + if unreadable := time.Since(unreadableSince); unreadable >= grace { + return neo4jRestoreUnreadable(restore, err, unreadable) + } + logrus.Debugf("Could not read the state of %s (attempt %d), retrying: %v", restore.Endpoint.Database, attempt, err) + } else { + // The server answered, so whatever failed before it was the moment + // the grace exists for. + unreadableSince = time.Time{} + unreadableErr = nil + + verdict = classifySeedStatuses(parseNeo4jDatabaseStatuses(output)) + if verdict.Online { + logrus.Infof("%s is online after %d status read(s): the seed was fetched, validated and loaded", restore.Endpoint.Database, attempt) + + return nil + } + if verdict.Failed { + return neo4jRestoreFailure(restore, verdict.Detail, false) + } + + logrus.Infof("%s is not online yet (%s); still waiting", restore.Endpoint.Database, verdict.Detail) + } + + if sleepUntilNextPoll(deadline, neo4jSeedPollInterval) { + // A bound shorter than the grace expires before it, and the same + // distinction has to hold there: the last thing that happened was + // still a question that went unanswered, not a database that failed + // to come up. + if unreadableErr != nil { + return neo4jRestoreUnreadable(restore, unreadableErr, time.Since(unreadableSince)) + } + + return neo4jRestoreFailure(restore, verdict.Detail, true) + } + } +} + +// neo4jRestoreUnreadable is what an operator reads when the run can no longer +// ask the database how the restore went. +// +// It is deliberately not neo4jRestoreFailure with a different detail, and the +// difference is attribution. That message says the database is not usable and +// points at the server's own logs, which is right when the server answered and +// reported a database it cannot start. Here the server said nothing: the +// command that asks runs in a transient pod, through this cluster's API, under +// a permission to exec — and any of those failing produces exactly this, +// while the database may well be loading the seed successfully. +// +// So it names the three elements the failure-message contract requires against +// *that* condition: what happened (the state could not be read, for how long), +// the resource (the endpoint, and the three things between this run and it), +// and the action (look at the pod and the permission, then read the database's +// own state directly — it may already be online). +func neo4jRestoreUnreadable(restore *externalRestore, err error, unreadable time.Duration) error { + return fmt.Errorf( + "the restore of %s into %s was accepted, and this run has been unable to read the database's state for %v: %w. "+ + "That is a failure of the path to the database, not a report about it — the read runs in this run's transient pod, through this cluster's API, under permission to exec into that pod — "+ + "so the database may still be loading the seed. Check that pod and that permission, then read the database's own state directly before restoring again. "+ + "Infrahub is returned to its prior scale, and the staged artifact is left in place so the restore can be repeated without re-staging it", + restore.Endpoint.Database, restore.Endpoint.endpointTarget(), unreadable.Truncate(time.Second), err) +} + +// neo4jRestoreFailure is what an operator reads when a seed was accepted and +// the database did not come up. It names the database's reported status and +// where the server records the cause, which is the failure-message contract's +// requirement for this condition. +func neo4jRestoreFailure(restore *externalRestore, detail string, timedOut bool) error { + reason := "the server reported that it cannot start it" + if timedOut { + reason = fmt.Sprintf("it did not come online within %s, which --external-db-timeout sets", restore.Bound) + } + + return fmt.Errorf( + "the restore of %s into %s was accepted but the database is not usable: %s. Its reported state is %s. "+ + "A seed is downloaded and validated only as the database starts, so this is where an artifact the server could not fetch or could not read surfaces; "+ + "the cause is in that server's own logs (neo4j.log and debug.log on the database host). "+ + "Infrahub is returned to its prior scale, and the staged artifact is left in place so it can be examined", + restore.Endpoint.Database, restore.Endpoint.endpointTarget(), reason, detail) +} + +// --------------------------------------------------------------------------- +// The Neo4j restore itself +// --------------------------------------------------------------------------- + +// preflightExternalNeo4jRestore is FR-007's pre-flight, and every restore entry +// point that will write to Neo4j performs it after the archive has been +// extracted and before the first destructive step. +// +// It is where the artifact becomes something the database server can read, and +// where a run that cannot make it so stops — with Infrahub still up, nothing +// wiped and nothing scaled to zero. A restore that got as far as the seed +// without it is refused there rather than proceeding, which is what keeps the +// pre-flight from being a step a later entry point can forget (see +// restoreNeo4jExternal). +// +// It returns without doing anything for an in-deployment database, which is +// every deployment that exists today (FR-015). +func (iops *InfrahubOps) preflightExternalNeo4jRestore(workDir string) error { + restore, err := iops.externalRestoreFor(serviceNeo4j) + if err != nil { + return err + } + if restore == nil { + return nil + } + + seed, err := iops.stageNeo4jSeed(workDir) + if err != nil { + return err + } + + restore.Seed = seed + + return nil +} + +// restoreNeo4jExternal directs the server to seed the database from the staged +// artifact and then confirms it came up. +// +// The seed URI is established by the pre-flight, before anything was stopped, +// which is what makes FR-007's check a pre-flight: by the time this runs, the +// only remaining unknown is whether the server itself can reach what this run +// has already read. +func (iops *InfrahubOps) restoreNeo4jExternal(restore *externalRestore) error { + seed := restore.Seed + if seed.URI == "" { + return fmt.Errorf( + "cannot restore into the external %s database at %s: no artifact was staged for the server to fetch, so this run reached the destructive step without the pre-flight that establishes the server can read it", + serviceNeo4j, restore.Endpoint.endpointTarget()) + } + + uri, err := restore.boltURI() + if err != nil { + return err + } + + statements, err := neo4jSeedStatements(restore.Endpoint.Database, seed.URI, restore.Facts.ServerVersion) + if err != nil { + return err + } + + logrus.Infof("Restoring the external Neo4j database %s at %s by seeding it from %s", + restore.Endpoint.Database, restore.Endpoint.endpointTarget(), seed.URI) + + for _, statement := range statements { + if _, err := iops.execBoundedAgainst( + restore.Bound, "the seed request", restore.Endpoint.endpointTarget(), serviceNeo4j, + cypherShellCommand(uri, neo4jSystemDatabase, statement), restore.execOptions(), + ); err != nil { + return fmt.Errorf("failed to seed the external Neo4j database %s at %s: %w", + restore.Endpoint.Database, restore.Endpoint.endpointTarget(), err) + } + } + + if err := iops.waitForNeo4jRestore(restore, uri); err != nil { + return err + } + + // Only now, and only because the database is up: until it is, the server may + // still be fetching the object this would delete. + iops.discardStagedSeed(seed) + + logrus.Info("Neo4j restore completed") + + return nil +} + +// --------------------------------------------------------------------------- +// The PostgreSQL restore +// --------------------------------------------------------------------------- + +// externalPostgresRestoreFile is where the dump is placed inside the restore +// workload: the scratch volume, which is the one directory the pod is +// guaranteed to be able to write to. +const externalPostgresRestoreFile = transientScratchPath + "/infrahubops_restore.dump" + +// restorePostgreSQLExternal reads the dump back into a server outside the +// deployment. +// +// It is the internal path's own TCP branch with three substitutions — the host, +// the credentials' source, and certificate verification — because pg_restore is +// a client either way and the dump is the same file. What it does not do is +// start `task-manager-db`: there is no such workload here, which is exactly +// what the internal path's Kubernetes arm was already guessing at when it +// skipped the failure with "may be externally managed". +func (iops *InfrahubOps) restorePostgreSQLExternal(workDir string, restore *externalRestore) error { + dump := filepath.Join(workDir, "backup", prefectDumpFilename) + + logrus.Infof("Restoring the external PostgreSQL database at %s (of the endpoints supplied, %s answered the probe)...", + restore.Endpoint.endpointTarget(), restore.Facts.Answered) + + if err := iops.copyDumpIntoRestoreWorkload(restore, dump); err != nil { + return err + } + defer iops.removeTransientScratch(serviceTaskManagerDB, "the staged dump", &ExecOptions{Pod: restore.Pod}, externalPostgresRestoreFile) + + host := restore.Facts.Answered + command := postgresRestoreCommand(postgresRestoreRequest{ + Host: &host, + File: externalPostgresRestoreFile, + }) + + if output, err := iops.execBoundedAgainst( + restore.Bound, "the database restore", restore.Endpoint.endpointTarget(), serviceTaskManagerDB, + command, restore.execOptions(), + ); err != nil { + return fmt.Errorf("failed to restore the external PostgreSQL database at %s: %w\nOutput: %v", restore.Endpoint.endpointTarget(), err, output) + } + + logrus.Info("PostgreSQL restore completed") + + return nil +} + +// copyDumpIntoRestoreWorkload places the dump in the workload's scratch volume, +// having first established that it fits. +func (iops *InfrahubOps) copyDumpIntoRestoreWorkload(restore *externalRestore, dump string) error { + info, err := os.Stat(dump) + if err != nil { + return fmt.Errorf("cannot read the task-manager dump to restore: %w", err) + } + + if err := iops.refuseUnfittableRestoreDump(restore.Pod, info.Size()); err != nil { + return err + } + + if err := iops.copyToTransientWorkload(restore.Pod, dump, externalPostgresRestoreFile); err != nil { + return fmt.Errorf("failed to copy the task-manager dump into the transient workload: %w", err) + } + + return nil +} + +// refuseUnfittableRestoreDump is the fit check for the workload a restore +// holds: the Job's sized scratch volume, or — in direct mode (FR-032) — the +// free space of the filesystem this process's scratch directory is on, which +// is the only limit there is. +func (iops *InfrahubOps) refuseUnfittableRestoreDump(target string, dumpBytes int64) error { + if local := iops.localWorkloadFor(target); local != nil { + return local.requireFree("the task-manager dump", dumpBytes, "free space there or point TMPDIR at a larger volume") + } + + return refuseUnfittableDump(iops.config.ExternalDB.ScratchSize, dumpBytes) +} + +// refuseUnfittableDump refuses a dump larger than the workload's scratch +// volume, before the copy rather than part-way through it. +// +// The volume is sized by a default the operator can override, so a dump that +// does not fit has a remedy — and hitting it as a full volume mid-transfer +// names neither the size nor the flag, leaving an operator to work backwards +// from a write error inside a pod that no longer exists. +func refuseUnfittableDump(override string, dumpBytes int64) error { + size, err := resolveRestoreScratchSize(serviceTaskManagerDB, override) + if err != nil { + return err + } + + scratch, err := parseStorageQuantity(size) + if err != nil { + return fmt.Errorf("cannot check whether the task-manager dump fits the transient workload's scratch space of %s: %w", size, err) + } + + if dumpBytes > scratch { + return fmt.Errorf( + "the task-manager dump is %s, which does not fit the transient workload's scratch space of %s: supply --external-db-scratch-size with a quantity larger than the dump. "+ + "No data was changed", formatBytes(dumpBytes), size) + } + + return nil +} + +// copyToTransientWorkload copies a file into a named transient pod, bounded +// like every other call this path makes (FR-025). +// +// It names the pod rather than resolving the service, for the reason +// copyFromTransientWorkload does: the workload stands in for a database with no +// container, so resolving `task-manager-db` here would find nothing. +func (iops *InfrahubOps) copyToTransientWorkload(pod, src, dest string) error { + return iops.copyAcrossTransientWorkload(copyIntoTransientWorkload, pod, src, dest) +} diff --git a/src/internal/app/external_restore_test.go b/src/internal/app/external_restore_test.go new file mode 100644 index 00000000..d9f84d7e --- /dev/null +++ b/src/internal/app/external_restore_test.go @@ -0,0 +1,940 @@ +package app + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// externalNeo4jRestore is a prepared restore against a scripted deployment, +// which is what lets the seed, the confirmation and the refusals be driven +// without a cluster and without a database. +// The backend is taken as the interface rather than as scriptedProbeBackend so +// a double that models a *transient* failure can be driven through the same +// setup; the script is only ever used as the deployment. +func externalNeo4jRestore(t *testing.T, backend EnvironmentBackend, seed restoreSeed) (*InfrahubOps, *externalRestore) { + t.Helper() + + iops := &InfrahubOps{config: &Configuration{S3: &S3Config{}}, backend: backend, executor: NewCommandExecutor()} + restore := &externalRestore{ + externalSource: &externalSource{ + Endpoint: &DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "neo4j-a.example", Port: 7687}, {Host: "neo4j-b.example", Port: 7687}}, + Database: "infrahub", + }, + TLS: externalTLSDecision{Service: serviceNeo4j, Encrypt: true}, + Facts: probedFacts{ServerVersion: "2025.10.1", Answered: HostPort{Host: "neo4j-a.example", Port: 7687}}, + }, + Pod: "infrahub-backup-xdb-neo4j-restore-abc123", + Bound: 2 * time.Second, + Seed: seed, + } + iops.recordExternalRestore(serviceNeo4j, restore) + + return iops, restore +} + +// showDatabaseOutput is what `cypher-shell --format plain` prints for the +// status query, with a notice above the header — the shape every parser on this +// path has to survive (carried from T085). +func showDatabaseOutput(rows ...string) string { + lines := []string{ + "info: Bolt server version 2025.10.1", + neo4jStatusColumn + ", " + neo4jStatusMessageColumn, + } + + return strings.Join(append(lines, rows...), "\n") + "\n" +} + +// TestNeo4jSeedStatements pins the Cypher the seed is issued as, in both forms +// the server versions require (research R2). +func TestNeo4jSeedStatements(t *testing.T) { + t.Run("a current server replaces the database atomically", func(t *testing.T) { + statements, err := neo4jSeedStatements("infrahub", "s3://bucket/seed/infrahub-2026.backup", "2025.10.1") + if err != nil { + t.Fatalf("neo4jSeedStatements() = %v, want the statement", err) + } + if len(statements) != 1 { + t.Fatalf("statements = %v, want one: CREATE OR REPLACE is a single destructive step", statements) + } + for _, want := range []string{"CREATE OR REPLACE DATABASE infrahub", "existingData: 'use'", "seedURI: 's3://bucket/seed/infrahub-2026.backup'"} { + if !strings.Contains(statements[0], want) { + t.Errorf("statement = %q, want it to contain %q", statements[0], want) + } + } + }) + + t.Run("a server older than the floor drops before it creates", func(t *testing.T) { + statements, err := neo4jSeedStatements("infrahub", "s3://bucket/seed/artifact.backup", "5.26.1") + if err != nil { + t.Fatalf("neo4jSeedStatements() = %v, want the statements", err) + } + if len(statements) != 2 { + t.Fatalf("statements = %v, want two: CREATE OR REPLACE did not accept a seed URI before %s", statements, neo4jSeedReplaceFloor) + } + if !strings.HasPrefix(statements[0], "DROP DATABASE infrahub IF EXISTS") { + t.Errorf("first statement = %q, want the drop", statements[0]) + } + if !strings.Contains(statements[1], "CREATE DATABASE infrahub") || !strings.Contains(statements[1], "seedURI:") { + t.Errorf("second statement = %q, want the seeded create", statements[1]) + } + }) + + // The calendar scheme is why the comparison cannot be a text one: "2025.01" + // against "5.26" orders backwards as text and correctly as versions. + t.Run("the version comparison spans both numbering schemes", func(t *testing.T) { + for version, atomic := range map[string]bool{ + "5.26.1": false, + "5.26": false, + "2024.12": false, + "2025.01": true, + "2025.10.1": true, + "2026.07": true, + } { + if got := atomicSeedReplaceSupported(version); got != atomic { + t.Errorf("atomicSeedReplaceSupported(%q) = %t, want %t", version, got, atomic) + } + } + }) + + t.Run("an unreadable server version takes the form every version accepts", func(t *testing.T) { + if atomicSeedReplaceSupported("not-a-version") { + t.Error("atomicSeedReplaceSupported(unreadable) = true, want the drop-then-create form that works everywhere") + } + }) + + t.Run("a URI that could close the literal is refused", func(t *testing.T) { + for _, uri := range []string{ + "s3://bucket/a'} ; DROP DATABASE neo4j //", + "s3://bucket/with a space", + "s3://bucket/line\nbreak", + } { + if _, err := neo4jSeedStatements("infrahub", uri, "2025.10.1"); err == nil { + t.Errorf("neo4jSeedStatements(%q) = nil, want the URI refused before it reaches a statement", uri) + } + } + }) + + // Only the object-store providers are built in; every other scheme needs + // dbms.databases.seed_from_uri_providers configured on each server, which + // this run cannot see (research R2). + t.Run("only the built-in cloud seed providers are accepted", func(t *testing.T) { + for _, uri := range []string{"s3://bucket/a.backup", "gs://bucket/a.backup", "azb://bucket/a.backup"} { + if _, err := safeSeedURI(uri); err != nil { + t.Errorf("safeSeedURI(%q) = %v, want it accepted", uri, err) + } + } + + for _, uri := range []string{"https://host/a.backup", "file:///tmp/a.backup", "ftp://host/a.backup", "/tmp/a.backup"} { + err := safeSeedURI2(t, uri) + if err == nil { + t.Fatalf("safeSeedURI(%q) = nil, want a scheme no server supports as shipped to be refused", uri) + } + if !strings.Contains(err.Error(), "s3") { + t.Errorf("err = %v, want it to name the schemes that do work", err) + } + } + }) +} + +// safeSeedURI2 is safeSeedURI's error alone, so the loop above reads as one +// assertion per case. +func safeSeedURI2(t *testing.T, uri string) error { + t.Helper() + + _, err := safeSeedURI(uri) + + return err +} + +// TestNeo4jSeedArtifact covers what the pre-flight will stage, which has to be +// exactly one artifact named from the archive rather than a guess. +func TestNeo4jSeedArtifact(t *testing.T) { + write := func(t *testing.T, names ...string) string { + t.Helper() + + workDir := t.TempDir() + databaseDir := filepath.Join(workDir, "backup", "database") + if err := os.MkdirAll(databaseDir, 0o755); err != nil { + t.Fatalf("creating the extract directory = %v, want nil", err) + } + for _, name := range names { + if err := os.WriteFile(filepath.Join(databaseDir, name), []byte("artifact"), 0o600); err != nil { + t.Fatalf("writing %s = %v, want nil", name, err) + } + } + + return workDir + } + + t.Run("the one artifact an Enterprise archive carries", func(t *testing.T) { + workDir := write(t, "infrahub-2026-09-03T10-00-00.backup") + + artifact, err := neo4jSeedArtifact(workDir) + if err != nil { + t.Fatalf("neo4jSeedArtifact() = %v, want the artifact", err) + } + if filepath.Base(artifact) != "infrahub-2026-09-03T10-00-00.backup" { + t.Errorf("artifact = %q, want the .backup file", artifact) + } + }) + + // A Community archive carries a dump, which no server can be seeded from: + // loading one writes into the server's own store directory. + t.Run("an archive with no online-backup artifact is refused", func(t *testing.T) { + _, err := neo4jSeedArtifact(write(t, "neo4j.dump")) + if err == nil { + t.Fatal("neo4jSeedArtifact() = nil, want the archive refused") + } + for _, want := range []string{neo4jSeedArtifactSuffix, "no data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) + + t.Run("two artifacts are ambiguous rather than a choice", func(t *testing.T) { + _, err := neo4jSeedArtifact(write(t, "infrahub-a.backup", "other-b.backup")) + if err == nil { + t.Fatal("neo4jSeedArtifact() = nil, want the ambiguity refused rather than one picked by sort order") + } + }) +} + +// TestStageNeo4jSeedRefusesWithoutAnObjectStore is FR-007's first arm: the +// server fetches the artifact itself, so a run with nowhere to stage it cannot +// restore — and must say so before anything is stopped (FR-020). +func TestStageNeo4jSeedRefusesWithoutAnObjectStore(t *testing.T) { + workDir := t.TempDir() + databaseDir := filepath.Join(workDir, "backup", "database") + if err := os.MkdirAll(databaseDir, 0o755); err != nil { + t.Fatalf("creating the extract directory = %v, want nil", err) + } + if err := os.WriteFile(filepath.Join(databaseDir, "infrahub-2026.backup"), []byte("artifact"), 0o600); err != nil { + t.Fatalf("writing the artifact = %v, want nil", err) + } + + iops := &InfrahubOps{config: &Configuration{S3: &S3Config{}}, backend: &scriptedProbeBackend{}, executor: NewCommandExecutor()} + + _, err := iops.stageNeo4jSeed(workDir) + if err == nil { + t.Fatal("stageNeo4jSeed() = nil, want the missing object store refused") + } + for _, want := range []string{"--s3-bucket", "fetches the backup artifact itself", "no data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } +} + +// TestVerifySeedReadableRefusesAnUnreadableURI is FR-007's second arm, and the +// one the failure-message contract names: a URI the tool itself cannot read is +// one the database server certainly cannot, and the refusal has to name the URI +// and the server-side prerequisite. +func TestVerifySeedReadableRefusesAnUnreadableURI(t *testing.T) { + t.Run("a URI that is not an object store's is refused", func(t *testing.T) { + _, err := verifySeedReadable(t.Context(), &S3Config{}, "https://artifacts.example/infrahub.backup") + if err == nil { + t.Fatal("verifySeedReadable() = nil, want a non-object-store URI refused") + } + if !strings.Contains(err.Error(), "https://artifacts.example/infrahub.backup") { + t.Errorf("err = %v, want it to name the URI attempted", err) + } + }) + + t.Run("an object that cannot be read back is refused, naming the URI", func(t *testing.T) { + // No endpoint resolves, so the read fails the way an object that is not + // there fails: the point is what the refusal says and that it says it + // before anything destructive. + uri := "s3://infrahub-backups/seed/infrahub-2026.backup" + + _, err := verifySeedReadable(t.Context(), &S3Config{Endpoint: "http://127.0.0.1:1"}, uri) + if err == nil { + t.Fatal("verifySeedReadable() = nil, want the unreadable object refused") + } + for _, want := range []string{uri, "fetch this artifact themselves", "no data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) +} + +// TestSeedPrefixKeepsTheStagedArtifactOutOfRetention pins where the staged copy +// is written. Retention only recognises an object whose key is exactly the one +// an archive of that name would have, so a segment deeper is invisible to it. +func TestSeedPrefixKeepsTheStagedArtifactOutOfRetention(t *testing.T) { + const artifact = "neo4j-2026-10-13T10-24-18.backup" + folder := seedKeySegment + "/neo4j-2026-10-13T10-24-18" + + for prefix, want := range map[string]string{ + "": folder, + "backups": "backups/" + folder, + "backups/": "backups/" + folder, + " /backups/ ": "backups/" + folder, + } { + if got := seedPrefix(prefix, artifact); got != want { + t.Errorf("seedPrefix(%q, %q) = %q, want %q", prefix, artifact, got, want) + } + } + + client := &S3Client{config: &S3Config{Bucket: "infrahub-backups", Prefix: seedPrefix("backups", artifact)}} + key := client.buildS3Key("infrahub_backup_2026-09-03_10-00-00.tar.gz") + if _, recognised := client.backupRefForKey(key); recognised { + t.Errorf("a staged artifact at %q is recognised as a retention candidate, want it invisible to retention", key) + } +} + +// TestSeedPrefixGivesEachArtifactItsOwnFolder pins that two artifacts never +// share a folder. A Neo4j server reads the artifacts beside a seed as one backup +// chain, so an earlier restore's staged copy left in the same folder makes the +// next seed fail ("is not part of a valid backup chain"). +func TestSeedPrefixGivesEachArtifactItsOwnFolder(t *testing.T) { + first := seedPrefix("backups", "neo4j-2026-10-13T10-17-43.backup") + second := seedPrefix("backups", "neo4j-2026-10-13T10-24-18.backup") + + if first == second { + t.Fatalf("two artifacts share the staging folder %q, so one restore's leftover breaks the next", first) + } + if again := seedPrefix("backups", "neo4j-2026-10-13T10-17-43.backup"); again != first { + t.Errorf("restaging the same artifact moved it from %q to %q, want the same folder", first, again) + } +} + +// TestParseNeo4jDatabaseStatuses covers the reading FR-021 rests on, including +// the two shapes that have bitten this path before: a notice above the header, +// and a clustered database reported once per member. +func TestParseNeo4jDatabaseStatuses(t *testing.T) { + t.Run("every member is read, not the first row", func(t *testing.T) { + statuses := parseNeo4jDatabaseStatuses(showDatabaseOutput(`"online", ""`, `"starting", ""`)) + + if len(statuses) != 2 { + t.Fatalf("statuses = %+v, want one per member: a seed that loaded on one server and failed on another is a half-restored database", statuses) + } + if verdict := classifySeedStatuses(statuses); verdict.Online { + t.Errorf("verdict = %+v, want not online while a member is still starting", verdict) + } + }) + + t.Run("a notice above the header is skipped", func(t *testing.T) { + statuses := parseNeo4jDatabaseStatuses(showDatabaseOutput(`"online", ""`)) + + if len(statuses) != 1 || !statuses[0].online() { + t.Fatalf("statuses = %+v, want the one online member: the notice line must not be read as data", statuses) + } + }) + + t.Run("a failed seed is recognised from the status message", func(t *testing.T) { + verdict := classifySeedStatuses(parseNeo4jDatabaseStatuses( + showDatabaseOutput(`"offline", "Unable to start database"`))) + + if verdict.Online { + t.Error("verdict.Online = true on a database the server cannot start") + } + if !verdict.Failed { + t.Error("verdict.Failed = false, want the server's own report of a seed it could not load to end the wait") + } + if !strings.Contains(verdict.Detail, "Unable to start database") { + t.Errorf("verdict.Detail = %q, want the server's own message carried through", verdict.Detail) + } + }) + + t.Run("offline with a message ends the wait", func(t *testing.T) { + // Observed on 2025.10.1: a rejected seed leaves the database offline with + // the reason, and the marker above is not in it. + message := "The backup artifact (s3://infrahub-seed/seed/neo4j-2026-10-13T10-24-18.backup) to restore is not part of a valid backup chain." + verdict := classifySeedStatuses(parseNeo4jDatabaseStatuses( + showDatabaseOutput(`"offline", "` + message + `"`))) + + if !verdict.Failed { + t.Error("verdict.Failed = false, want an offline database with a reason to end the wait instead of running out --external-db-timeout") + } + if !strings.Contains(verdict.Detail, "not part of a valid backup chain") { + t.Errorf("verdict.Detail = %q, want the server's own message carried through", verdict.Detail) + } + }) + + t.Run("offline without a message keeps waiting", func(t *testing.T) { + // A database being replaced can pass through offline with nothing to say. + for _, row := range []string{`"offline", ""`, `"starting", ""`} { + verdict := classifySeedStatuses(parseNeo4jDatabaseStatuses(showDatabaseOutput(row))) + if verdict.Online || verdict.Failed { + t.Errorf("%s: verdict = %+v, want neither: the database may still be starting", row, verdict) + } + } + }) + + t.Run("no rows is neither online nor failed", func(t *testing.T) { + // A database being replaced is briefly absent from the listing, and + // reading that as a failure would fail a restore that is working. + verdict := classifySeedStatuses(parseNeo4jDatabaseStatuses("info: nothing to report\n")) + + if verdict.Online || verdict.Failed { + t.Errorf("verdict = %+v, want neither: the database is not in the listing yet", verdict) + } + }) +} + +// TestRestoreNeo4jExternalRequiresThePreflight is what keeps FR-007's check +// from being a step a later entry point can forget: a restore that reached the +// destructive step with no staged artifact is refused there. +func TestRestoreNeo4jExternalRequiresThePreflight(t *testing.T) { + backend := &scriptedProbeBackend{} + iops, restore := externalNeo4jRestore(t, backend, restoreSeed{}) + + err := iops.restoreNeo4jExternal(restore) + if err == nil { + t.Fatal("restoreNeo4jExternal() = nil, want the missing pre-flight to stop the destructive step") + } + if !strings.Contains(err.Error(), "pre-flight") { + t.Errorf("err = %v, want it to name the pre-flight that was not performed", err) + } + if len(backend.calls) > 0 { + t.Errorf("commands ran = %v, want none: no statement may be issued without a staged artifact", backend.calls) + } +} + +// TestRestoreNeo4jExternalSeedsAndConfirms is the FR-021 pair: the seed is +// issued, and the run does not report success until the database is actually +// online. +func TestRestoreNeo4jExternalSeedsAndConfirms(t *testing.T) { + seed := restoreSeed{URI: "s3://infrahub-backups/seed/infrahub-2026.backup", Bucket: "infrahub-backups", Key: "seed/infrahub-2026.backup", Bytes: 8} + + t.Run("a database that comes online is a successful restore", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{ + // The first read finds it still starting, the second finds it up: + // the wait is what turns an accepted seed into a landed one. + "cypher-shell": {"", showDatabaseOutput(`"starting", ""`), showDatabaseOutput(`"online", ""`)}, + }} + iops, restore := externalNeo4jRestore(t, backend, seed) + // The staged copy is removed only once the database is up, and this + // run has no bucket to remove it from, so the delete's own failure is + // warned about rather than returned. + iops.config.S3 = &S3Config{Endpoint: "http://127.0.0.1:1"} + + if err := iops.restoreNeo4jExternal(restore); err != nil { + t.Fatalf("restoreNeo4jExternal() = %v, want the restore to succeed", err) + } + + calls := backend.argvFor("cypher-shell") + if len(calls) < 2 { + t.Fatalf("cypher-shell calls = %v, want the seed and at least one status read", calls) + } + + seeded := strings.Join(calls[0], " ") + if !strings.Contains(seeded, "CREATE OR REPLACE DATABASE infrahub") || !strings.Contains(seeded, seed.URI) { + t.Errorf("seed statement = %q, want it to seed infrahub from %s", seeded, seed.URI) + } + if !strings.Contains(seeded, "neo4j-a.example:7687") { + t.Errorf("seed statement = %q, want it addressed at the member that answered the probe", seeded) + } + // FR-014: the credentials reach the client through the workload's own + // secret, never through an argument or an exec environment. + for _, call := range backend.calls { + for _, arg := range call { + if strings.Contains(arg, "NEO4J_PASSWORD") || strings.Contains(arg, "-p") && len(arg) > 2 { + t.Errorf("argv %v carries what looks like a credential", call) + } + } + } + }) + + t.Run("a seed the server cannot load fails the run", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{ + "cypher-shell": {"", showDatabaseOutput(`"offline", "Unable to start database"`)}, + }} + iops, restore := externalNeo4jRestore(t, backend, seed) + + err := iops.restoreNeo4jExternal(restore) + if err == nil { + t.Fatal("restoreNeo4jExternal() = nil, want the failed seed reported: accepting the request is not evidence the data loaded") + } + for _, want := range []string{"not usable", "Unable to start database", "neo4j.log", "prior scale"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) + + t.Run("a database that never comes online fails at the bound", func(t *testing.T) { + backend := &scriptedProbeBackend{answers: map[string][]string{ + "cypher-shell": {"", showDatabaseOutput(`"starting", ""`)}, + }} + iops, restore := externalNeo4jRestore(t, backend, seed) + restore.Bound = 50 * time.Millisecond + + err := iops.restoreNeo4jExternal(restore) + if err == nil { + t.Fatal("restoreNeo4jExternal() = nil, want the run to fail rather than report a restore that never landed") + } + for _, want := range []string{"did not come online", "--external-db-timeout", "starting"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) + + t.Run("a server that stops answering fails rather than hanging", func(t *testing.T) { + backend := &scriptedProbeBackend{ + answers: map[string][]string{"cypher-shell": {""}}, + fails: map[string]error{}, + } + iops, restore := externalNeo4jRestore(t, backend, seed) + restore.Bound = 50 * time.Millisecond + + // The seed is accepted and every status read then returns nothing, + // which is a server that answered and said nothing: the window ends the + // wait, so a restore cannot sit with Infrahub scaled to zero (FR-025). + err := iops.restoreNeo4jExternal(restore) + if err == nil { + t.Fatal("restoreNeo4jExternal() = nil, want the bound to end the wait") + } + }) +} + +// TestRefuseExternalCommunityRestore is FR-008's restore side. A Community +// server cannot create the database a seed would populate, so there is no +// statement to issue — and the refusal must not read as a missing container. +func TestRefuseExternalCommunityRestore(t *testing.T) { + endpoint := &DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "neo4j.example", Port: 7687}}, + } + + t.Run("a Community server is refused", func(t *testing.T) { + err := refuseExternalCommunityRestore(endpoint, "community") + if err == nil { + t.Fatal("refuseExternalCommunityRestore(community) = nil, want the refusal") + } + if !errors.Is(err, errExternalCommunityRestore) { + t.Errorf("err = %v, want it to wrap %v", err, errExternalCommunityRestore) + } + for _, want := range []string{"Enterprise", "Nothing is missing from the deployment", "no data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) + + t.Run("an Enterprise server is not refused", func(t *testing.T) { + if err := refuseExternalCommunityRestore(endpoint, "enterprise"); err != nil { + t.Errorf("refuseExternalCommunityRestore(enterprise) = %v, want nil", err) + } + }) + + // The same asymmetry the capture arm documents: an edition the server did + // not report is not read as Community, because guessing the destructive + // branch from a probe that did not answer is what Principle II forbids. + t.Run("an edition the server did not report is not read as Community", func(t *testing.T) { + if err := refuseExternalCommunityRestore(endpoint, ""); err != nil { + t.Errorf("refuseExternalCommunityRestore(unreported) = %v, want nil", err) + } + }) +} + +// TestPostgresRestoreCommand pins the argv of both branches. The internal one +// is asserted character for character against the command that path produced +// before this feature existed (FR-015). +func TestPostgresRestoreCommand(t *testing.T) { + t.Run("the in-container TCP branch is unchanged", func(t *testing.T) { + got := postgresRestoreCommand(postgresRestoreRequest{User: "postgres", File: "/tmp/infrahubops_prefect.dump"}) + want := []string{"pg_restore", "-h", "localhost", "-d", "postgres", "-U", "postgres", "--clean", "--create", "/tmp/infrahubops_prefect.dump"} + + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("command = %v, want %v", got, want) + } + }) + + t.Run("the in-container socket branch is unchanged", func(t *testing.T) { + got := postgresRestoreCommand(postgresRestoreRequest{Socket: true, File: "/tmp/infrahubops_prefect.dump"}) + want := []string{"pg_restore", "-d", "postgres", "--clean", "--create", "/tmp/infrahubops_prefect.dump"} + + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("command = %v, want %v", got, want) + } + }) + + t.Run("the external branch names the resolved host and no credential", func(t *testing.T) { + host := HostPort{Host: "pg.example", Port: 5433} + got := postgresRestoreCommand(postgresRestoreRequest{Host: &host, File: externalPostgresRestoreFile}) + want := []string{"pg_restore", "-h", "pg.example", "-p", "5433", "-d", "postgres", "--clean", "--create", externalPostgresRestoreFile} + + if strings.Join(got, " ") != strings.Join(want, " ") { + t.Errorf("command = %v, want %v", got, want) + } + // FR-014: no role name and no password on the command line; PGUSER and + // PGPASSWORD come from the workload's secret. + for _, arg := range got { + if arg == "-U" { + t.Error("the external restore command names a role on the command line") + } + } + }) +} + +// TestRefuseUnfittableDump keeps the dump's size a refusal that names the flag +// rather than a full volume mid-transfer. +func TestRefuseUnfittableDump(t *testing.T) { + t.Run("a dump that fits the default is accepted", func(t *testing.T) { + if err := refuseUnfittableDump("", 1<<20); err != nil { + t.Errorf("refuseUnfittableDump() = %v, want nil for a 1MiB dump", err) + } + }) + + t.Run("a dump larger than the scratch space names the override", func(t *testing.T) { + err := refuseUnfittableDump("1Gi", 2<<30) + if err == nil { + t.Fatal("refuseUnfittableDump() = nil, want the dump refused before the copy") + } + for _, want := range []string{"--external-db-scratch-size", "1Gi", "No data was changed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) + + t.Run("an override that is not a quantity is refused naming the flag", func(t *testing.T) { + if _, err := resolveRestoreScratchSize(serviceTaskManagerDB, "quite big"); err == nil { + t.Error("resolveRestoreScratchSize() = nil, want a malformed quantity refused before any object is created") + } + }) +} + +// TestRestoreWorkloadSpec covers the one workload a restore uses: its role, its +// deadline covering the operations it hosts, and its credentials reaching it +// through the secret rather than the manifest. +func TestRestoreWorkloadSpec(t *testing.T) { + // A release's configuration, so the spec carries the default image; a + // development build has none and is refused before any spec is built. + withBuildVersion(t, "v2.3.0") + cfg := NewInfrahubOps().Config() + cfg.Neo4jUsername = "neo4j" + cfg.Neo4jPassword = "hunter2-not-in-the-manifest" + + spec, err := restoreWorkloadSpec(cfg, "infrahub", "abc123", serviceNeo4j) + if err != nil { + t.Fatalf("restoreWorkloadSpec() = %v, want the spec", err) + } + + if spec.Role != workloadRoleRestore { + t.Errorf("role = %q, want %q", spec.Role, workloadRoleRestore) + } + if !strings.Contains(spec.jobName(), string(workloadRoleRestore)) { + t.Errorf("Job name = %q, want the role in it so a restore's workload is not a capture's", spec.jobName()) + } + + // The pod has to outlive the copy, the seed and the wait for the database + // to come online, or the cluster removes it from under an operation still + // in flight (FR-025). + if want := externalDBRestoreFullBoundCalls * externalDBBound(cfg); spec.Deadline <= want { + t.Errorf("deadline = %v, want more than the %v of bounded work it hosts", spec.Deadline, want) + } + // The deployment-ID reset runs through this pod after the database is + // online; the pod must outlive every attempt of it as well. + if want := externalDBRestoreFullBoundCalls*externalDBBound(cfg) + resetDeploymentIDBudget(cfg) + externalDBWorkloadReadyTimeout; spec.Deadline <= want { + t.Errorf("deadline = %v, want more than the %v of bounded work it hosts, including the deployment-ID reset", spec.Deadline, want) + } + if spec.Deadline <= externalDBCaptureDeadline(cfg) { + t.Errorf("deadline = %v, want it to exceed a capture's %v: this pod is created at the gate and held for the whole run", + spec.Deadline, externalDBCaptureDeadline(cfg)) + } + + if spec.Credentials["NEO4J_PASSWORD"] != cfg.Neo4jPassword { + t.Errorf("credentials = %v, want the password carried for the owned secret", spec.Credentials) + } + manifest, err := buildTransientWorkloadManifest(spec) + if err != nil { + t.Fatalf("buildTransientWorkloadManifest() = %v, want the manifest", err) + } + if strings.Contains(string(manifest), cfg.Neo4jPassword) { + t.Error("the pod manifest carries the password, want it referenced from the owned secret only (FR-014)") + } +} + +// TestExternalRestoreForIsLocationAuthoritative is the T083 property on the +// restore side: a nil prepared restore is only the internal branch where the +// database actually is internal. +func TestExternalRestoreForIsLocationAuthoritative(t *testing.T) { + locating := func(location EndpointLocation, err error) *InfrahubOps { + backend := newGatedBackend() + backend.locateErr = err + if err == nil { + backend.locations[serviceNeo4j] = location + } + + return &InfrahubOps{config: &Configuration{}, backend: backend, executor: NewCommandExecutor()} + } + + t.Run("an in-deployment database is the internal branch", func(t *testing.T) { + restore, err := locating(EndpointLocationInternal, nil).externalRestoreFor(serviceNeo4j) + if err != nil || restore != nil { + t.Errorf("externalRestoreFor() = %v, %v, want nil, nil: the existing path must be unchanged (FR-015)", restore, err) + } + }) + + t.Run("an external database with nothing prepared is an error, not the internal branch", func(t *testing.T) { + restore, err := locating(EndpointLocationExternal, nil).externalRestoreFor(serviceNeo4j) + if err == nil { + t.Fatalf("externalRestoreFor() = %v, nil, want the unprepared external database reported rather than written to in a container it does not live in", restore) + } + if !strings.Contains(err.Error(), "nothing was prepared to restore into it") { + t.Errorf("err = %v, want it to name what was not done", err) + } + }) + + t.Run("a location that could not be established is an error", func(t *testing.T) { + _, err := locating("", fmt.Errorf("pods is forbidden")).externalRestoreFor(serviceNeo4j) + if err == nil { + t.Fatal("externalRestoreFor() = nil, want the unanswered location query reported (FR-002)") + } + }) +} + +// TestExternalRestoreAuthDescribesItsChannel covers the fact FR-010 turns on: +// which channel authorised a destructive write to unmanaged infrastructure, +// since only one of them is available to an unattended run. +func TestExternalRestoreAuthDescribesItsChannel(t *testing.T) { + for auth, want := range map[ExternalRestoreAuth]string{ + {Source: ExternalRestoreAuthFlag}: "--" + AllowExternalRestoreFlag, + {Source: ExternalRestoreAuthConfig}: AllowExternalRestoreEnvVar, + {}: "no channel", + } { + if got := auth.describe(); !strings.Contains(got, want) { + t.Errorf("describe(%+v) = %q, want it to name %q", auth, got, want) + } + } +} + +// TestPrepareExternalRestoresIsOncePerDatabase covers the gate being called +// twice in one run, which `restore --latest --s3` does: once before the +// download, and again inside RestoreBackup. A second pass must do nothing at +// all — a second workload for the same database carries the same run +// identifier, so it has the same pod name and the create collides. +func TestPrepareExternalRestoresIsOncePerDatabase(t *testing.T) { + backend := newGatedBackend() + backend.locations[serviceNeo4j] = EndpointLocationExternal + iops := &InfrahubOps{config: &Configuration{S3: &S3Config{}}, backend: backend, executor: NewCommandExecutor()} + + // Out of the gate's own constructor rather than written as a literal: the + // preparation refuses a target that did not come from there, because the + // location and FR-009's authorisation are what holding one attests to (see + // databaseTarget.requireGated). + gated, err := databaseTargetWith(deploymentQueries{ + locate: func(string) (EndpointLocation, error) { return EndpointLocationExternal, nil }, + }, databaseRestore, serviceNeo4j, ExternalRestoreAuth{Source: ExternalRestoreAuthFlag}) + if err != nil { + t.Fatalf("databaseTargetWith(restore) = %v, want a target", err) + } + targets := databaseTargets{gated} + + // Nothing is prepared yet, so the first pass reaches for a workload — and + // fails, because this deployment double cannot host one. That failure is + // what proves the second pass below is a no-op rather than the same refusal + // arriving twice. + if err := iops.prepareExternalRestores(targets); err == nil { + t.Fatal("prepareExternalRestores() = nil on an unprepared external database, want the workload attempted") + } + + iops.recordExternalRestore(serviceNeo4j, &externalRestore{ + externalSource: &externalSource{Endpoint: &DatabaseEndpoint{Service: serviceNeo4j, Location: EndpointLocationExternal}}, + Pod: "infrahub-backup-xdb-neo4j-restore-abc123", + }) + + if err := iops.prepareExternalRestores(targets); err != nil { + t.Errorf("prepareExternalRestores() = %v on a database already prepared, want nil: the second pass must not build a second workload", err) + } + if len(iops.unpreparedExternalRestores(targets)) != 0 { + t.Error("the prepared database is still reported as unprepared") + } +} + +// TestWaitForNeo4jRestoreSeparatesTheDatabaseFromThePathToIt is FR-021 read +// against FR-025's attribution. +// +// The wait retried *every* exec failure until the operator's bound — two hours +// by default, with Infrahub scaled to zero — and then reported "the database is +// not usable… Its reported state is the server could not be asked for its +// state: ". A transient pod that was evicted, a `pods/exec` +// permission that was revoked and an API server that has gone all produce +// exactly that, and none of them is a statement about the customer's database, +// which may well be loading the seed. +// +// The bound belongs to the database, because a large store legitimately takes +// hours. The question does not. +func TestWaitForNeo4jRestoreSeparatesTheDatabaseFromThePathToIt(t *testing.T) { + seed := restoreSeed{URI: "s3://infrahub-backups/seed/infrahub-2026.backup", Bucket: "infrahub-backups", Key: "seed/infrahub-2026.backup", Bytes: 8} + denied := errors.New(`Error from server (Forbidden): pods "infrahub-backup-xdb-neo4j-restore-abc123" is forbidden: User cannot create resource "pods/exec"`) + + t.Run("a read that never reached the server is not the database's state", func(t *testing.T) { + backend := &scriptedProbeBackend{fails: map[string]error{"cypher-shell": denied}} + iops, restore := externalNeo4jRestore(t, backend, seed) + // The operator's bound, which is what this must not spend. + restore.Bound = time.Hour + + err := iops.waitForNeo4jRestoreWithin(restore, seed.URI, 0) + if err == nil { + t.Fatal("waitForNeo4jRestoreWithin() = nil, want the run to fail: it cannot tell whether the restore landed") + } + + // Attribution, which is the whole finding: the message must not say the + // database is unusable on evidence that never reached it. + if strings.Contains(err.Error(), "the database is not usable") { + t.Errorf("err = %v, want a failure of the path to the database rather than a verdict on it", err) + } + for _, want := range []string{"pods/exec", "not a report about it", "may still be loading the seed"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + + // And it must not have waited out the hour to say so. + if calls := len(backend.argvFor("cypher-shell")); calls != 1 { + t.Errorf("status reads = %d, want 1: with no grace the first unanswered read is the verdict, not the start of an hour", calls) + } + }) + + t.Run("a bound shorter than the grace still attributes the failure", func(t *testing.T) { + // The grace is a minute and an operator may set --external-db-timeout + // well below it, so the deadline expires first. The last thing that + // happened is still a question that went unanswered. + backend := &scriptedProbeBackend{fails: map[string]error{"cypher-shell": denied}} + iops, restore := externalNeo4jRestore(t, backend, seed) + restore.Bound = 30 * time.Millisecond + + err := iops.waitForNeo4jRestoreWithin(restore, seed.URI, time.Minute) + if err == nil { + t.Fatal("waitForNeo4jRestoreWithin() = nil, want the run to fail") + } + if strings.Contains(err.Error(), "the database is not usable") { + t.Errorf("err = %v, want the unanswered question reported rather than a verdict on the database", err) + } + }) + + t.Run("a moment's refused session is still retried", func(t *testing.T) { + // The condition the grace exists for: a server that has just replaced a + // database refuses a session and then answers. Losing that would turn a + // successful restore into a failed one. + answers := map[string][]string{"cypher-shell": {showDatabaseOutput(`"online", ""`)}} + backend := &recoveringProbeBackend{scriptedProbeBackend: scriptedProbeBackend{answers: answers}, failFirst: denied} + iops, restore := externalNeo4jRestore(t, backend, seed) + restore.Bound = time.Minute + + if err := iops.waitForNeo4jRestoreWithin(restore, seed.URI, time.Minute); err != nil { + t.Fatalf("waitForNeo4jRestoreWithin() = %v, want nil: the second read found the database online", err) + } + }) + + t.Run("a server that answers and is not online is still the database's verdict", func(t *testing.T) { + // The other direction of the same split: a read that *did* reach the + // server keeps the database-attributed message and the operator's bound. + backend := &scriptedProbeBackend{answers: map[string][]string{ + "cypher-shell": {showDatabaseOutput(`"starting", ""`)}, + }} + iops, restore := externalNeo4jRestore(t, backend, seed) + restore.Bound = 30 * time.Millisecond + + err := iops.waitForNeo4jRestoreWithin(restore, seed.URI, time.Minute) + if err == nil { + t.Fatal("waitForNeo4jRestoreWithin() = nil, want the bound to fail the run") + } + for _, want := range []string{"not usable", "did not come online", "starting"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("err = %v, want it to mention %q", err, want) + } + } + }) +} + +// recoveringProbeBackend fails its first command and then behaves like its +// embedded script. It is the "refused a session for a moment" shape, which no +// existing double can express: scriptedProbeBackend's fails map is permanent. +type recoveringProbeBackend struct { + scriptedProbeBackend + + failFirst error + failed bool +} + +func (b *recoveringProbeBackend) ExecSeparateContext(ctx context.Context, bound time.Duration, service string, command []string, opts *ExecOptions) (string, string, error) { + if !b.failed { + b.failed = true + b.calls = append(b.calls, command) + + return "", "", b.failFirst + } + + return b.scriptedProbeBackend.ExecSeparateContext(ctx, bound, service, command, opts) +} + +// TestRefuseExternalCommunityDumpRestoreIsRaisedAtTheGate is Finding 6, and it +// is an ordering rather than a message. +// +// A Neo4j dump is loaded by writing into the server's own store directory, so it +// cannot be loaded into a server this run does not host. Both Plakar restore +// paths knew that from the snapshot's own tags before they touched anything — +// and raised it from inside restoreNeo4jCommunityStream, which they reach only +// after stopAppContainers, confirmAppContainersQuiesced and +// restartDependencies. The run took the deployment down to discover something it +// already knew, and then told the operator "No Infrahub service was stopped", +// which by then was false. Every other refusal on this feature is raised before +// the first destructive step (see external_backup_gate.go). +func TestRefuseExternalCommunityDumpRestoreIsRaisedAtTheGate(t *testing.T) { + seed := restoreSeed{URI: "s3://infrahub-backups/seed/infrahub-2026.backup"} + + t.Run("a dump bound for an external database is refused", func(t *testing.T) { + iops, _ := externalNeo4jRestore(t, &scriptedProbeBackend{}, seed) + + err := iops.refuseExternalCommunityDumpRestore(true) + if err == nil { + t.Fatal("refuseExternalCommunityDumpRestore(true) = nil, want the refusal: a dump cannot be written into a store this run has no access to") + } + // The claim the gate placement is what makes true. + if !strings.Contains(err.Error(), "No Infrahub service was stopped and no data was changed") { + t.Errorf("err = %v, want it to state that nothing was stopped — which is only true at the gate", err) + } + if !strings.Contains(err.Error(), "Enterprise online backup") { + t.Errorf("err = %v, want the artifact an external server can be restored from named", err) + } + }) + + t.Run("an Enterprise artifact bound for an external database is not refused", func(t *testing.T) { + iops, _ := externalNeo4jRestore(t, &scriptedProbeBackend{}, seed) + + if err := iops.refuseExternalCommunityDumpRestore(false); err != nil { + t.Errorf("refuseExternalCommunityDumpRestore(false) = %v, want nil: an online backup is exactly what a remote server can seed from", err) + } + }) + + t.Run("a dump bound for an in-deployment database is not refused", func(t *testing.T) { + // FR-015: the deployment every customer has today streams its dump into + // its own container, and the gate must be invisible to it. + iops := &InfrahubOps{config: &Configuration{}, executor: NewCommandExecutor()} + + if err := iops.refuseExternalCommunityDumpRestore(true); err != nil { + t.Errorf("refuseExternalCommunityDumpRestore(true) = %v with no external database, want nil", err) + } + }) + + t.Run("the backstop inside the load does not claim the deployment is untouched", func(t *testing.T) { + // It is kept because restoreNeo4jCommunityStream is where the container + // is actually needed, and a path added later could reach it without + // passing a gate. What it must not do is repeat the gate's claim. + endpoint := &DatabaseEndpoint{Service: serviceNeo4j, Location: EndpointLocationExternal, Hosts: []HostPort{{Host: "neo4j.example", Port: 7687}}} + + err := refuseExternalDumpRestoreAfterQuiescing(endpoint) + if strings.Contains(err.Error(), "No Infrahub service was stopped") { + t.Errorf("err = %v, want it not to claim nothing was stopped: by this point something has been", err) + } + if !strings.Contains(err.Error(), "returned to their prior scale") { + t.Errorf("err = %v, want it to say what state the deployment is left in", err) + } + }) +} diff --git a/src/internal/app/external_timeout.go b/src/internal/app/external_timeout.go new file mode 100644 index 00000000..c0c1334c --- /dev/null +++ b/src/internal/app/external_timeout.go @@ -0,0 +1,389 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// Every operation against a database outside the deployment is time-bounded, so +// that an endpoint which completes a handshake and then stops responding cannot +// hang the run (FR-025). The hazard is worst on the restore path, where Infrahub +// is scaled down for the whole of the operation's life: an unbounded stall there +// leaves the deployment down indefinitely, which is precisely the outcome +// constitution Principle II forbids. +// +// The bounding mechanism is not new — CommandExecutor already has +// runCommandContext and its pipe variants, which the collect primitives use. +// What was missing is that the external-database paths went through the +// unbounded runCommand and the unbounded Exec instead. This file is where they +// are routed onto the bounded runner, where the bounds are resolved, and where a +// timeout becomes an operator-facing failure, so that the capture and restore +// steps added later are bounded by construction rather than by remembering to +// be. +// +// The bounds are deliberately not one number. A capture of a large store can +// legitimately stall for a long time — research R1 names checkpointing as a +// known cause — so its bound is the operator's --external-db-timeout, generous +// by default. Nothing else in the feature has that excuse. +const ( + // externalDBControlTimeout bounds a control operation: a cluster or + // deployment metadata call made on behalf of an external database, such as + // listing the pods of a namespace, reading a pod's UID, reading a + // component's environment, or deleting a transient workload. None of them + // touches the database and none has any reason to take minutes, so the + // bound is short enough that a wedged API server is reported rather than + // waited out. + externalDBControlTimeout = 2 * time.Minute + + // externalDBProbeTimeout bounds an operation the probe workload performs + // against the database itself. + // + // It is far shorter than the capture bound on purpose. The probe reads + // three facts — the server's version, the utility's version and the store + // size — and a server that has accepted a connection and cannot answer + // those is not slow, it is not answering; there is no legitimate probe that + // takes minutes. Sizing it to the capture bound would let a restore sit + // scaled down for two hours before saying so. It also stays well below + // externalDBProbeDeadline, so the tool's own timeout is what reports the + // stall rather than the cluster removing the pod underneath it. + externalDBProbeTimeout = 5 * time.Minute + + // minExternalDBBound floors the resolved operation bound. + // + // Every other bound in this file is derived from that one by min(), so a + // bound small enough to be absurd is not a slow operation reported early — + // it is an operation that cannot complete. A `kubectl get pods` in the + // location decision takes a fraction of a second against a healthy API + // server and several against a loaded one, and the first thing to fail + // under a microsecond bound is the internal-versus-external decision, on + // an internal deployment that has nothing to do with any of this. The run + // then aborts advising the operator to raise a flag they never passed. + // + // So no configuration resolves below this. It is short enough that an + // operator who wants a tight bound still gets one, and long enough that + // the bound expiring means something stalled. + minExternalDBBound = 30 * time.Second + + // externalDBReadyBoundMargin is added to a readiness wait's own timeout to + // get the bound on the process performing it. + // + // `kubectl wait` is the one cluster call whose duration its own argument + // sets, and it is a watch: if the API server stops answering mid-watch, + // kubectl's --timeout is not what returns. The margin is what makes the + // outer bound fire second rather than first, so a readiness failure is + // still reported as one. + externalDBReadyBoundMargin = 2 * time.Minute +) + +// externalDBBound is the bound on a single operation against an external +// database: the operator's --external-db-timeout, or the default when the +// configuration carries none. +// +// A non-positive value is read as unset rather than as "no bound". Passing zero +// to context.WithTimeout yields an already-expired context, so taking a missing +// setting literally would fail every operation instantly — the opposite of the +// generous default FR-025 asks for, and a far worse failure than the one it +// exists to prevent. +// +// A positive value is floored at minExternalDBBound. resolveExternalDBTimeout +// already applies that floor where the operator's value is read, so this is the +// same rule stated at the point every bound is derived — a Configuration built +// in code, or one whose Timeout was set through a channel that grows later, +// cannot reach the bounded runners with a value nothing could complete in. +func externalDBBound(cfg *Configuration) time.Duration { + if cfg == nil || cfg.ExternalDB.Timeout <= 0 { + return defaultExternalDBTimeout + } + + return max(minExternalDBBound, cfg.ExternalDB.Timeout) +} + +// resolveExternalDBTimeout reads the operator's --external-db-timeout / +// INFRAHUB_EXTERNAL_DB_TIMEOUT value, and is why the configuration channel does +// not go through viper.GetDuration. +// +// viper.GetDuration reads a unit-less value as nanoseconds, so +// INFRAHUB_EXTERNAL_DB_TIMEOUT=3600 — which reads as an hour to anyone who +// writes it — resolves to 3.6µs. Every bound below is derived from it, so the +// first thing to expire is the location decision's `kubectl get pods`, and a +// Kubernetes backup of a deployment whose databases are all internal aborts +// telling the operator to raise a flag they never passed. The flag channel +// rejects the same value outright, because pflag's duration parser requires a +// unit; this is what makes the channel FR-010 requires for unattended runs +// agree with it. +// +// An unparseable value resolves to zero, which the caller reads as "leave the +// bound as it was" — the same reading resolveExternalRestoreAuth gives a +// malformed authorisation. A configuration error is reported and ignored rather +// than made fatal: the value bounds operations against an external database, +// and a deployment with none of those must not fail to back up because an +// unrelated variable was misspelt. +func resolveExternalDBTimeout(raw string) (time.Duration, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return 0, nil + } + + bound, err := time.ParseDuration(trimmed) + if err != nil { + return 0, fmt.Errorf( + "--external-db-timeout / %s is not a duration (%q): it needs a unit — 90s, 30m, 2h, or a combination such as 1h30m. "+ + "A bare number is refused rather than read as nanoseconds, which would bound every operation at a fraction of a millisecond. "+ + "The bound is left at its previous value", + externalDBTimeoutEnvVar, trimmed) + } + if bound <= 0 { + // Not a bound: see externalDBBound for why zero is read as unset + // rather than as "no limit". + return 0, nil + } + if bound < minExternalDBBound { + logrus.Warnf("--external-db-timeout / %s is %v, which is below the %v floor an operation against a database needs; using %v", + externalDBTimeoutEnvVar, bound, minExternalDBBound, minExternalDBBound) + + return minExternalDBBound, nil + } + + return bound, nil +} + +// externalDBControlBound bounds a control operation. It is the shorter of the +// control bound and the configured operation bound: an operator who lowered +// --external-db-timeout below two minutes has said the whole operation must +// finish sooner than that, and a metadata read allowed to outlive it would make +// the setting a lie. +func externalDBControlBound(cfg *Configuration) time.Duration { + return min(externalDBControlTimeout, externalDBBound(cfg)) +} + +// externalDBProbeBound bounds an operation the probe performs, capped against +// the configured bound the same way and for the same reason. +func externalDBProbeBound(cfg *Configuration) time.Duration { + return min(externalDBProbeTimeout, externalDBBound(cfg)) +} + +// isCommandTimeout reports whether an error is a bound expiring rather than the +// operation failing on its own terms. Both shapes count: the executor's +// *timeoutError, and a bare context deadline from a path that bounds something +// other than a subprocess. +func isCommandTimeout(err error) bool { + var timeout *timeoutError + + return errors.As(err, &timeout) || errors.Is(err, context.DeadlineExceeded) +} + +// wrapExternalTimeout turns an expired bound into the failure FR-025 asks for: +// one that says it timed out, names what stalled and what it was talking to, +// states the bound it exceeded, and names the setting that changes it. +// +// Anything that is not a timeout is returned untouched. Labelling an ordinary +// failure — a refused connection, a permission error — as a stall would send the +// operator after the wrong problem, and this wrapper sits on paths where most +// failures are ordinary. +func wrapExternalTimeout(err error, bound time.Duration, operation, target string) error { + if !isCommandTimeout(err) { + return err + } + + return fmt.Errorf("%s against %s timed out at its %s bound: the operation started and then stopped making progress, so check that the endpoint is still responding — or raise --external-db-timeout if it legitimately needs longer: %w", operation, target, bound, err) +} + +// boundedExternalRunner is the timeout-bounded runner every kubectl call made on +// behalf of an external database goes through, in place of the unbounded +// executor.runCommand the shared pod helpers use. +// +// It is shaped as a podRunner so it drops straight into the injection seams the +// package already has — locateServiceWith, getPodForServiceWith, +// transientClusterOps — which is what makes the routing a substitution rather +// than a rewrite. It reads through separatedRunner, so what it returns is the +// command's stdout alone (see there for why a merged stream is a leaked +// Secret); all it adds is FR-025's account of a bound that expired. +func boundedExternalRunner(executor *CommandExecutor, bound time.Duration, target string) podRunner { + run := separatedRunner(context.Background(), executor, bound) + + return func(name string, args ...string) (string, error) { + output, err := run(name, args...) + + return output, wrapExternalTimeout(err, bound, describeExternalCommand(name, args), target) + } +} + +// externalDBRunner is boundedExternalRunner bound to this backend's executor. +func (k *KubernetesBackend) externalDBRunner(bound time.Duration, target string) podRunner { + return boundedExternalRunner(k.executor, bound, target) +} + +// describeExternalCommand names a stalled operation by its command and +// subcommand — `kubectl get`, `kubectl wait`, `kubectl delete` — which is enough +// for an operator to tell a wedged API server from a wedged database. +// +// It quotes no further arguments. They are the only place a value could appear, +// and FR-014 keeps credentials off command lines precisely so that a message +// like this one is safe to print; not printing them anyway keeps that true even +// if a future call site is careless. +func describeExternalCommand(name string, args []string) string { + if len(args) == 0 { + return name + } + + return name + " " + args[0] +} + +// externalDBTarget names the deployment object an operation was talking to, for +// a timeout raised before any endpoint has been resolved: the location query +// that decides internal from external, and the transient workload's own +// lifecycle. Once an endpoint exists, endpointTarget is the more useful name. +func externalDBTarget(subject, namespace string) string { + if strings.TrimSpace(subject) == "" { + return "namespace " + namespace + } + + return fmt.Sprintf("%s in namespace %s", subject, namespace) +} + +// endpointTarget names a resolved endpoint, which is what FR-025 wants a timeout +// to point at: the host list the stalled operation was actually made to. It +// names no credential, for the reason DatabaseEndpoint's own note gives. +func (e DatabaseEndpoint) endpointTarget() string { + if len(e.Hosts) == 0 { + return fmt.Sprintf("the external %s database (no address resolved)", e.Service) + } + + return fmt.Sprintf("the external %s database at %s", e.Service, renderHostPorts(e.Hosts)) +} + +// execBounded is Exec under a bound, returning the command's stdout alone. +// +// Stdout alone, because every caller reads what comes back as data: an `env` +// dump parsed for the deployment's own database settings today, and a version, +// a store size and a member-role listing when the capture path lands. Merged +// output is not that data — it is that data with whatever the container runtime +// had to say mixed into it, and kubectl has something to say on any pod with +// more than one container (`Defaulted container "x" out of: …`). A notice at +// the top of a version read does not fail; it parses, as a version. +// +// A backend that cannot bound an exec, or cannot keep the streams apart, is +// refused rather than silently falling back — which is what the collect +// primitives do for their optional capabilities. The difference is what a +// fallback would cost here: FR-025's guarantee has to be structural, and a path +// that can quietly lose its bound is a path that can hang a restore with +// Infrahub scaled down, while one that can quietly re-merge the streams is the +// corrupted parse above. Both real backends implement it (see the separateExecer +// assertions in environment_docker.go and environment_kubernetes.go), so the +// refusal is unreachable in a real deployment. +func (iops *InfrahubOps) execBounded(ctx context.Context, bound time.Duration, service string, command []string, opts *ExecOptions) (string, error) { + // Direct mode (FR-032): a target this run's own process stands in for runs + // as a child process here, under the same bound and with the same streams + // apart, and with its credentials in the child's environment only. + if local := iops.localWorkloadForOptions(opts); local != nil { + stdout, stderr, err := local.run(ctx, bound, command, opts) + if err != nil { + return stdout, withStderr(err, stderr) + } + + return stdout, nil + } + + backend, err := iops.ensureBackend() + if err != nil { + return "", err + } + + execer, ok := backend.(separateExecer) + if !ok { + return "", fmt.Errorf("cannot run a time-bounded %s command: the %s environment offers no bounded execution with the output streams kept apart, and an unbounded operation against an external database is not permitted", service, backend.Name()) + } + + stdout, stderr, err := execer.ExecSeparateContext(ctx, bound, service, command, opts) + if err != nil { + return stdout, withStderr(err, stderr) + } + + return stdout, nil +} + +// withStderr puts a failed command's own account of itself back into its error. +// +// Keeping the streams apart is about the payload, not about discarding the +// diagnostics: merged output is where a failure's reason used to be readable, +// and dropping it would trade a corrupted parse for an unexplained failure. +// Only the error path reads stderr, so a command that succeeded while writing a +// warning still yields its payload alone. +func withStderr(err error, stderr string) error { + trimmed := strings.TrimSpace(stderr) + if err == nil || trimmed == "" { + return err + } + + return fmt.Errorf("%w: %s", err, trimmed) +} + +// execBoundedAgainst runs a command in a service's container under the supplied +// bound and reports a timeout as a failure naming what it was talking to. +// +// This is the seam every external-database command step goes through in place of +// the unbounded Exec. The capture and restore steps added later MUST call this +// rather than Exec whenever the database they operate on is external: the +// 30-minute stream idle timeout in backup_neo4j.go guards the data stream only, +// and the command run before it — separated there so its logs do not contaminate +// the stream — is exactly the step FR-025 found unbounded. +func (iops *InfrahubOps) execBoundedAgainst(bound time.Duration, operation, target, service string, command []string, opts *ExecOptions) (string, error) { + // One deadline over the whole operation rather than one per phase. + // + // A bounded exec is two cluster calls, not one: the backend resolves the + // service to a pod and then execs into it, and it applies the bound it was + // handed to each of them separately. So a resolution that took the entire + // bound and then answered left a second, full bound for the exec — and the + // operation ran for twice the number its own timeout message reports, which + // is the FR-025 guarantee stated wrongly rather than merely loosely. + // + // A context deadline is what makes the two phases share the budget: every + // bound underneath is applied with context.WithTimeout against this one, and + // that takes whichever of the two expires first. + ctx, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + + output, err := iops.execBounded(ctx, bound, service, command, opts) + + return output, wrapExternalTimeout(err, bound, operation, target) +} + +// sleepUntilNextPoll is the pause between two reads of a state a run is waiting +// on. It reports true once the deadline has passed, so the caller gives its own +// verdict; otherwise it sleeps until the next poll or the deadline, whichever +// is sooner, so the last read lands on the deadline rather than an interval +// past it. +func sleepUntilNextPoll(deadline time.Time, interval time.Duration) (expired bool) { + remaining := time.Until(deadline) + if remaining <= 0 { + return true + } + + time.Sleep(min(remaining, interval)) + + return false +} + +// transientWorkloadTarget names a service's transient workload the way a +// timeout report names what it was waiting on. +func transientWorkloadTarget(service string) string { + return "the transient " + service + " workload" +} + +// removeTransientScratch clears what a run staged in a transient workload's +// scratch volume. It is best-effort: the workload is about to be given back and +// its volume goes with it, so a failure here costs nothing (FR-011, FR-023). +// what names the staged content for the timeout report and the debug line. +func (iops *InfrahubOps) removeTransientScratch(service, what string, opts *ExecOptions, paths ...string) { + if _, err := iops.execBoundedAgainst( + externalDBControlBound(iops.config), "removing "+what, transientWorkloadTarget(service), service, + append([]string{"rm", "-rf"}, paths...), opts, + ); err != nil { + logrus.Debugf("Could not remove %s from the transient %s workload; its scratch volume is removed with it: %v", what, service, err) + } +} diff --git a/src/internal/app/external_timeout_test.go b/src/internal/app/external_timeout_test.go new file mode 100644 index 00000000..5457890d --- /dev/null +++ b/src/internal/app/external_timeout_test.go @@ -0,0 +1,703 @@ +package app + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + "time" +) + +// stallingEndpoint is an operation that accepts and then stops responding: the +// process starts, writes the line that says so, and then never exits. It is the +// same shape command_executor_test.go uses for the bounded pipe runners, and it +// is what FR-025's verification describes — a connection that completes and then +// stalls, as opposed to one that is refused. +// +// `exec sleep` rather than a bare `sleep` so the shell replaces itself with the +// stalling process: the bound must be shown to reach through to what is actually +// hanging, not merely to kill a shell that would have reaped its child anyway. +var stallingEndpoint = []string{"sh", "-c", "echo accepted; exec sleep 60"} + +// testStallBound is short enough that the suite stays fast and long enough that +// process start-up cannot be mistaken for the bound expiring. +const testStallBound = 150 * time.Millisecond + +// TestExternalDBStallTerminatesAtTheBound is FR-025's own verification, minus +// the cluster: an endpoint that accepts and then stalls must terminate the +// operation at the bound rather than hang the run, and the failure must say it +// was a timeout and name what it was talking to. +// +// It drives the runner every external-database kubectl call now goes through, +// with a real stalling child process. Without the bound this test does not fail, +// it never returns — which is exactly the production symptom: on the restore +// path the run would sit there with Infrahub scaled down. +func TestExternalDBStallTerminatesAtTheBound(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceNeo4j, + Location: EndpointLocationExternal, + Hosts: []HostPort{{Host: "core-1.db.example.com", Port: 6362}}, + } + target := endpoint.endpointTarget() + + run := boundedExternalRunner(NewCommandExecutor(), testStallBound, target) + + started := time.Now() + output, err := run(stallingEndpoint[0], stallingEndpoint[1:]...) + elapsed := time.Since(started) + + if err == nil { + t.Fatalf("the operation against a stalled endpoint succeeded (output %q), want a timeout", output) + } + + // Terminated, and terminated at the bound: a generous ceiling, because what + // is being asserted is that the run does not sit there, not the scheduler's + // precision. + if elapsed > 10*time.Second { + t.Fatalf("the operation took %v to terminate, want termination at its %v bound", elapsed, testStallBound) + } + + // The stall was reached: the endpoint answered before it stopped answering, + // so this is the accept-then-stall case rather than a command that never ran. + if !strings.Contains(output, "accepted") { + t.Errorf("output = %q, want the endpoint's own acceptance before the stall", output) + } + + // It is recognised as a bound expiring rather than as an ordinary failure, + // which is what lets the message below be trusted. + if !isCommandTimeout(err) { + t.Fatalf("error = %v (%T), want a timeout", err, err) + } + var timeout *timeoutError + if !errors.As(err, &timeout) { + t.Fatalf("error = %v (%T), want the executor's *timeoutError to survive wrapping with %%w", err, err) + } + if timeout.timeout != testStallBound { + t.Errorf("timeout bound = %v, want the bound the operation was given (%v)", timeout.timeout, testStallBound) + } + + // The message is the actionable half of FR-025: that it timed out, the + // endpoint it timed out against, and the setting that changes the bound. + message := err.Error() + for _, want := range []string{ + "timed out", + "core-1.db.example.com:6362", + serviceNeo4j, + "--external-db-timeout", + testStallBound.String(), + } { + if !strings.Contains(message, want) { + t.Errorf("error message %q does not mention %q", message, want) + } + } +} + +// TestExternalDBStallTerminatesEveryBoundedOperation walks the operations the +// feature performs and asserts each one terminates at its own bound against the +// same stalling endpoint. +// +// One case per bound rather than one for the runner, because the bounds are what +// a regression would break: a control operation quietly re-sized to the capture +// bound would still pass a single-runner test while letting a wedged API server +// hold a scaled-down restore for two hours. +func TestExternalDBStallTerminatesEveryBoundedOperation(t *testing.T) { + cfg := NewInfrahubOps().Config() + + tests := []struct { + name string + bound time.Duration + }{ + {name: "a control operation", bound: externalDBControlBound(cfg)}, + {name: "a probe operation", bound: externalDBProbeBound(cfg)}, + {name: "a capture operation", bound: externalDBBound(cfg)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.bound <= 0 { + t.Fatalf("bound = %v, want a positive bound: a non-positive one expires instantly and fails every operation", tt.bound) + } + + // The configured bound is deliberately generous, so the operation is + // driven at a bound of its own; what the case above pins is that the + // configured value is finite and positive, and what this pins is that + // the runner it will be handed to stops at whatever bound it is given. + run := boundedExternalRunner(NewCommandExecutor(), testStallBound, externalDBTarget("the database", "infrahub")) + + started := time.Now() + if _, err := run(stallingEndpoint[0], stallingEndpoint[1:]...); !isCommandTimeout(err) { + t.Fatalf("error = %v, want a timeout", err) + } + if elapsed := time.Since(started); elapsed > 10*time.Second { + t.Fatalf("the operation took %v to terminate, want termination at its bound", elapsed) + } + }) + } +} + +// TestExternalDBBoundsAreOrdered pins the relationships the bounds have to each +// other. A probe that is allowed the capture's two hours defeats the reason the +// probe is bounded separately at all, and a bound above the configured operation +// limit would make --external-db-timeout a suggestion rather than a limit. +func TestExternalDBBoundsAreOrdered(t *testing.T) { + cfg := NewInfrahubOps().Config() + + if got := externalDBBound(cfg); got != defaultExternalDBTimeout { + t.Errorf("externalDBBound = %v, want the configured default %v", got, defaultExternalDBTimeout) + } + if externalDBControlBound(cfg) >= externalDBProbeBound(cfg) { + t.Errorf("control bound %v is not shorter than the probe bound %v", externalDBControlBound(cfg), externalDBProbeBound(cfg)) + } + if externalDBProbeBound(cfg) >= externalDBBound(cfg) { + t.Errorf("probe bound %v is not shorter than the operation bound %v", externalDBProbeBound(cfg), externalDBBound(cfg)) + } + + t.Run("an operator's lower limit caps every bound", func(t *testing.T) { + lowered := &Configuration{ExternalDB: ExternalDBConfig{Timeout: 30 * time.Second}} + + for name, bound := range map[string]time.Duration{ + "operation": externalDBBound(lowered), + "control": externalDBControlBound(lowered), + "probe": externalDBProbeBound(lowered), + } { + if bound > 30*time.Second { + t.Errorf("%s bound = %v, want no more than the configured 30s", name, bound) + } + } + }) + + t.Run("an unset timeout falls back to the default rather than to none", func(t *testing.T) { + // Zero is the trap: handed to context.WithTimeout it expires + // immediately, so reading it literally would fail every operation + // instantly instead of bounding it generously. + for _, cfg := range []*Configuration{nil, {}, {ExternalDB: ExternalDBConfig{Timeout: -time.Second}}} { + if got := externalDBBound(cfg); got != defaultExternalDBTimeout { + t.Errorf("externalDBBound(%+v) = %v, want %v", cfg, got, defaultExternalDBTimeout) + } + } + }) +} + +// TestWrapExternalTimeoutLeavesOrdinaryFailuresAlone is the other half of the +// message contract. Most failures on these paths are ordinary — a refused +// connection, an RBAC denial — and relabelling one as a stall would send the +// operator after the wrong system and suggest raising a timeout that was never +// reached. +func TestWrapExternalTimeoutLeavesOrdinaryFailuresAlone(t *testing.T) { + ordinary := errors.New("Error from server (Forbidden): pods is forbidden") + + got := wrapExternalTimeout(ordinary, time.Minute, "kubectl get", "the database in namespace infrahub") + if !errors.Is(got, ordinary) { + t.Fatalf("wrapExternalTimeout dropped the original error: %v", got) + } + if got.Error() != ordinary.Error() { + t.Errorf("wrapExternalTimeout rewrote an ordinary failure as %q, want it untouched", got.Error()) + } + + if wrapExternalTimeout(nil, time.Minute, "kubectl get", "anything") != nil { + t.Error("wrapExternalTimeout invented an error where there was none") + } +} + +// TestExternalOperationTargetsNameTheirSubject covers what a timeout message +// points the operator at. FR-025 asks the failure to be actionable, and a +// message that named neither the endpoint nor the namespace would not be. +func TestExternalOperationTargetsNameTheirSubject(t *testing.T) { + t.Run("a resolved endpoint names its hosts", func(t *testing.T) { + endpoint := DatabaseEndpoint{ + Service: serviceNeo4j, + Hosts: []HostPort{{Host: "core-1", Port: 6362}, {Host: "core-2", Port: 6362}}, + } + + got := endpoint.endpointTarget() + for _, want := range []string{serviceNeo4j, "core-1:6362", "core-2:6362"} { + if !strings.Contains(got, want) { + t.Errorf("endpointTarget() = %q, does not mention %q", got, want) + } + } + }) + + t.Run("an unresolved endpoint says so rather than naming nothing", func(t *testing.T) { + endpoint := DatabaseEndpoint{Service: serviceTaskManagerDB} + + if got := endpoint.endpointTarget(); !strings.Contains(got, serviceTaskManagerDB) || !strings.Contains(got, "no address") { + t.Errorf("endpointTarget() = %q, want the service named and the missing address stated", got) + } + }) + + t.Run("a deployment object names its namespace", func(t *testing.T) { + if got := externalDBTarget("the database service", "infrahub"); got != "the database service in namespace infrahub" { + t.Errorf("externalDBTarget() = %q", got) + } + if got := externalDBTarget("", "infrahub"); got != "namespace infrahub" { + t.Errorf("externalDBTarget() with no subject = %q", got) + } + }) +} + +// TestDescribeExternalCommandNamesNoArguments keeps the message safe to print. +// FR-014 keeps credentials off command lines, and naming only the command and +// its subcommand keeps a timeout message harmless even if a future call site +// puts a value in an argument. +func TestDescribeExternalCommandNamesNoArguments(t *testing.T) { + got := describeExternalCommand("kubectl", []string{"get", "secret", "creds", "-o", "jsonpath={.data.password}"}) + if got != "kubectl get" { + t.Errorf("describeExternalCommand() = %q, want only the command and its subcommand", got) + } + + if got := describeExternalCommand("kubectl", nil); got != "kubectl" { + t.Errorf("describeExternalCommand() with no arguments = %q", got) + } +} + +// TestTransientWorkloadOperationsAreBounded is the routing half of T032: the +// lifecycle's cluster calls must go through the bounded runner, and the +// readiness wait must not be bounded at the control bound — two minutes would +// kill the wait long before the ten-minute image pull it exists to cover. +func TestTransientWorkloadOperationsAreBounded(t *testing.T) { + backend := testBackend() + ops := backend.transientOps() + + if ops.query == nil || ops.queryWithin == nil || ops.create == nil || ops.remove == nil { + t.Fatal("transientOps left a cluster operation unbound") + } + + // Every operation the lifecycle performs terminates against a stalled + // endpoint. The query runner is the one that can be driven directly. + started := time.Now() + if _, err := boundedExternalRunner(backend.executor, testStallBound, externalDBTarget("the transient external-database workload", backend.namespace))(stallingEndpoint[0], stallingEndpoint[1:]...); !isCommandTimeout(err) { + t.Fatalf("a stalled lifecycle query returned %v, want a timeout", err) + } + if elapsed := time.Since(started); elapsed > 10*time.Second { + t.Fatalf("a stalled lifecycle query took %v to terminate", elapsed) + } + + cluster := &fakeCluster{uid: "8f14e45f-ea3b-4b0e-9d3e-2a1c5f6b7d80"} + if _, err := backend.createTransientWorkloadWith(cluster.ops(), testCaptureSpec()); err != nil { + t.Fatalf("createTransientWorkloadWith() error = %v", err) + } + + if len(cluster.waitBounds) != 2 { + t.Fatalf("pod creation and readiness asked for %d bounds, want exactly two", len(cluster.waitBounds)) + } + want := externalDBWorkloadReadyTimeout + externalDBReadyBoundMargin + for _, bound := range cluster.waitBounds { + if bound != want { + t.Errorf("workload wait bound = %v, want %v", bound, want) + } + if bound <= externalDBWorkloadReadyTimeout { + t.Errorf("workload wait bound %v does not outlive the %v readiness timeout it wraps", bound, externalDBWorkloadReadyTimeout) + } + } +} + +// TestRunCommandWritePipeContextTimeout covers the one bounded primitive this +// chunk had to add: the manifest piped to `kubectl create` had no bound at all, +// so a wedged API server that accepted the connection and then stopped reading +// would hang workload creation. +func TestRunCommandWritePipeContextTimeout(t *testing.T) { + ce := NewCommandExecutor() + + wait, err := ce.runCommandWritePipeContext(t.Context(), testStallBound, strings.NewReader("manifest"), "sh", "-c", "cat >/dev/null; exec sleep 60") + if err != nil { + t.Fatalf("runCommandWritePipeContext failed to start: %v", err) + } + + started := time.Now() + err = wait() + elapsed := time.Since(started) + + var timeout *timeoutError + if !errors.As(err, &timeout) { + t.Fatalf("error = %v (%T), want *timeoutError", err, err) + } + if elapsed > 10*time.Second { + t.Fatalf("wait() took %v to return, want termination at the %v bound", elapsed, testStallBound) + } +} + +// TestExecBoundedRefusesAnUnboundedBackend pins the decision not to fall back to +// the unbounded Exec. The collect primitives treat bounded execution as an +// optional capability and fall back when a backend lacks it; here the fallback +// would silently un-bound an external-database operation, which on the restore +// path means a stall with Infrahub scaled down. +func TestExecBoundedRefusesAnUnboundedBackend(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}, backend: unboundedBackend{}} + + _, err := iops.execBounded(t.Context(), time.Minute, serviceNeo4j, []string{"env"}, nil) + if err == nil { + t.Fatal("execBounded accepted a backend that cannot bound an exec, want a refusal") + } + if !strings.Contains(err.Error(), "unbounded") { + t.Errorf("error = %q, want it to say the operation would have been unbounded", err) + } +} + +// unboundedBackend is an EnvironmentBackend that does not implement +// contextExecer. Both real backends do, so this shape exists only to assert the +// refusal above. +type unboundedBackend struct{} + +func (unboundedBackend) Name() string { return "unbounded" } +func (unboundedBackend) Detect() error { + return nil +} +func (unboundedBackend) Info() string { return "test" } +func (unboundedBackend) Exec(string, []string, *ExecOptions) (string, error) { + return "", fmt.Errorf("the unbounded exec must not be reached") +} +func (unboundedBackend) ExecStream(string, []string, *ExecOptions) (string, error) { + return "", nil +} +func (unboundedBackend) ExecStreamPipe(string, []string, *ExecOptions) (io.ReadCloser, func() error, error) { + return nil, nil, nil +} +func (unboundedBackend) ExecWritePipe(string, []string, *ExecOptions, io.Reader) (func() error, error) { + return nil, nil +} +func (unboundedBackend) CopyTo(string, string, string) error { return nil } +func (unboundedBackend) CopyFrom(string, string, string) error { return nil } +func (unboundedBackend) Start(...string) error { return nil } +func (unboundedBackend) Stop(...string) error { return nil } +func (unboundedBackend) IsRunning(string) (bool, error) { return false, nil } + +// TestResolveExternalDBTimeoutRefusesAUnitLessValue is the defect at the +// channel FR-010 requires. `INFRAHUB_EXTERNAL_DB_TIMEOUT=3600` reads as an hour +// to whoever writes it; viper.GetDuration read it as 3600 nanoseconds, every +// bound is derived from that value by min(), and the first thing to expire was +// the location decision's `kubectl get pods` — so a Kubernetes backup of a +// deployment whose databases are all internal aborted, advising the operator to +// raise a flag they never passed. The flag channel refuses the same value +// outright, and the two must agree. +func TestResolveExternalDBTimeoutRefusesAUnitLessValue(t *testing.T) { + for _, raw := range []string{"3600", "0.5", "90 s", "1hour", "two hours"} { + bound, err := resolveExternalDBTimeout(raw) + if err == nil { + t.Errorf("resolveExternalDBTimeout(%q) = %v, nil; want a refusal: a value without a unit is read as nanoseconds", raw, bound) + + continue + } + if bound != 0 { + t.Errorf("resolveExternalDBTimeout(%q) = %v alongside an error; want zero so the caller leaves the bound as it was", raw, bound) + } + for _, want := range []string{externalDBTimeoutEnvVar, "--" + externalDBTimeoutFlag, "30m", "2h"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("resolveExternalDBTimeout(%q) error = %v, want it to name %q", raw, err, want) + } + } + } +} + +// TestResolveExternalDBTimeoutAcceptsTheFormsPflagAccepts pins the agreement +// between the two channels: everything the flag's own parser takes resolves to +// the same duration here. +func TestResolveExternalDBTimeoutAcceptsTheFormsPflagAccepts(t *testing.T) { + for raw, want := range map[string]time.Duration{ + "90s": 90 * time.Second, + "30m": 30 * time.Minute, + "2h": 2 * time.Hour, + "1h30m": 90 * time.Minute, + " 45m ": 45 * time.Minute, + "2h0m0s": 2 * time.Hour, + } { + bound, err := resolveExternalDBTimeout(raw) + if err != nil { + t.Errorf("resolveExternalDBTimeout(%q) failed: %v", raw, err) + + continue + } + if bound != want { + t.Errorf("resolveExternalDBTimeout(%q) = %v, want %v", raw, bound, want) + } + } + + t.Run("an unset value leaves the bound alone", func(t *testing.T) { + for _, raw := range []string{"", " ", "0s", "-1h"} { + bound, err := resolveExternalDBTimeout(raw) + if err != nil { + t.Errorf("resolveExternalDBTimeout(%q) failed: %v", raw, err) + + continue + } + if bound != 0 { + t.Errorf("resolveExternalDBTimeout(%q) = %v, want zero: the caller reads that as leave-it-as-it-was", raw, bound) + } + } + }) +} + +// TestExternalDBBoundIsFloored is the second half of the same defect: the +// <= 0 guard let any positive value through, so a bound of a few microseconds +// reached every derived bound and aborted the run. No configuration may resolve +// below the floor, whichever channel supplied it. +func TestExternalDBBoundIsFloored(t *testing.T) { + for _, tiny := range []time.Duration{time.Nanosecond, 3600 * time.Nanosecond, time.Millisecond, 29 * time.Second} { + cfg := &Configuration{ExternalDB: ExternalDBConfig{Timeout: tiny}} + + for name, bound := range map[string]time.Duration{ + "operation": externalDBBound(cfg), + "control": externalDBControlBound(cfg), + "probe": externalDBProbeBound(cfg), + } { + if bound < minExternalDBBound { + t.Errorf("with Timeout %v the %s bound = %v, want no less than the %v floor: nothing completes in that, and the run aborts on an internal deployment", + tiny, name, bound, minExternalDBBound) + } + } + } + + t.Run("the floor is applied where the operator's value is read too", func(t *testing.T) { + bound, err := resolveExternalDBTimeout("1ms") + if err != nil { + t.Fatalf("resolveExternalDBTimeout failed: %v", err) + } + if bound != minExternalDBBound { + t.Errorf("resolveExternalDBTimeout(\"1ms\") = %v, want the %v floor", bound, minExternalDBBound) + } + }) + + t.Run("a bound above the floor is left exactly as configured", func(t *testing.T) { + cfg := &Configuration{ExternalDB: ExternalDBConfig{Timeout: 45 * time.Minute}} + if got := externalDBBound(cfg); got != 45*time.Minute { + t.Errorf("externalDBBound = %v, want the configured 45m untouched", got) + } + }) +} + +// separateBoundedBackend is a backend that bounds an exec and keeps the streams +// apart, scripted with what each one carries. +type separateBoundedBackend struct { + unboundedBackend + stdout string + stderr string + err error +} + +func (b separateBoundedBackend) ExecSeparateContext(context.Context, time.Duration, string, []string, *ExecOptions) (string, string, error) { + return b.stdout, b.stderr, b.err +} + +// deadlineRecordingBackend records the context an exec was handed, so a test +// can assert what the whole operation was bounded by rather than what each +// phase inside it was bounded by. +type deadlineRecordingBackend struct { + unboundedBackend + deadline time.Time + hasDeadline bool +} + +func (b *deadlineRecordingBackend) ExecSeparateContext(ctx context.Context, _ time.Duration, _ string, _ []string, _ *ExecOptions) (string, string, error) { + b.deadline, b.hasDeadline = ctx.Deadline() + + return "", "", nil +} + +// TestExecBoundedAgainstSpendsItsBoundOnce is FR-025 stated correctly rather +// than merely loosely. A bounded exec is two cluster calls — resolve the +// service to a pod, then exec into it — and the backend bounds each of them at +// the bound it was handed. Handing it a context with no deadline gave each +// phase a full bound of its own, so an operation could run for twice the number +// its own timeout message reports. +// +// The assertion is on the deadline rather than on elapsed time, because the +// deadline is the mechanism: every bound underneath is applied with +// context.WithTimeout against this context, and that takes whichever of the two +// expires first. A wall-clock assertion would test the scheduler. +func TestExecBoundedAgainstSpendsItsBoundOnce(t *testing.T) { + const bound = 90 * time.Second + + backend := &deadlineRecordingBackend{} + iops := &InfrahubOps{config: &Configuration{}, backend: backend} + + if _, err := iops.execBoundedAgainst(bound, "reading the server version", "the external neo4j database", serviceNeo4j, []string{"cypher-shell"}, nil); err != nil { + t.Fatalf("execBoundedAgainst() error = %v", err) + } + + if !backend.hasDeadline { + t.Fatal("the exec was handed a context with no deadline, so each phase inside it gets a fresh bound and the operation can run for a multiple of the one reported") + } + if remaining := time.Until(backend.deadline); remaining > bound { + t.Errorf("the operation's deadline is %v away, want no more than the %v bound", remaining, bound) + } +} + +// mergedOnlyBackend bounds an exec but merges the streams. It is the capability +// the collect primitives fall back on, and the one this path must not accept. +type mergedOnlyBackend struct { + unboundedBackend +} + +func (mergedOnlyBackend) ExecContext(context.Context, time.Duration, string, []string, *ExecOptions) (string, error) { + return "", fmt.Errorf("the merged exec must not be reached") +} + +// TestExecBoundedReadsStdoutAlone is the defect T085 removes. Every caller of +// execBounded reads what comes back as data — an `env` dump today, and a +// version, a store size and a member-role listing when the capture path lands — +// and merged output is that data with kubectl's own notices mixed into it. A +// `Defaulted container` line at the top of a version read does not fail: it +// parses, as a version. +func TestExecBoundedReadsStdoutAlone(t *testing.T) { + const notice = `Defaulted container "infrahub-server" out of: infrahub-server, wait-for-db` + + t.Run("a notice on stderr never reaches the payload", func(t *testing.T) { + backend := separateBoundedBackend{ + stdout: "INFRAHUB_DB_ADDRESS=neo4j.example.internal\nINFRAHUB_DB_PORT=7687", + stderr: notice, + } + iops := &InfrahubOps{config: &Configuration{}, backend: backend} + + out, err := iops.execBounded(t.Context(), time.Minute, "infrahub-server", []string{"env"}, nil) + if err != nil { + t.Fatalf("execBounded() error = %v", err) + } + if strings.Contains(out, "Defaulted container") { + t.Errorf("output = %q, want the container runtime's notice kept out of the payload", out) + } + if out != backend.stdout { + t.Errorf("output = %q, want exactly what the command wrote to stdout", out) + } + }) + + t.Run("a failure keeps what the command said about it", func(t *testing.T) { + backend := separateBoundedBackend{ + stderr: `error: unable to upgrade connection: container not found ("infrahub-server")`, + err: fmt.Errorf("exit status 1"), + } + iops := &InfrahubOps{config: &Configuration{}, backend: backend} + + _, err := iops.execBounded(t.Context(), time.Minute, "infrahub-server", []string{"env"}, nil) + if err == nil { + t.Fatal("execBounded() = nil error, want the command's failure") + } + if !strings.Contains(err.Error(), "container not found") { + t.Errorf("err = %v, want it to keep the reason the command gave on stderr", err) + } + }) + + t.Run("a bound expiring is still a timeout the wrapper recognises", func(t *testing.T) { + backend := separateBoundedBackend{stderr: "some noise", err: &timeoutError{timeout: time.Minute}} + iops := &InfrahubOps{config: &Configuration{}, backend: backend} + + _, err := iops.execBoundedAgainst(time.Minute, "reading the database settings", "the infrahub-server container", "infrahub-server", []string{"env"}, nil) + if !isCommandTimeout(err) { + t.Fatalf("err = %v, want the timeout to survive being annotated with stderr", err) + } + if !strings.Contains(err.Error(), "timed out at its") { + t.Errorf("err = %v, want FR-025's timeout message", err) + } + }) + + t.Run("a backend that can only merge is refused, not accepted quietly", func(t *testing.T) { + iops := &InfrahubOps{config: &Configuration{}, backend: mergedOnlyBackend{}} + + _, err := iops.execBounded(t.Context(), time.Minute, serviceNeo4j, []string{"env"}, nil) + if err == nil { + t.Fatal("execBounded accepted a backend that merges the streams, want a refusal: falling back is the corrupted parse") + } + if !strings.Contains(err.Error(), "streams kept apart") { + t.Errorf("err = %q, want it to name the capability that is missing", err) + } + }) +} + +// TestBoundedExternalRunnerReadsStdoutAlone is the defect T151 removes, and it +// is TestExecBoundedReadsStdoutAlone's twin one layer down: the exec path was +// fixed by T085, the kubectl runner every lifecycle query goes through was not. +// It drives real child processes through the real CommandExecutor, because the +// stream separation under test is the executor's — a fake that hands back a +// stdout string would pass with either primitive underneath. +// +// The field that makes the class a leak is the pod UID. kubectl writes its +// notices to stderr on a perfectly healthy call, and merged output puts the +// notice in front of the value: the credential Secret's ownerReference is then +// built from `"\n"`, matches no live pod, and the garbage +// collector never collects the Secret holding the database password (FR-023). +func TestBoundedExternalRunnerReadsStdoutAlone(t *testing.T) { + const ( + uid = "8f14e45f-ea3b-4b0e-9d3e-2a1c5f6b7d80" + warning = "Warning: kubectl plugin 'kubectl-foo' is overshadowed by a similarly named builtin command" + ) + target := externalDBTarget("the transient external-database workload", "infrahub") + + t.Run("a warning on stderr never reaches the UID the Secret is bound to", func(t *testing.T) { + run := boundedExternalRunner(NewCommandExecutor(), time.Minute, target) + + // The lifecycle asks the cluster for the UID through ops.query; here + // that question is answered by a real process that behaves like kubectl + // with a warning to give — the value on stdout, the notice on stderr. + cluster := &fakeCluster{} + ops := cluster.ops() + query := ops.query + ops.query = func(_ string, args ...string) (string, error) { + if strings.Contains(strings.Join(args, " "), "metadata.uid") { + return run("sh", "-c", "echo '"+warning+"' >&2; echo "+uid) + } + + return query("kubectl", args...) + } + + workload, err := testBackend().createTransientWorkloadWith(ops, testCaptureSpec()) + if err != nil { + t.Fatalf("createTransientWorkloadWith() error = %v", err) + } + if workload.JobUID != uid { + t.Errorf("JobUID = %q, want exactly %q: anything else is an ownerReference no live Job matches, and a Secret the garbage collector never collects", workload.JobUID, uid) + } + + if len(cluster.created) != 2 { + t.Fatalf("the lifecycle created %d objects, want the Job and then its Secret", len(cluster.created)) + } + secret := string(cluster.created[1]) + if !strings.Contains(secret, `"uid":"`+uid+`"`) { + t.Errorf("Secret manifest = %s, want its ownerReference to carry the UID %q exactly", secret, uid) + } + if strings.Contains(secret, "Warning") { + t.Errorf("Secret manifest = %s, want kubectl's warning kept out of it", secret) + } + }) + + t.Run("a failure keeps what kubectl said about it", func(t *testing.T) { + run := boundedExternalRunner(NewCommandExecutor(), time.Minute, target) + + _, err := run("sh", "-c", `echo 'Error from server (Forbidden): pods "x" is forbidden: User "system:serviceaccount:infrahub:backup" cannot get resource "pods"' >&2; exit 1`) + if err == nil { + t.Fatal("run() = nil error, want the command's failure") + } + if !strings.Contains(err.Error(), "cannot get resource") { + t.Errorf("err = %v, want it to keep the reason kubectl gave on stderr", err) + } + // FR-019's refusal reads the error: a denial the runner used to leave in + // the discarded output is one workloadPermissionError could never see. + if !isForbiddenError(err) { + t.Errorf("err = %v, want an RBAC denial on stderr to be recognisable as one", err) + } + }) + + t.Run("a bound expiring is still a timeout once stderr is folded in", func(t *testing.T) { + run := boundedExternalRunner(NewCommandExecutor(), testStallBound, target) + + _, err := run("sh", "-c", "echo 'some noise' >&2; exec sleep 60") + if !isCommandTimeout(err) { + t.Fatalf("err = %v, want the timeout to survive being annotated with stderr", err) + } + if !strings.Contains(err.Error(), "timed out at its") { + t.Errorf("err = %v, want FR-025's timeout message", err) + } + }) + + t.Run("the collect primitives' bounded runner answers the same way", func(t *testing.T) { + // getPodForServiceContext and getAllPodsContext parse pod names out of + // this runner's output; a notice in front of the first name is a pod + // that does not exist. + out, err := testBackend().boundedRunner(t.Context(), time.Minute)("sh", "-c", "echo '"+warning+"' >&2; echo infrahub-server-0") + if err != nil { + t.Fatalf("boundedRunner() error = %v", err) + } + if out != "infrahub-server-0" { + t.Errorf("output = %q, want exactly what the command wrote to stdout", out) + } + }) +} diff --git a/src/internal/app/kubernetes_ownership.go b/src/internal/app/kubernetes_ownership.go new file mode 100644 index 00000000..7bee55e0 --- /dev/null +++ b/src/internal/app/kubernetes_ownership.go @@ -0,0 +1,746 @@ +package app + +import ( + "slices" + "strings" + + "github.com/sirupsen/logrus" +) + +// Scaling a workload to zero is the only thing the backup does to a Kubernetes +// namespace that the namespace does not get back by itself, and it is decided +// from a service name that is not Infrahub's to own. `cache` and +// `message-queue` in particular name whatever a namespace happens to co-host: +// `redis-cache`, `othertool-cache-0`, `rabbitmq-message-queue`, +// `nats-message-queue`. Resolving a stop target by "the name contains the +// service" therefore does not resolve Infrahub's workload — it resolves the +// first workload in the namespace whose name reads like one, reports it as +// Infrahub's, stops it and scales it to zero. +// +// A token boundary alone does not separate them either: `redis-cache` names +// `cache` at a boundary exactly as `infrahub-cache-0` does. What separates them +// is not the shape of the name but who the name belongs to, so this file +// answers a narrower question than "which pod is this service?": +// +// does this deployment claim it? +// +// Two things count as a claim, and nothing else does: +// +// - a label. A pod or workload carrying one of serviceLabelKeys set to the +// service — or to the chart's shortened form of it, since `infrahub-server` +// pods carry `infrahub/service=server` — is the resource declaring what it +// is. This is the same evidence locateServiceWith decides on, for the same +// reason: a name resemblance is evidence about a name. +// - a name. Either the service's own name — nothing before it — or a name +// carrying a release prefix the deployment declared; and in either case +// nothing after the service a controller did not generate +// (`task-manager-db-1`, `infrahub-database-0`; see +// controllerGeneratedSuffix). `infrahub-cache-0` is claimed because +// `infrahub-server-…` beside it, labelled `infrahub/service=server`, says +// this deployment's release is named `infrahub`; `othertool-cache-0` is not +// claimed, because nothing in the namespace says `othertool` is this +// deployment; `infrahub-cache-exporter` is not claimed, because a word +// after the service names a different component, whatever precedes it. +// This is what keeps a deployment whose pods are labelled differently +// working, instead of trading a destructive defect for a silently skipped +// stop. +// +// Anything else is *unknown*, and unknown never authorises a stop. Absence of +// evidence is not evidence: neither "no label matched" nor "the query failed" +// says the workload is Infrahub's, and neither says it is not. +// +// Where no ownership evidence exists at all — a namespace in which nothing +// carries a service label — a *prefixed* name is read as the current release +// reads it for the app services, and refused for the two databases. Being +// stricter than the shipped behaviour requires evidence to be strict with, and +// there is none; being permissive about a database means running a capture, a +// scale or a restore against a server that is not the deployment's. Which of +// the two applies is unanchoredNamePolicyFor's answer and no caller's. + +// serviceLabelKeys are the label keys a deployment declares its services with, +// in the order they are trusted. podSelectors builds its selectors from these, +// so the keys the tool queries by and the keys it reads ownership from cannot +// drift apart — and they are the same four externalMisdiagnosisHint names to +// the operator. +// +// infrahub/service comes first because the Infrahub Helm chart and the HA +// deployment examples set it on every component. The generic keys come after +// it because other software in the namespace sets them too: CloudNativePG +// labels its PostgreSQL pods app.kubernetes.io/component=database, the same +// value as the Neo4j service. +var serviceLabelKeys = []string{ + "infrahub/service", + "app.kubernetes.io/component", + "app", + "component", +} + +// infrahubServiceLabelValues is the deployment's own vocabulary: the values a +// service label takes on a resource that belongs to Infrahub. A resource +// declaring one of these is what establishes the release prefix the rest of the +// deployment's names are built from. +// +// `server` is listed beside `infrahub-server` because the Helm chart labels +// those pods with the shortened form (see serviceLabelValues). +var infrahubServiceLabelValues = map[string]bool{ + "infrahub-server": true, + "server": true, + "task-worker": true, + "task-manager": true, + "task-manager-background-svc": true, + "cache": true, + "message-queue": true, + "database": true, + "task-manager-db": true, +} + +// serviceLabelValues returns the values a deployment may label service with: +// the canonical name, plus the chart's shortened form for the services whose +// canonical name carries the product prefix. `infrahub-server` pods carry +// `infrahub/service=server`, which is the one documented reason the name +// fallback exists at all — reading it as a label puts that case back on label +// evidence rather than on a name guess. +func serviceLabelValues(service string) []string { + values := []string{service} + if short := strings.TrimPrefix(service, "infrahub-"); short != service && short != "" { + values = append(values, short) + } + + return values +} + +// declaredServiceIn returns the service the supplied label sets declare, +// reading serviceLabelKeys in the order podSelectors tries them so a resource +// carrying several of the keys is read the same way a selector query would +// have matched it. It returns "" when none of the keys names a service. +func declaredServiceIn(labelSets ...map[string]string) string { + for _, key := range serviceLabelKeys { + for _, labels := range labelSets { + service := declaredServiceValue(labels[key]) + if service == "" { + continue + } + // The shortened `server` is the chart's own form, and only the + // chart's own key carries it. On a generic key it is somebody + // else's vocabulary: the Prefect chart labels the task manager + // `app.kubernetes.io/component: server` beside + // `infrahub/service: task-manager`, and reading the first made the + // task manager declare itself infrahub-server. + if service == infrahubServerShortLabel && key != infrahubServiceLabelKey { + continue + } + + return service + } + } + + return "" +} + +// infrahubServiceLabelKey is the label key the Infrahub chart itself declares +// services with, and infrahubServerShortLabel the shortened value it gives +// infrahub-server under that key. +const ( + infrahubServiceLabelKey = "infrahub/service" + infrahubServerShortLabel = "server" +) + +// declaredServiceValue returns value when it names a service in this +// deployment's own vocabulary, and "" when it does not. +// +// It is what makes "the first key that is set" and "the first key that declares +// a service" different answers, and only the second is ownership evidence. The +// Helm chart labels every pod `app: `, so a resource carrying both +// `app: infrahub` and `infrahub/service: server` declares its service in the +// *later* key — and reading the first non-empty value returned the release +// name, which is not a service, from every pod in the namespace. Nothing then +// declared a service, releasePrefixes found no prefix to anchor a name to, and +// the two decisions that rest on that evidence both failed silently: the +// running check stopped nothing and the location decision read an in-cluster +// database as external. +// +// A value outside the vocabulary is not evidence either way, and is skipped +// rather than returned: `app: redis` on `redis-cache-0` is a co-hosted chart +// describing itself, which is exactly as unclaimed as a pod carrying no label +// at all. +func declaredServiceValue(value string) string { + trimmed := strings.TrimSpace(value) + if !infrahubServiceLabelValues[trimmed] { + return "" + } + + return trimmed +} + +// declaresService reports whether a declared service label value names service, +// allowing for the chart's shortened form. +func declaresService(declared, service string) bool { + if declared == "" { + return false + } + + return slices.Contains(serviceLabelValues(service), declared) +} + +// releasePrefixes reads the deployment's own naming from the resources that +// declared themselves: `infrahub-server-7d9f4-abcde` carrying +// `infrahub/service=server` says the release is named `infrahub`, so +// `infrahub-cache-0` is this deployment's and `othertool-cache-0` is not. +// +// declared maps a resource name to the service value its labels declare. Only +// values in the deployment's own vocabulary are read, so a co-hosted chart's +// pod cannot contribute its prefix. The result is sorted, so a namespace +// produces one stable answer rather than one per map iteration. +func releasePrefixes(declared map[string]string) []string { + prefixes := map[string]bool{} + for name, service := range declared { + if !infrahubServiceLabelValues[service] { + continue + } + if prefix := releasePrefixOf(name, service); prefix != "" { + prefixes[prefix] = true + } + } + + ordered := make([]string, 0, len(prefixes)) + for prefix := range prefixes { + ordered = append(ordered, prefix) + } + slices.Sort(ordered) + + return ordered +} + +// releasePrefixOf returns the text a name carries before the service it +// declares — `infrahub` for (`infrahub-cache-0`, `cache`) — or "" when the name +// starts with the service or does not carry it at a token boundary. An empty +// prefix is not a prefix: a deployment naming its pods bare (`cache`) declares +// nothing about the names of its other services. +func releasePrefixOf(name, service string) string { + index := serviceBoundaryIndex(name, service) + if index <= 0 { + return "" + } + + return name[:index-1] +} + +// nameIsDeploymentOwned reports whether a name-shaped match may be acted on. +// +// A name is read at both ends, and the two ends answer different questions. +// +// What follows the service says whether this is the service's resource at all, +// and it has one answer for every name, whatever precedes it: nothing a +// controller did not generate (see controllerGeneratedSuffix). +// `infrahub-task-manager-db-exporter` carries the deployment's own release +// prefix and is still a metrics sidecar; claiming it read an external RDS as +// the in-deployment database and ran `pg_restore --clean --create` into the +// sidecar. `infrahub-task-manager-db-0` carries `task-manager` at a boundary +// and is the database, not the task manager. +// +// What precedes the service says whose it is, and there are two ways it can be +// this deployment's; they are not the same evidence: +// +// - Nothing (see nameIsServiceOwnName): `cache`, `database-0`, +// `task-manager-db-1`. There is no release name for the match to be +// misreading. This is the shape an externally-managed CloudNativePG +// cluster has — pods `-1`, labelled `cnpg.io/cluster` and so +// declaring nothing this tool reads — and refusing it is what read an +// in-cluster task-manager database as external and refused its restore +// outright. +// - A release prefix this deployment declared. `infrahub-cache-0` is claimed +// because `infrahub-server-…` beside it says this deployment's release is +// named `infrahub`; `postgres-database-0` and `pg-task-manager-db-1` are +// not, because nothing in the namespace says `postgres` or `pg` is this +// deployment. That refusal does not turn on whether some *other* release +// declared itself: a database under a prefix nothing declared is refused +// with `infrahub` declared beside it, and — through +// unanchoredNamePolicyFor — with nothing declared at all. +// +// Every resource the note at the top of this file was written to keep out +// carries something before the service — `redis-cache`, `othertool-cache-0`, +// `rabbitmq-message-queue`, `nats-message-queue`, `postgres-database-0` — so +// the first rule admits none of them; each of them has to clear the second. +func nameIsDeploymentOwned(prefixes []string, name, service string) bool { + if nameIsServiceOwnName(name, service) { + return true + } + if serviceWithGeneratedSuffixIndex(name, service) < 0 { + return false + } + + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix+"-") { + return true + } + } + + return false +} + +// nameIsServiceOwnName reports whether a resource is named for the service +// itself: the service's name, with nothing before it and nothing after it but +// what a controller generated. +// +// It is the one name-shaped claim that needs no release prefix behind it, +// because there is no prefix in it to attribute. `task-manager-db-1` is a +// CloudNativePG cluster named for the service; `postgres-database-0` is another +// product's, and says so in the nine characters before the service name. +func nameIsServiceOwnName(name, service string) bool { + return serviceWithGeneratedSuffixIndex(name, service) == 0 +} + +// serviceWithGeneratedSuffixIndex returns the index at which name carries +// service at a token boundary with nothing after it a controller did not +// generate, or -1 when it does not. +// +// It is the one place the right-hand side of a name is read as a claim. +// nameIsServiceOwnName asks whether the index is 0, nameIsDeploymentOwned +// whether what precedes it is a declared release, and the unanchored tier of +// deploymentClaimsResource whether it exists at all — so the three cannot come +// to disagree about what an `-exporter` is. nameMatchesService is deliberately +// not this: it answers whether a name names the service, which +// `infrahub-cache-exporter` does, and that is the near miss +// logUnclaimedResource reports. +func serviceWithGeneratedSuffixIndex(name, service string) int { + index := serviceBoundaryIndex(name, service) + if index < 0 || !controllerGeneratedSuffix(name[index+len(service):]) { + return -1 + } + + return index +} + +// controllerGeneratedSuffix reports whether what follows the service in a name +// is something a controller generated rather than a word somebody chose, which +// is what separates the service's own resource from a different component named +// after it: +// +// - nothing (`cache`, `infrahub-task-manager-db`) — the workload itself. +// - one segment carrying a digit (`database-0`, `infrahub-task-manager-5f8c9`) +// — a StatefulSet ordinal, or a hash on its own. +// - a digit-carrying segment and then five characters +// (`infrahub-server-7d9f4b6c8-abcde`) — a ReplicaSet pod's template hash +// and random suffix. The suffix may be letters alone, because five +// characters drawn at random often are; the hash slot may not, because +// that is where `exporter`, `migrate` and `db` sit. +// - anything else — a *different* workload named after this one. +// `-exporter` is a metrics sidecar, `-backup-28123456-x9v2t` is a +// CronJob's pod, `-exporter-6b8f7c9d4-lm2xq` is the sidecar's Deployment +// pod, and `-db-0` is what `infrahub-task-manager-db-0` carries after +// `task-manager`. A restore that resolved any of them would scale it and +// then try to load a database into it. +// +// A template hash that happens to contain no digit reads as a word here and is +// declined. That is the safe direction, and a rare one: the hash alphabet is +// six digits to four letters. +func controllerGeneratedSuffix(rest string) bool { + if rest == "" { + return true + } + if !strings.HasPrefix(rest, "-") { + // `databasetool-0` starts with `database` and is not it: the service + // name has to end where a segment ends. + return false + } + + segments := strings.Split(rest[1:], "-") + switch len(segments) { + case 1: + return isGeneratedSegment(segments[0]) + case 2: + return isGeneratedSegment(segments[0]) && isRandomSuffix(segments[1]) + default: + return false + } +} + +// isGeneratedSegment reports whether a name segment is lowercase letters and +// digits with at least one digit among them: an ordinal or a hash, and not a +// word anyone chose. +func isGeneratedSegment(segment string) bool { + digit := false + for _, r := range segment { + switch { + case r >= '0' && r <= '9': + digit = true + case r >= 'a' && r <= 'z': + default: + return false + } + } + + return digit +} + +// isRandomSuffix reports whether a segment has the shape of the five characters +// a controller appends to a pod's name. +func isRandomSuffix(segment string) bool { + if len(segment) != 5 { + return false + } + for _, r := range segment { + if (r < '0' || r > '9') && (r < 'a' || r > 'z') { + return false + } + } + + return true +} + +// unanchoredNamePolicy is what a *prefixed* name match means in a namespace +// that declared no release prefix at all — the one thing the callers of +// deploymentClaimsResource legitimately disagree about. A name that is the +// service's own is not covered by it: that case is decided in +// nameIsDeploymentOwned and decided the same way for every caller. +type unanchoredNamePolicy bool + +const ( + // unanchoredNameClaims keeps the shipped reading of a prefix nothing + // anchors: it claims. It is what the app services use: with nothing in the + // namespace declaring a service label there is no prefix to be strict with, + // and narrowing would leave a workload running through an offline capture + // on a deployment that has always worked — `infrahub-server-7d9f4-abcde` in + // a kustomize namespace has a prefix and nothing to check it against. What + // follows the service is not the policy's to decide; see + // deploymentClaimsResource. + unanchoredNameClaims unanchoredNamePolicy = true + + // unanchoredNameIsNotEvidence refuses it. It is what the databases use, + // because there the wrong answer is not a skipped stop: an unrelated + // `postgres-database-0` accepted as the deployment's own routes a database + // that lives outside the cluster onto the in-deployment path, and the run + // then execs `neo4j-admin` — on restore, a load — inside it. + unanchoredNameIsNotEvidence unanchoredNamePolicy = false +) + +// unanchoredNamePolicyFor is the package's single answer to "what does an +// unanchored prefixed name mean for this service?", and every caller of +// deploymentClaimsResource takes its policy from here. +// +// It used to be one hard-coded constant per call site, which is how the same +// evidence produced two answers to one question. Pod resolution claimed a +// prefixed name nothing anchored while the location decision refused it, so a +// co-hosted `postgres-database-0` was read as external — correctly — and then +// exec'd into anyway by the discovery read that adopts *its* connection URL as +// the endpoint a restore writes into. The policy belongs to the service, not to +// the caller: +// +// - An app service. Refusing a name nothing anchors leaves a workload running +// through an offline capture on a deployment that has always worked, and +// there is no evidence in such a namespace to be strict with. This is the +// case unanchoredNameClaims was written for. +// - A database. Every wrong answer about a database is a command run against +// the wrong server: a capture read from it, a scale applied to it, a +// restore loaded into it. Nothing in this tool stops a database for a +// backup, and the one caller that scales one is a restore starting +// `task-manager-db` — a step it already skips when the workload cannot be +// resolved. So the permissive reading buys nothing and costs an unrelated +// `postgres-database-0` being acted on. +func unanchoredNamePolicyFor(service string) unanchoredNamePolicy { + if service == serviceNeo4j || service == serviceTaskManagerDB { + return unanchoredNameIsNotEvidence + } + + return unanchoredNameClaims +} + +// logUnclaimedResource records a resource this deployment did not claim but +// whose name names the service. That near miss is what an operator +// investigating a skipped stop, an unexpected "external" or an empty pod +// listing needs to see; a name that does not name the service at all is not a +// near miss and is not logged. +// +// It is here rather than at each call site so that nameMatchesService — a +// question about the *shape* of a name — has no caller outside this file, where +// the question every decision actually asks is "does this deployment claim it?". +func logUnclaimedResource(kind, name, declared, service string, prefixes []string) { + if !nameMatchesService(name, service) { + return + } + + logrus.Debugf("Ignoring %s %s for service %s: it names the service but this deployment does not claim it (declared %q, release prefixes %v)", + kind, name, service, declared, prefixes) +} + +// deploymentClaimsResource is the whole rule, in one place: a named resource is +// this deployment's when it declares the service outright, or when it carries a +// name this deployment's declared release prefixes claim. +// +// It exists because two callers were asking the same question in two shapes. +// The running check read a namespace's pods for what they declared and what +// they were called; the location decision read the same pods and expressed the +// same test in its own code, so the two could drift — and they had, on the +// unanchored case, without either saying that it was a choice. +// +// It is a choice, and it is a choice about one case only: a name carrying a +// prefix in a namespace that declared none. The choice is taken from +// unanchoredNamePolicyFor here, so it follows the service rather than the call +// site and the same evidence cannot yield two answers — every caller used to +// pass that policy in, and none could have passed another. What none of them +// does any more is express the rule in its own code: there is one +// implementation, and the asymmetry lives in the policy. +func deploymentClaimsResource(prefixes []string, name, declared, service string) bool { + if declaresService(declared, service) { + return true + } + if nameIsDeploymentOwned(prefixes, name, service) { + return true + } + + // A prefixed name and nothing that declared a prefix to measure it against. + // The policy decides whether an unknown *prefix* claims; what follows the + // service is read the same way as in every other tier, so an + // `infrahub-cache-exporter` is not the cache in a namespace that declared + // nothing any more than in one that declared `infrahub`. + return len(prefixes) == 0 && bool(unanchoredNamePolicyFor(service)) && serviceWithGeneratedSuffixIndex(name, service) >= 0 +} + +// podOwnershipJSONPath lists one `;;;