databricks.yml

# OntoBricks Dev Sandbox — Databricks Asset Bundle
#
# This bundle deploys two apps:
#
# - ``ontobricks_dev_app`` → app name ``ontobricks-030`` (FastAPI UI,
#   ``source_code_path: .``). Uses the repository root as source.
# - ``mcp_ontobricks_app`` → app name ``mcp-ontobricks`` (FastMCP
#   server, ``source_code_path: src/mcp-server``). Calls back to
#   the main app over HTTP.
#
# The bundle is intentionally separate from any previous
# ``ontobricks_bundle`` deployment:
#
# - Distinct ``bundle.name`` (``ontobricks_dev_bundle``) so DAB
#   state does not collide.
# - Distinct ``workspace.root_path`` so the sync directory and the
#   ``.bundle/state/`` tracking files live under their own folder
#   (``/Workspace/Users/<me>/.bundle/ontobricks-030/<target>``).
#
# When this bundle takes over an app that was previously deployed
# from a different bundle (e.g. ``ontobricks_bundle``), DAB requires
# a one-time bind step:
#
#   databricks bundle deployment bind mcp_ontobricks_app mcp-ontobricks --auto-approve
#
# ``scripts/deploy.sh --bind`` does both binds in one go.
#
# Source code paths:
# - The main app is shipped from the repository root. The
#   sandbox auto-detects its app name (``ontobricks-030``) from the
#   Databricks-Apps-injected ``DATABRICKS_APP_NAME`` env var — see
#   ``Settings.ontobricks_app_name`` for the alias-chain logic.
# - The MCP server is shipped from ``src/mcp-server`` (its own
#   ``pyproject.toml`` + ``app.yaml`` + ``server/`` package live there).
#
# Usage:
#   databricks bundle validate -t dev-lakebase                  # validate
#   databricks bundle deploy   -t dev-lakebase                  # deploy both apps + Lakebase binding
#   databricks bundle run ontobricks_dev_app -t dev-lakebase    # start the UI
#   databricks bundle run mcp_ontobricks_app -t dev-lakebase    # start the MCP server
#   databricks bundle deploy   -t dev                           # sandbox target (no explicit postgres binding)

bundle:
  name: ontobricks_dev_bundle

# Resource definitions kept out of this file. Currently the serverless graph
# analytics job (``resources/graph_analytics.job.yml``); the two apps are still
# declared inline below. Per-instance targets (``dev-lakebase-<id>``)
# are written to ``resources/_generated_instance_target.yml``
# by ``scripts/deploy.sh`` from ``DEFAULT_INSTANCE_ID`` — that file is
# gitignored and also matched by the include below.
include:
  - resources/*.yml

workspace:
  # Sandbox path — distinct from any other ``ontobricks*`` bundle so
  # we cannot share/clobber sync state with the production apps.
  root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${var.app_name}/${bundle.target}

sync:
  # DAB syncs git-tracked files minus .gitignore, then applies include/exclude
  # (gitignore syntax). `.databricksignore` is a human/test mirror of this list
  # — the CLI does **not** read it. Keep both files aligned.
  #
  # Runtime payload: src/ (including mcp-server + jobs), run.py, app.yaml,
  # pyproject.toml / uv.lock / requirements.txt, LICENSE.txt / NOTICE.txt,
  # and the Help Center set under docs/ (catalogued *.md + images/ + screenshots/).
  include:
    # app.yaml is .gitignored (it is rendered from app.yaml.template at deploy
    # time with workspace-specific values).  Explicitly include it here so DAB
    # uploads the rendered file on every `databricks bundle deploy` run.
    - app.yaml
    - src/mcp-server/app.yaml
    # Help Center serves markdown from docs/ at runtime (/api/help/docs/*).
    # Force-include tracked Help Center files even if a broader ignore matches.
    - docs/**
  exclude:
    - tests/
    - data/
    - .venv/
    - .git/
    - .github/
    - .cursor/
    - .claude/
    - .planning/
    - .hypothesis/
    - .superpowers/
    - .pytest_cache/
    - htmlcov/
    - logs/
    - fastapi_session/
    - mlruns/
    - mlartifacts/
    - artifacts/
    - changelogs/
    - ci/
    - licenses/
    - releases/
    - scripts/
    # DAB still *parses* resources/*.yml locally via `include:`; this only
    # keeps the YAML out of the Apps workspace files tree.
    - resources/
    - "*.pid"
    - "*.log"
    - ".env"
    - databricks.yml
    - .databricksignore
    - .gitignore
    - .cursorrules
    - .env.example
    - .pre-commit-config.yaml
    - pytest.ini
    - commitlint.config.js
    - mypy_baseline.txt
    - /Makefile
    - CHANGELOG.md
    # Keep both README.md files. CLI 0.298 applies even `/README.md`
    # recursively, which removes docs/README.md and 404s Help Overview.
    - /CONTRIBUTING.md
    - /AGENTS.md
    - /CLAUDE.md
    - /SECURITY.md
    - /CONTRIBUTORS.md
    - src/.coding_rules.md
    # Do NOT exclude src/mcp-server/ — both apps share one workspace files tree;
    # MCP `source_code_path` is ./src/mcp-server.
    # Do NOT exclude src/jobs/ — graph_analytics_job.py is a workspace file.
    # docs/ ships ONLY the Help Center runtime set (catalogued *.md +
    # images/, screenshots/). Everything else under docs/ is internal-only.
    - docs/sphinx/
    - docs/superpowers/
    - docs/diagrams/
    - docs/pr47-neo4j-demo/

variables:
  # ── App names ──────────────────────────────────────────────────────
  # Single source of truth for the deployed app names.  Overridden by
  # deploy.config.sh via ``--var=app_name=…`` so you only need to
  # change DEFAULT_APP_NAME there.
  app_name:
    description: "Databricks Apps name for the main FastAPI UI app"
    default: "ontobricks-030"
  mcp_app_name:
    description: "Databricks Apps name for the MCP companion server"
    default: "mcp-ontobricks"

  warehouse_id:
    description: "SQL Warehouse ID for query execution"
    default: "d2096aa075ad44a3"
  registry_catalog:
    description: "Unity Catalog catalog for the domain registry"
    default: "benoit_cayla"
  registry_schema:
    description: "Unity Catalog schema for the domain registry"
    default: "ontobricks_030"

  # ── Lakebase resource binding (Autoscaling-only) ────────────────
  # OntoBricks targets **Lakebase Autoscaling** exclusively (the
  # default tier for every instance created after 2026-03-12). The
  # legacy Provisioned tier is no longer supported — the runtime
  # auth code (``LakebaseAuth``) and the admin UI both assume an
  # Autoscaling project resource.
  #
  # Autoscaling uses the Apps ``postgres: { branch, database }``
  # binding where both fields are *full project-relative resource
  # paths*. New instances created via ``databricks api create
  # /database/instances`` show up as Autoscaling projects with a
  # default branch named ``production`` (NOT ``main``).
  #
  # Override defaults for your Autoscaling project / branch / database.
  # Example: ``-t dev-lakebase --var=lakebase_project=my-project``.
  #
  # The Apps ``postgres`` binding requires **full** branch and database
  # resource paths. The bundle composes them as
  # ``projects/<project>/branches/<branch>`` and
  # ``…/databases/<lakebase_database_resource_segment>``.
  #
  # **Important:** The final segment must be the Lakebase **resource id**
  # (``db-…``) from the API ``name`` field — **not** the Postgres
  # ``datname`` / ``status.postgres_database`` string (e.g. you cannot use
  # ``databricks_postgres`` or ``ontobricks_registry`` here even though
  # those appear in the SQL UI). Resolve the id with::
  #
  #   databricks postgres list-databases \
  #     "projects/<project>/branches/<branch>" -o json
  #
  # Pick the row whose ``status.postgres_database`` matches the database
  # you want, then set ``lakebase_database_resource_segment`` to the
  # substring after ``…/databases/`` in that row's ``name``.
  #
  # Caveat — **resource id** vs **registry schema**: ``ontobricks_registry``
  # in OntoBricks is usually the **schema** (``lakebase_registry_schema`` /
  # ``LAKEBASE_SCHEMA``), not the ``db-…`` segment. A separate Postgres
  # database named ``ontobricks_registry`` would show both in
  # ``status.postgres_database`` and as its own ``db-…`` row — do not
  # confuse schema name with resource id.
  #
  # ``lakebase_registry_schema`` is the OntoBricks registry **Postgres
  # schema** (default ``ontobricks_registry``). Mirror it in
  # ``app.yaml`` / ``src/mcp-server/app.yaml`` as ``LAKEBASE_SCHEMA``.
  lakebase_project:
    description: "Lakebase Autoscaling project id (``projects/<this>/…``)"
    default: "ontobricks-app"
  lakebase_branch:
    description: "Branch id within the project (resource segment after ``…/branches/``, e.g. ``production``)"
    default: "v030"
  lakebase_database_resource_segment:
    description: "Final segment of Postgres API ``name`` (``db-…``) for the Apps ``postgres.database`` path — from ``list-databases``, not ``datname``"
    # Example: row with ``status.postgres_database`` ``ontobricks_registry`` (or
    # ``databricks_postgres`` on older single-DB layouts) on
    # ``ontobricks-app`` / ``production`` — replace when your project/branch/DB differs.
    #
    # Print each ``db-…`` segment next to ``postgres_database`` (pick the row, use the first column)::
    #
    #   databricks postgres list-databases \
    #     "projects/ontobricks-app/branches/production" -o json \
    #     | jq -r '.[] | [.name | split("/") | last, .status.postgres_database] | @tsv'
    #
    # Same without ``jq`` (one line)::
    #
    #   databricks postgres list-databases "projects/ontobricks-app/branches/production" -o json | python3 -c "import json,sys;[print(r['name'].split('/')[-1], (r.get('status')or{}).get('postgres_database')) for r in json.load(sys.stdin)]"
    #
    # Replace ``ontobricks-app`` / ``production`` with your ``lakebase_project`` / ``lakebase_branch`` when they differ.
    default: "db-8u4l-0na72ub5sp"
  lakebase_registry_schema:
    description: "OntoBricks registry schema inside Postgres (``LAKEBASE_SCHEMA`` in app.yaml)"
    default: "ontobricks_registry"

resources:
  apps:
    # ── Dev sandbox app (FastAPI UI) ────────────────────────────
    ontobricks_dev_app:
      name: "${var.app_name}"
      description: "OntoBricks (development sandbox app)"
      source_code_path: .
      # App ACL (not top-level bundle permissions — those reject CAN_USE; see #134).
      permissions:
        - level: CAN_MANAGE
          user_name: ${workspace.current_user.userName}
        - level: CAN_USE
          group_name: users
      resources:
        - name: sql-warehouse
          description: "SQL Warehouse for executing queries and metadata operations"
          sql_warehouse:
            id: ${var.warehouse_id}
            permission: CAN_USE

    # ── MCP companion server ────────────────────────────────────
    # FastMCP server exposing OntoBricks Digital Twin tools to the
    # Databricks Playground. Deployed from ``src/mcp-server`` (its own
    # ``pyproject.toml`` + ``app.yaml``). The MCP server's service
    # principal needs ``CAN_USE`` on ``ontobricks-030`` so it can call
    # back into the UI's REST API — ``scripts/deploy.sh`` grants this
    # automatically after every deploy.
    mcp_ontobricks_app:
      name: "${var.mcp_app_name}"
      description: "OntoBricks MCP companion server (FastMCP)"
      source_code_path: ./src/mcp-server
      # App ACL (not top-level bundle permissions — those reject CAN_USE; see #134).
      permissions:
        - level: CAN_MANAGE
          user_name: ${workspace.current_user.userName}
        - level: CAN_USE
          group_name: users
      resources:
        - name: sql-warehouse
          description: "SQL Warehouse for executing queries and metadata operations"
          sql_warehouse:
            id: ${var.warehouse_id}
            permission: CAN_USE

targets:
  # Legacy unsuffixed targets — still valid via
  # ``DAB_TARGET=dev-lakebase make deploy``. New instances use
  # ``dev-lakebase-<INSTANCE_ID>`` generated into
  # ``resources/_generated_instance_target.yml`` so each instance keeps
  # a separate ``.databricks/bundle/<target>/`` Terraform state.

  # Default sandbox target (no explicit Lakebase postgres binding).
  dev:
    mode: development
    default: true

  # Sandbox with the Lakebase Postgres registry backend bound
  # (Autoscaling-only). Uses the Apps ``postgres`` resource type
  # with ``branch`` + ``database`` resource paths — both must be
  # full project-relative paths, not bare names. See the
  # ``lakebase_*`` variables above; paths are composed automatically.
  #
  # The Apps runtime injects PGHOST / PGPORT / PGDATABASE /
  # PGUSER / PGAPPNAME / PGSSLMODE for the bound database, so the
  # in-app code path (``LakebaseAuth``) keeps working without any
  # extra config.
  dev-lakebase:
    mode: development
    resources:
      apps:
        ontobricks_dev_app:
          resources:
            - name: postgres
              description: "Lakebase Autoscaling Postgres database for the OntoBricks registry"
              postgres:
                branch: projects/${var.lakebase_project}/branches/${var.lakebase_branch}
                database: projects/${var.lakebase_project}/branches/${var.lakebase_branch}/databases/${var.lakebase_database_resource_segment}
                permission: "CAN_CONNECT_AND_CREATE"
            # Neo4j is optional and configured post-deploy in Settings → Neo4j
            # (Databricks secret scope/key at runtime). Do NOT bind an Apps
            # secret resource here — a missing workspace secret fails
            # ``terraform apply`` on first deploy (GH #136). The unbound
            # ``neo4j-password`` declaration in ``app.yaml.template`` remains
            # for legacy ``NEO4J_PASSWORD`` / Apps UI binding only.
            # See docs/pr47-neo4j-demo/secret-configuration.md.
        mcp_ontobricks_app:
          resources:
            - name: postgres
              description: "Lakebase Autoscaling Postgres database for the OntoBricks registry"
              postgres:
                branch: projects/${var.lakebase_project}/branches/${var.lakebase_branch}
                database: projects/${var.lakebase_project}/branches/${var.lakebase_branch}/databases/${var.lakebase_database_resource_segment}
                permission: "CAN_CONNECT_AND_CREATE"