LCORE-3537: E2E tests rework (lightspeed-core#2421)

* E2E tests rework

* Set default otel env variable values

Author: asimurka

.github/workflows/e2e_tests.yaml

@@ -11,9 +11,32 @@ jobs:
       matrix:
         mode: ["server", "library"]
         environment: ["ci"]
-        e2e_group: [1, 2, 3]
+        # Config-aligned shards (@cfg_*). Packed small groups keep job count reasonable
+        # while avoiding mixed-config restarts inside large suites.
+        shard:
+          - name: default
+            tags: "not @skip and @cfg_default"
+          - name: authorized
+            tags: "not @skip and @cfg_authorized"
+          - name: mcp
+            tags: "not @skip and (@cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth)"
+          - name: rbac
+            tags: "not @skip and @cfg_rbac"
+          - name: skills
+            tags: "not @skip and (@cfg_skills or @cfg_skills_directory)"
+          - name: other
+            tags: "not @skip and (@cfg_rh_identity or @cfg_negative or @cfg_byok_pdf or @cfg_degraded or @cfg_unified)"
+          # Server-only; listed in shard (not matrix.include) so it expands with
+          # mode=server before any library jobs. include would append after library.
+          - name: tls
+            tags: "not @skip and @cfg_tls"
+        exclude:
+          - mode: library
+            shard:
+              name: tls
+              tags: "not @skip and @cfg_tls"
 
-    name: "E2E: ${{ matrix.mode }} mode / ${{ matrix.environment }} / group ${{ matrix.e2e_group }}"
+    name: "E2E: ${{ matrix.mode }} / ${{ matrix.environment }} / ${{ matrix.shard.name }}"
 
     env:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -43,7 +66,7 @@ jobs:
           persist-credentials: ${{ github.event_name != 'pull_request_target' }}
 
           # Fetch submodules (required for lightspeed-providers)
-          submodules: ‘recursive’
+          submodules: "recursive"
 
       - name: Verify actual git checkout result
         run: |
@@ -112,7 +135,7 @@ jobs:
           echo "=== Configuration Summary ==="
           echo "Deployment mode: ${{ matrix.mode }}"
           echo "Environment: ${{ matrix.environment }}"
-          echo "E2E shard (Makefile test-e2e-tagged): @e2e_group_${{ matrix.e2e_group }} (with not @skip)"
+          echo "E2E shard (Makefile test-e2e-tagged): ${{ matrix.shard.tags }}"
           echo "Source config: tests/e2e/configs/run-${{ matrix.environment }}.yaml"
           echo ""
           echo "=== Configuration Preview ==="
@@ -239,8 +262,8 @@ jobs:
           TERM: xterm-256color
           FORCE_COLOR: 1
           E2E_DEPLOYMENT_MODE: ${{ matrix.mode }}
-          # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @e2e_group_* per job).
-          E2E_BEHAVE_TAG_EXPR: "not @skip and @e2e_group_${{ matrix.e2e_group }}"
+          # Matches Makefile test-e2e-tagged / E2E_BEHAVE_TAG_EXPR (one @cfg_* pack per job).
+          E2E_BEHAVE_TAG_EXPR: "${{ matrix.shard.tags }}"
         run: |
           echo "Installing test dependencies..."
           pip install uv

.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml

@@ -270,6 +270,10 @@ spec:
                 value: "$(params.namespace)"
               - name: SNAPSHOT
                 value: $(params.SNAPSHOT)
+              - name: OTEL_SDK_DISABLED
+                value: "true"
+              - name: OTEL_ANONYMIZATION_SECRET
+                value: "lightspeed-stack-otel-anonymization-dev-default"
             image: registry.access.redhat.com/ubi9/ubi-minimal
             script: |
               set +e

.tekton/integration-tests/pipeline/lightspeed-stack-rhelai-test.yaml

@@ -396,6 +396,10 @@ spec:
                 value: "$(params.vllm-api-key)"
               - name: VLLM_MODEL
                 value: "$(params.model)"
+              - name: OTEL_SDK_DISABLED
+                value: "true"
+              - name: OTEL_ANONYMIZATION_SECRET
+                value: "lightspeed-stack-otel-anonymization-dev-default"
             image: registry.access.redhat.com/ubi9/ubi-minimal
             script: |
               set +e

Makefile

@@ -162,11 +162,11 @@ test-e2e: ## Run end to end tests for the service
 test-e2e-local: ## Run end to end tests for the service (no script wrapper)
 	uv run behave --color --format pretty --tags=-skip -D dump_errors=true @tests/e2e/test_list.txt
 
-# Tag-based subsets (@e2e_group_* on feature files). Default runs all groups; override for one shard, e.g.
-#   E2E_BEHAVE_TAG_EXPR='not @skip and @e2e_group_2' make test-e2e-tagged-local
-E2E_BEHAVE_TAG_EXPR ?= not @skip and (e2e_group_1 or e2e_group_2 or e2e_group_3)
+# Tag-based subsets (@cfg_* on features/scenarios). Default runs all config groups; override for one shard, e.g.
+#   E2E_BEHAVE_TAG_EXPR='not @skip and @cfg_authorized' make test-e2e-tagged-local
+E2E_BEHAVE_TAG_EXPR ?= not @skip and (@cfg_default or @cfg_authorized or @cfg_mcp or @cfg_mcp_invalid or @cfg_mcp_api_auth or @cfg_rbac or @cfg_rh_identity or @cfg_negative or @cfg_skills or @cfg_skills_directory or @cfg_byok_pdf or @cfg_tls or @cfg_degraded or @cfg_unified)
 
-test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*)
+test-e2e-tagged: ## Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 	script -q -e -c "uv run behave --color --format pretty --tags=\"$(E2E_BEHAVE_TAG_EXPR)\" -D dump_errors=true @tests/e2e/test_list.txt"
 
 test-e2e-tagged-local: ## Same as test-e2e-tagged without script wrapper

README.md

@@ -920,7 +920,7 @@ test-unit                         Run the unit tests
 test-integration                  Run integration tests tests
 test-e2e                          Run end to end tests for the service
 test-e2e-local                    Run end to end tests for the service (no script wrapper)
-test-e2e-tagged                   Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @e2e_group_*)
+test-e2e-tagged                   Run e2e tests with E2E_BEHAVE_TAG_EXPR (default: all @cfg_*)
 test-e2e-tagged-local             Same as test-e2e-tagged without script wrapper
 benchmarks                        Run benchmarks
 check-types-src                   Check type hints in sources only

docker-compose-library.yaml

@@ -65,6 +65,7 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
       - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]

docker-compose.yaml

@@ -107,6 +107,7 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
       - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
       - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
       - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
     depends_on:
         llama-stack:

docs/testing/e2e_testing.md

@@ -214,7 +214,7 @@ You can put several tags on one scenario. To document why a scenario is skipped,
 
 ## Configuration Files
 
-- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Names like `lightspeed-stack.yaml`, `lightspeed-stack-auth-noop-token.yaml`, `lightspeed-stack-rbac.yaml`, etc.
+- **Lightspeed-stack**: Under `tests/e2e/configuration/server-mode/` and `library-mode/`. Switched via `switch_config()` and copied into the container's config path (or applied via ConfigMap in Prow). Bootstrap: `lightspeed-stack.yaml`; variants: `lightspeed-stack-default.yaml`, `lightspeed-stack-authorized.yaml`, `lightspeed-stack-rbac.yaml`, etc. (see `tests/e2e/configuration/grouped/README.md`).
 - **Llama Stack**: Under `tests/e2e/configs/` (e.g. `run-ci.yaml`). Used by the Llama Stack container; not switched by Behave step-by-step, but the stack is started with the appropriate run config.
 
 See `tests/e2e/configuration/README.md` for a short description of each config.

tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml

@@ -57,6 +57,10 @@ spec:
         # Unused for server-mode FAISS (llama pod owns the fixture); keep out of ~/.llama.
         - name: KV_RAG_PATH
           value: "/app-root/.e2e-rag-work/kv_store.db"
+        - name: OTEL_SDK_DISABLED
+          value: "true"
+        - name: OTEL_ANONYMIZATION_SECRET
+          value: "lightspeed-stack-otel-anonymization-dev-default"
         - name: VLLM_MODEL
           valueFrom:
             secretKeyRef:

tests/e2e/configuration/README.md

@@ -54,8 +54,27 @@ Both server-mode and library-mode default configurations include:
 
 3. **User Data Collection** - Enabled for feedback and transcripts testing
 
-### Special-Purpose Configurations
+### Config variants (`@cfg_*` tags)
+
+Named configs merge compatible options and map to `@cfg_*` Behave tags so CI
+can shard by config (fewer restarts per job). See
+[grouped/README.md](grouped/README.md).
+
+| Config | Purpose |
+|--------|---------|
+| `lightspeed-stack-default.yaml` | Default + inline RAG |
+| `lightspeed-stack-authorized.yaml` | Authorization (noop-token) |
+| `lightspeed-stack-negative.yaml` | No cache + invalid feedback storage |
+| `lightspeed-stack-rbac.yaml` | RBAC (jwk-token auth) |
+| `lightspeed-stack-rh-identity.yaml` | RH Identity header auth |
+| `lightspeed-stack-skills.yaml` / `-skills-directory.yaml` | Agent skills |
+| `lightspeed-stack-mcp.yaml` | All valid MCP servers (`mcp_servers_api`) |
+| `lightspeed-stack-mcp-invalid.yaml` | Invalid MCP file token |
+| `lightspeed-stack-mcp-api-auth.yaml` | MCP server API auth |
+| `lightspeed-stack-mcp-{file,kubernetes,client,oauth}-auth.yaml` | Per-auth MCP (`mcp.feature`) |
+| `lightspeed-stack-byok-pdf.yaml` | BYOK PDF (library only) |
+| `lightspeed-stack-tls.yaml` / `-degraded.yaml` | TLS / degraded mode (server only) |
+
+Bootstrap default: `lightspeed-stack.yaml`. Environment-specific:
+`lightspeed-stack-rhelai.yaml`, `lightspeed-stack-rhoai.yaml`.
 
-- `lightspeed-stack-auth-noop-token.yaml` - For authorization testing
-- `lightspeed-stack-invalid-feedback-storage.yaml` - For negative feedback testing
-- `lightspeed-stack-no-cache.yaml` - For cache-disabled scenarios