Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions src/skillspector/llm_analyzer_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
outcome_for_llm_batch_failure,
)
from skillspector.llm_utils import (
AgentCLIChatModel,
StructuredOutputParseError,
_AgentCLIMessage,
_ainvoke_with_usage,
Expand Down Expand Up @@ -135,6 +136,33 @@ def _uses_native_connection_retries(
return False


def _retarget_request_timeout(chat_model: object, timeout: float | None) -> bool:
"""Point an existing chat model at *timeout* and report whether it took effect.

Returns ``False`` for transports that keep no mutable deadline, so the caller can
fall back to constructing a replacement model for that call.
"""
if isinstance(chat_model, ChatOpenAI):
clients = (chat_model.root_client, chat_model.root_async_client)
if any(client is None for client in clients):
return False
for client in clients:
client.timeout = timeout
chat_model.request_timeout = timeout
return True
if isinstance(chat_model, ChatAnthropic):
# ``timeout <= 0`` is how ChatAnthropic spells "leave the SDK default alone"; an
# expired deadline never reaches here because ``_require_time_remaining`` raises first.
for client in (chat_model._client, chat_model._async_client):
client.timeout = timeout
chat_model.default_request_timeout = timeout
return True
if isinstance(chat_model, AgentCLIChatModel):
chat_model.set_timeout(timeout)
return True
return False


# ONE BUDGET FOR THE PROCESS, AND IT CANNOT BE AN asyncio.Semaphore.
#
# The analyzers are separate graph nodes and each node is a *synchronous* function: it reaches
Expand Down Expand Up @@ -690,6 +718,10 @@ def _model_for_call(self) -> tuple[object, object | None]:
remaining = self._require_time_remaining()
if not self._dynamic_timeout:
return self._llm, self._structured_llm
if _retarget_request_timeout(self._llm, remaining):
# Native retries were already disabled for the dynamic-deadline case in
# ``__init__``, and the structured runnable wraps this same model instance.
return self._llm, self._structured_llm
llm = get_chat_model(model=self.model, timeout=remaining)
_uses_native_connection_retries(llm, max_retries=0)
structured = (
Expand Down
32 changes: 13 additions & 19 deletions src/skillspector/llm_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,19 +252,11 @@ class _StructuredAgentCLIModel:
``complete()``, then parses and validates the response into *schema*.
"""

def __init__(
self,
provider: object,
model: str,
max_output_tokens: int,
schema: type,
timeout: float | None = None,
) -> None:
self._provider = provider
self._model = model
self._max_output_tokens = max_output_tokens
def __init__(self, owner: AgentCLIChatModel, schema: type) -> None:
# Read transport settings through the owner so a structured wrapper does not pin the
# deadline it happened to be created with.
self._owner = owner
self._schema = schema
self._timeout = timeout

def _augment(self, prompt: str) -> str:
schema_json = json.dumps(self._schema.model_json_schema(), indent=2)
Expand All @@ -278,11 +270,11 @@ def _augment(self, prompt: str) -> str:
def _complete(self, prompt: str) -> str:
"""Return provider output before structured parsing begins."""
return _complete_agent_cli(
self._provider,
self._owner._provider,
self._augment(prompt),
model=self._model,
max_output_tokens=self._max_output_tokens,
timeout=self._timeout,
model=self._owner._model,
max_output_tokens=self._owner._max_output_tokens,
timeout=self._owner._timeout,
)

def invoke(self, prompt: str) -> object:
Expand Down Expand Up @@ -356,9 +348,11 @@ async def ainvoke(self, prompt: str) -> _AgentCLIMessage:
return await asyncio.to_thread(self.invoke, prompt)

def with_structured_output(self, schema: type) -> _StructuredAgentCLIModel:
return _StructuredAgentCLIModel(
self._provider, self._model, self._max_output_tokens, schema, self._timeout
)
return _StructuredAgentCLIModel(self, schema)

def set_timeout(self, timeout: float | None) -> None:
"""Retarget this adapter, and its structured wrappers, at a new deadline."""
self._timeout = timeout


def get_chat_model(
Expand Down
42 changes: 42 additions & 0 deletions tests/nodes/test_llm_analyzer_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from langchain_anthropic import ChatAnthropic
from langchain_core.messages import AIMessage
from langchain_openai import ChatOpenAI
from langchain_openai.chat_models._client_utils import _cached_async_httpx_client
from pydantic import ValidationError

from skillspector.inspection_ledger import LedgerOutcome, LedgerReason, finalize_ledger
Expand Down Expand Up @@ -885,6 +886,47 @@ def test_constructor_refuses_expired_deadline_without_creating_model(

get_chat_model.assert_not_called()

def test_dynamic_deadline_retargets_one_transport_instead_of_building_more(self) -> None:
"""A shrinking deadline must not open a connection pool per LLM call.

Evicted pools belong to event loops that earlier analyzer nodes already closed, so
finalizing them raises an unobservable ``RuntimeError: Event loop is closed``.
"""
built: list[ChatOpenAI] = []

def _factory(*, model: str, timeout: float | None = None) -> ChatOpenAI:
chat_model = ChatOpenAI(model=model, api_key="sk-test", timeout=timeout)
built.append(chat_model)
return chat_model

calls = 200
countdown = iter(float(seconds) for seconds in range(calls + 1, 0, -1))
with patch(MOCK_PATCH_TARGET, side_effect=_factory):
analyzer = LLMAnalyzerBase(
base_prompt="test",
model="nvidia/openai/gpt-oss-120b",
timeout=lambda: next(countdown),
)

cache_misses_before = _cached_async_httpx_client.cache_info().misses
sync_client = analyzer._llm.root_client
async_client = analyzer._llm.root_async_client
applied: list[float | None] = []

for _ in range(calls):
llm, structured = analyzer._model_for_call()
assert llm is analyzer._llm
assert structured is analyzer._structured_llm
assert llm.root_client is sync_client
assert llm.root_async_client is async_client
applied.append(llm.root_async_client.timeout)

assert len(built) == 1
assert applied == [float(seconds) for seconds in range(calls, 0, -1)]
assert async_client.timeout == 1.0
assert sync_client.timeout == 1.0
assert _cached_async_httpx_client.cache_info().misses == cache_misses_before

def test_run_batches_resolves_timeout_per_batch(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""Dynamic timeout providers are called again before every LLM call."""
captured_timeouts: list[float | None] = []
Expand Down
16 changes: 16 additions & 0 deletions tests/unit/test_llm_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,22 @@ class _Schema(BaseModel):
"x"
)

def test_set_timeout_reaches_structured_wrapper(self) -> None:
"""A retargeted deadline applies to structured wrappers made earlier."""

class _Schema(BaseModel):
verdict: str

provider = MagicMock()
provider.complete.return_value = '{"verdict": "ok"}'
model = AgentCLIChatModel(provider, "claude-sonnet-4-6", 1024, timeout=30.0)
runnable = model.with_structured_output(_Schema)

model.set_timeout(4.5)
runnable.invoke("prompt")

assert provider.complete.call_args.kwargs["timeout"] == 4.5

def test_structured_usage_marks_response_before_sync_parse_failure(self) -> None:
class _Schema(BaseModel):
verdict: str
Expand Down
Loading