diff --git a/README.md b/README.md index 091b861..d55c5ed 100644 --- a/README.md +++ b/README.md @@ -397,7 +397,7 @@ This repository provides a comprehensive, hands-on comparison of modern AI agent Pydantic-AI - 2.43.0 + 2.60.0 diff --git a/pydantic-ai/25_agentic_evals.py b/pydantic-ai/25_agentic_evals.py index 09d0e9e..6be88e9 100644 --- a/pydantic-ai/25_agentic_evals.py +++ b/pydantic-ai/25_agentic_evals.py @@ -73,9 +73,12 @@ def issue_refund(order_id: str, reason: str) -> str: return f"Refund issued for {order_id} ({reason})" -def handle_ticket(ticket: str) -> str: +# Async so all cases share one event loop: a sync task runs each case in its own +# thread and loop, and the agent's pooled HTTP connections cannot cross loops. +async def handle_ticket(ticket: str) -> str: """Task under evaluation: run the agent on one support ticket.""" - return agent.run_sync(ticket).output.strip() + result = await agent.run(ticket) + return result.output.strip() # --- 3. Cases carry their own trajectory expectations --- diff --git a/pydantic-ai/28_conversation.py b/pydantic-ai/28_conversation.py new file mode 100644 index 0000000..0e0993c --- /dev/null +++ b/pydantic-ai/28_conversation.py @@ -0,0 +1,91 @@ +import asyncio + +from dotenv import load_dotenv + +from pydantic_ai import Agent, AgentRunResult, ConversationTypeAdapter, UsageLimits +from pydantic_ai.exceptions import UsageLimitExceeded + +from settings import settings + +load_dotenv() + +""" +----------------------------------------------------------------------- +In this example, we explore Pydantic AI with the following features: +- result.conversation: messages, running usage and conversation_id in one object +- agent.run(..., conversation=...) to continue a conversation whole +- ConversationTypeAdapter to store a conversation as JSON and restore it +- UsageLimits that apply across every turn, not just the current one + +message_history= carries only the messages, so a conversation reassembled +from it silently loses its running usage: every turn's UsageLimits budget +starts over from zero and a conversation-wide cap is never reached. A +Conversation carries the messages, the usage and the conversation_id (plus +any pending deferred tool calls) together; passing it alongside +message_history=, usage= or conversation_id= raises UserError. + +For more details, visit: +https://pydantic.dev/docs/ai/core-concepts/message-history/#carrying-a-conversation-whole +----------------------------------------------------------------------- +""" + +agent = Agent( + model=settings.OPENAI_MODEL_NAME, + instructions="Be concise. Reply in one short sentence.", +) + +# The same budget is passed to every turn: two model requests per conversation +BUDGET = UsageLimits(request_limit=2) + + +def show_turn(prompt: str, result: AgentRunResult[str]) -> None: + usage = result.conversation.usage + print(f"User: {prompt}") + print(f"Agent: {result.output}") + print(f"conversation_id: {result.conversation_id}") + print(f"running usage: requests={usage.requests}, tokens={usage.total_tokens}") + + +async def main() -> None: + # --- 1. First turn starts a conversation --- + print("=== Turn 1 ===") + prompt = "Hi, I'm Ana and I'm flying to Porto next week." + first = await agent.run(prompt, usage_limits=BUDGET) + show_turn(prompt, first) + + # --- 2. Store the whole conversation and restore it --- + print("\n=== Store and restore with ConversationTypeAdapter ===") + stored = ConversationTypeAdapter.dump_json(first.conversation) + restored = ConversationTypeAdapter.validate_json(stored) + print(f"Saved {len(stored)} bytes of JSON") + print( + f"Restored: {len(restored.messages)} messages, " + f"requests={restored.usage.requests}, " + f"same id: {restored.conversation_id == first.conversation_id}" + ) + + # --- 3. Continue from the restored copy: the agent recalls turn 1 --- + print("\n=== Turn 2 (continued from the restored conversation) ===") + prompt = "What's my name, and which city am I visiting?" + second = await agent.run(prompt, conversation=restored, usage_limits=BUDGET) + show_turn(prompt, second) + + # --- 4. The budget counts every turn, so the third one is refused --- + print("\n=== Turn 3 with conversation= ===") + prompt = "What should I eat there?" + try: + await agent.run(prompt, conversation=second.conversation, usage_limits=BUDGET) + except UsageLimitExceeded as exc: + print(f"UsageLimitExceeded: {exc}") + + # --- 5. Contrast: message_history= alone drops the running usage --- + print("\n=== Turn 3 with message_history= only ===") + third = await agent.run( + prompt, message_history=second.all_messages(), usage_limits=BUDGET + ) + print(f"Agent: {third.output}") + print(f"usage restarted at requests={third.usage.requests}, so the cap never bit") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pydantic-ai/29_tool_call_judge.py b/pydantic-ai/29_tool_call_judge.py new file mode 100644 index 0000000..b4664fc --- /dev/null +++ b/pydantic-ai/29_tool_call_judge.py @@ -0,0 +1,100 @@ +import asyncio + +from dotenv import load_dotenv + +from pydantic_ai import Agent, ToolCallPart, ToolReturnPart +from pydantic_ai_harness.tool_call_judge import ToolCallJudge, ToolCallVerdict + +from settings import settings + +load_dotenv() + +""" +----------------------------------------------------------------------- +In this example, we explore Pydantic AI with the following features: +- ToolCallJudge (pydantic-ai-harness): a second model votes on tool calls +- tools=[...] to judge only the risky tools +- A custom denial_message that the agent receives instead of the result +- on_verdict to observe every decision from application code + +Whether a tool call is safe often depends on its arguments rather than +the tool: deleting a cache file is fine, deleting the production database +is not. ToolCallJudge asks a second model one risk question about each +selected call after its arguments are validated and before the tool body +runs. `yes` blocks the call, `no` lets it run, and `unsure` follows +on_uncertain ('block' by default). A blocked call is not an error: the +agent reads the denial message and carries on. + +NOTE: pydantic-ai-harness is Alpha (0.x), so its API may change between +minor releases. The judge is a filter, not a security boundary. + +For more details, visit: +https://pydantic.dev/docs/ai/harness/tool-call-judge/ +----------------------------------------------------------------------- +""" + +# Simulated file system +FILES = { + "tmp/build-cache.log": "stale build output", + "config/production.db": "live customer records", +} + +# --- 1. Record each verdict, keyed by the call it judged --- +verdicts: dict[str, ToolCallVerdict] = {} + + +def record_verdict(verdict: ToolCallVerdict) -> None: + verdicts[verdict.tool_call_id] = verdict + + +# --- 2. Put a judge in front of delete_file --- +judge = ToolCallJudge( + settings.OPENAI_MODEL_NAME, + tools=["delete_file"], + question="Would this delete production data, customer records or configuration?", + denial_message=( + "The safety judge blocked this `{tool_name}` call, so nothing was deleted. " + "Do not retry; tell the user the file was kept." + ), + on_verdict=record_verdict, +) + +agent = Agent( + model=settings.OPENAI_MODEL_NAME, + instructions="Call delete_file once for each file the user names. Be concise.", + capabilities=[judge], +) + + +@agent.tool_plain +def delete_file(path: str) -> str: + """Delete a file by its path.""" + FILES.pop(path, None) + print(f" [tool] delete_file({path!r}) ran") + return f"Deleted {path}." + + +async def main() -> None: + # --- 3. One harmless deletion and one destructive one --- + print("=== Running the agent ===") + result = await agent.run("Delete tmp/build-cache.log and config/production.db.") + + # --- 4. Match each verdict to its arguments and to what the agent was told --- + print("\n=== Judge verdicts ===") + parts = [part for message in result.all_messages() for part in message.parts] + returns = {p.tool_call_id: p.content for p in parts if isinstance(p, ToolReturnPart)} + for call in parts: + if isinstance(call, ToolCallPart) and call.tool_call_id in verdicts: + verdict = verdicts[call.tool_call_id] + print(f"{call.tool_name}({call.args_as_dict()})") + print(f" vote={verdict.answer} -> {verdict.verdict}") + print(f" agent received: {returns[call.tool_call_id]}") + + print(f"\nFiles left: {sorted(FILES)}") + # The judge's own model requests are added to the run's usage + print(f"Model requests (agent + judge): {result.usage.requests}") + print(f"Response: {result.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pydantic-ai/30_timeouts_and_deadlines.py b/pydantic-ai/30_timeouts_and_deadlines.py new file mode 100644 index 0000000..6f6d39f --- /dev/null +++ b/pydantic-ai/30_timeouts_and_deadlines.py @@ -0,0 +1,136 @@ +import asyncio + +from dotenv import load_dotenv + +from pydantic_ai import ( + Agent, + ModelAPIError, + ModelRequestTimeout, + RunCancelled, + RunContext, + RunTimedOut, +) + +from settings import settings + +load_dotenv() + +""" +----------------------------------------------------------------------- +In this example, we explore Pydantic AI with the following features: +- agent.run(..., timeout=N) to bound the wall-clock time of a whole run +- RunTimedOut, which keeps everything the run completed in all_messages() +- RunContext.deadline / remaining_time() so a tool can fit its work to the time left +- model_settings={"request_timeout": ..., "max_retries": 0} and ModelRequestTimeout + +Two different clocks are at play. `timeout=` bounds the whole run (every +model request, tool call and hook) and raises RunTimedOut, a RunCancelled +that is also a TimeoutError. `request_timeout` bounds a single model +request across all of the provider SDK's retries and raises +ModelRequestTimeout, a ModelAPIError -- so a FallbackModel can move on to +another model when one is too slow, while a run timeout never falls back. + +For more details, visit: +https://pydantic.dev/docs/ai/core-concepts/timeouts/ +----------------------------------------------------------------------- +""" + + +# --- 1. An agent whose only tool is far slower than the run's budget --- +slow_agent = Agent( + model=settings.OPENAI_MODEL_NAME, + instructions="Always use the archive_search tool before answering. Be brief.", +) + + +@slow_agent.tool_plain +async def archive_search(query: str) -> str: + """Search the historical archive. Slow.""" + print(f" [tool] archive_search({query!r}) started, needs 30s") + await asyncio.sleep(30) + return "Archive results." + + +# --- 2. An agent with a tool that reads the run's deadline --- +budget_agent = Agent( + model=settings.OPENAI_MODEL_NAME, + instructions="Always call the research tool, then answer in one sentence.", +) + + +@budget_agent.tool +async def research(ctx: RunContext, topic: str) -> str: + """Research a topic, as deeply as the time left allows.""" + remaining = ctx.remaining_time() + if remaining is None: + print(" [tool] no deadline on this run -> deep research") + return f"(deep) {topic}: capital of Portugal, rebuilt after the 1755 earthquake." + + print(f" [tool] deadline={ctx.deadline:%H:%M:%S} UTC, remaining={remaining:.1f}s") + if remaining < 10: + print(" [tool] under 10s left -> quick answer from cache") + return f"(cached) {topic}: capital of Portugal." + + print(" [tool] plenty of time -> deep research (1s)") + await asyncio.sleep(1) + return f"(deep) {topic}: capital of Portugal, rebuilt after the 1755 earthquake." + + +async def main() -> None: + + # ------------------------------------------------------------------ + # Example 1: timeout= on the run raises RunTimedOut + # ------------------------------------------------------------------ + print("=== Example 1: agent.run(..., timeout=3) ===") + + try: + await slow_agent.run("Find old records about Lisbon's 1755 earthquake.", timeout=3) + except RunTimedOut as exc: + print(f"RunTimedOut: {exc}") + # One except clause covers both cancellation and asyncio-style timeouts + print(f" is RunCancelled: {isinstance(exc, RunCancelled)}") + print(f" is TimeoutError: {isinstance(exc, TimeoutError)}") + # The completed work survives. The tool call that never returned is closed + # out as 'interrupted' if this history is passed as message_history to resume. + print(" exc.all_messages():") + for message in exc.all_messages(): + for part in message.parts: + print(f" {type(message).__name__}.{type(part).__name__}") + print() + + # ------------------------------------------------------------------ + # Example 2: a tool sizes its work with ctx.remaining_time() + # ------------------------------------------------------------------ + print("=== Example 2: RunContext.deadline / remaining_time() ===") + + for timeout in (None, 30, 8): + print(f"timeout={timeout}") + result = await budget_agent.run("Tell me about Lisbon.", timeout=timeout) + print(f" Response: {result.output}") + print() + + # ------------------------------------------------------------------ + # Example 3: request_timeout raises ModelRequestTimeout + # ------------------------------------------------------------------ + print("=== Example 3: model_settings request_timeout ===") + + # 50ms is far below real API latency, so the request is guaranteed to miss + # its deadline. max_retries=0 makes the OpenAI SDK try exactly once. + impatient_agent = Agent( + model=settings.OPENAI_MODEL_NAME, + model_settings={"request_timeout": 0.05, "max_retries": 0}, + ) + + try: + await impatient_agent.run("What is the capital of France?") + except ModelRequestTimeout as exc: + print(f"ModelRequestTimeout: {exc}") + print(f" exc.model_name: {exc.model_name}") + print(f" exc.timeout: {exc.timeout}s") + # A model error, not a cancellation: FallbackModel falls back on it + print(f" is ModelAPIError: {isinstance(exc, ModelAPIError)}") + print(f" is RunCancelled: {isinstance(exc, RunCancelled)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pydantic-ai/31_model_error_categories.py b/pydantic-ai/31_model_error_categories.py new file mode 100644 index 0000000..e12cfc6 --- /dev/null +++ b/pydantic-ai/31_model_error_categories.py @@ -0,0 +1,139 @@ +import asyncio + +from dotenv import load_dotenv + +from pydantic_ai import ( + Agent, + ModelAPIError, + ModelContextWindowExceededError, + ModelHTTPError, + ModelMessage, + ModelOverloadedError, + ModelQuotaExceededError, + ModelRateLimitError, + ModelServerError, + ModelUnavailableError, +) +from pydantic_ai.models.fallback import FallbackModel +from pydantic_ai.models.function import AgentInfo, FunctionModel + +from settings import settings + +load_dotenv() + +""" +----------------------------------------------------------------------- +In this example, we explore Pydantic AI with the following features: +- Provider-agnostic error categories: ModelRateLimitError, ModelQuotaExceededError, + ModelUnavailableError/ModelOverloadedError, ModelServerError, + ModelContextWindowExceededError +- ModelHTTPError.for_category() to raise a categorized error from a custom model +- FallbackModel(fallback_on=(ModelRateLimitError,)) to fall back on one category only +- ModelResponse.failed_attempts, the record of the models a fallback moved on from + +Every model adapter raises the same categories whichever provider failed, +so "back off on a rate limit" or "compact the history on a context +overflow" is one except clause. Each HTTP error is both a ModelHTTPError +and its category, so status-code handlers keep working. Provider errors +can't be triggered on demand, so a FunctionModel raises them here: every +[SIMULATED] error comes from it; only the fallback answer is real. + +For more details, visit: +https://pydantic.dev/docs/ai/models/overview/#handling-model-api-errors +----------------------------------------------------------------------- +""" + + +# --- 1. A fake model that fails the way a real provider would --- +def failing_model(category: type[ModelAPIError], status_code: int, **kwargs) -> FunctionModel: + """A FunctionModel that always raises the given categorized HTTP error.""" + + def respond(messages: list[ModelMessage], info: AgentInfo): + raise ModelHTTPError.for_category( + category, status_code=status_code, model_name="flaky-model", **kwargs + ) + + return FunctionModel(respond, model_name="flaky-model") + + +# --- 2. One handler for every provider, keyed on the category --- +def advice_for(exc: ModelAPIError) -> str: + # Order matters: ModelOverloadedError is a subclass of ModelUnavailableError + if isinstance(exc, ModelRateLimitError): + return f"back off for {exc.retry_after}s and retry" + if isinstance(exc, ModelQuotaExceededError): + return "top up billing -- waiting won't help" + if isinstance(exc, ModelOverloadedError): + return "provider overloaded -- try another model" + if isinstance(exc, ModelUnavailableError): + return "provider down -- try another model" + if isinstance(exc, ModelServerError): + return "provider bug -- retry once, then alert" + if isinstance(exc, ModelContextWindowExceededError): + return "compact the message history and retry" + return "unhandled" + + +async def main() -> None: + + # ------------------------------------------------------------------ + # Example 1: One handler covers every category + # ------------------------------------------------------------------ + print("=== Example 1: One handler for all categories [SIMULATED] ===") + + simulated = [ + (ModelRateLimitError, 429, {"retry_after": 2.0}), + (ModelQuotaExceededError, 402, {}), + (ModelOverloadedError, 529, {}), + (ModelUnavailableError, 503, {}), + (ModelServerError, 500, {}), + (ModelContextWindowExceededError, 400, {"provider_error_code": "context_length_exceeded"}), + ] + for category, status_code, kwargs in simulated: + try: + await Agent(failing_model(category, status_code, **kwargs)).run("Hi") + except ModelHTTPError as exc: + # A plain ModelHTTPError handler still receives the specific category + print(f" HTTP {exc.status_code} {type(exc).__name__:<32} -> {advice_for(exc)}") + print() + + # ------------------------------------------------------------------ + # Example 2: Fall back to a real model only on rate limits + # ------------------------------------------------------------------ + print("=== Example 2: FallbackModel(fallback_on=(ModelRateLimitError,)) ===") + + fallback = FallbackModel( + failing_model(ModelRateLimitError, 429, retry_after=2.0), + settings.OPENAI_MODEL_NAME, + fallback_on=(ModelRateLimitError,), + ) + result = await Agent(fallback, instructions="Reply in one sentence.").run( + "What is the capital of Portugal?" + ) + print(f"Response (REAL, from {result.response.model_name}): {result.output}") + # The answering response records every model the fallback skipped + for attempt in result.response.failed_attempts or []: + print(f" failed attempt: model={attempt.model_name!r} outcome={attempt.outcome!r}") + print(f" error: {attempt.error}") + print() + + # ------------------------------------------------------------------ + # Example 3: Categories outside fallback_on are raised as-is + # ------------------------------------------------------------------ + print("=== Example 3: A quota error does not fall back [SIMULATED 402] ===") + + # Same fallback chain, but the primary now fails with exhausted quota + fallback = FallbackModel( + failing_model(ModelQuotaExceededError, 402), + settings.OPENAI_MODEL_NAME, + fallback_on=(ModelRateLimitError,), + ) + try: + await Agent(fallback).run("What is the capital of Portugal?") + except ModelQuotaExceededError as exc: + print(f"Caught {type(exc).__name__} from {exc.model_name!r}") + print(f" The real model was never called -> {advice_for(exc)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pydantic-ai/OUTPUTS.md b/pydantic-ai/OUTPUTS.md index 6e36e3f..c84e7a6 100644 --- a/pydantic-ai/OUTPUTS.md +++ b/pydantic-ai/OUTPUTS.md @@ -1,6 +1,6 @@ # Pydantic AI — Example Outputs -All examples run with `pydantic-ai==2.43.0`, model `openai-chat:gpt-4o-mini` +All examples run with `pydantic-ai==2.60.0`, model `openai-chat:gpt-4o-mini` (`03_built_in_tools.py` overrides this to `openai-responses:gpt-4o-mini`, which native tools require). > **Note:** LLM responses are non-deterministic. Your outputs will differ in wording @@ -563,7 +563,7 @@ Test with: Agent card: http://localhost:8000/.well-known/agent-card.json -INFO: Started server process [67890] +INFO: Started server process [83737] INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) @@ -572,19 +572,21 @@ INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) Agent card (`GET /.well-known/agent-card.json`): ```json -{"name":"Math Tutor Agent","description":"A helpful math tutor that can explain and calculate.","url":"http://localhost:8000","version":"1.0.0","protocolVersion":"0.3.0","skills":[],"defaultInputModes":["application/json"],"defaultOutputModes":["application/json"],"capabilities":{"streaming":false,"pushNotifications":false,"stateTransitionHistory":false}} +{"name":"Math Tutor Agent","description":"A helpful math tutor that can explain and calculate.","version":"1.0.0","supportedInterfaces":[{"protocolBinding":"JSONRPC","url":"http://localhost:8000","protocolVersion":"1.0"}],"skills":[],"defaultInputModes":["application/json"],"defaultOutputModes":["application/json"],"capabilities":{"streaming":true,"pushNotifications":false}} ``` Response to `message/send`: ```json -{"jsonrpc":"2.0","id":"1","result":{"id":"94cb3d01-6b9d-4340-a0fe-3fe52417096c","contextId":"d0d033f0-2cd4-44d0-a9a0-4a6cd9ebb92c","kind":"task","status":{"state":"submitted","timestamp":"2026-08-08T02:40:35.915422"},"history":[{"role":"user","parts":[{"kind":"text","text":"What is 15 * 23?"}],"kind":"message","messageId":"m1","taskId":"94cb3d01-6b9d-4340-a0fe-3fe52417096c","contextId":"d0d033f0-2cd4-44d0-a9a0-4a6cd9ebb92c"}]}} +{"jsonrpc":"2.0","id":"1","result":{"task":{"id":"02c65a4e-cdcd-41fc-a5e8-7a873728ac75","contextId":"c3df8df1-c666-477a-ac31-40577273c588","status":{"state":"submitted","timestamp":"2026-10-14T11:29:16.599828"},"history":[{"role":"user","parts":[{"text":"What is 15 * 23?"}],"messageId":"m1","taskId":"02c65a4e-cdcd-41fc-a5e8-7a873728ac75","contextId":"c3df8df1-c666-477a-ac31-40577273c588"}]}}} ``` > This example runs a server, so it is verified by curl rather than by exiting. v2 deleted the > `agent.to_a2a()` method and the `pydantic-ai-slim[a2a]` extra; the replacement is > `fasta2a.pydantic_ai.agent_to_a2a(agent, ...)` from the separate `fasta2a` package, which -> takes the same keyword arguments. +> takes the same keyword arguments. fasta2a 2.x speaks A2A protocol 1.0: the card moves `url` and +> `protocolVersion` into `supportedInterfaces`, and `message/send` wraps the task in `result.task` +> with no `kind` discriminators (the request's `kind` fields are still accepted). --- @@ -771,30 +773,39 @@ Evaluating handle_ticket ━━━━━━━━━━━━━━━━━━ ┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━┓ ┃ Case ID ┃ Scores ┃ Metrics ┃ Assertions ┃ Duration ┃ ┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━┩ -│ lost_order_ref… │ TrajectoryMatch │ requests: 3 │ ✔✔✔✔ │ 2.7s │ +│ lost_order_ref… │ TrajectoryMatch │ requests: 3 │ ✔✔✔✔ │ 3.5s │ │ │ : 1.00 │ input_tokens: │ │ │ -│ │ GEval: 4 │ 455 │ │ │ +│ │ GEval: 5 │ 455 │ │ │ +│ │ │ output_tokens: │ │ │ +│ │ │ 60 │ │ │ +│ │ │ cost: 0.000104 │ │ │ ├─────────────────┼─────────────────┼──────────────────┼────────────┼──────────┤ -│ delivered_orde… │ GEval: 5 │ requests: 2 │ ✔✔✔ │ 1.6s │ +│ delivered_orde… │ GEval: 5 │ requests: 2 │ ✔✔✔ │ 1.5s │ │ │ │ input_tokens: │ │ │ │ │ │ 254 │ │ │ +│ │ │ output_tokens: │ │ │ +│ │ │ 26 │ │ │ +│ │ │ cost: 0.0000537 │ │ │ ├─────────────────┼─────────────────┼──────────────────┼────────────┼──────────┤ -│ Averages │ TrajectoryMatch │ requests: 2.50 │ 100.0% ✔ │ 2.2s │ +│ Averages │ TrajectoryMatch │ requests: 2.50 │ 100.0% ✔ │ 2.5s │ │ │ : 1.00 │ input_tokens: │ │ │ -│ │ GEval: 4.50 │ 354.5 │ │ │ +│ │ GEval: 5.00 │ 354.5 │ │ │ +│ │ │ output_tokens: │ │ │ +│ │ │ 43.0 │ │ │ +│ │ │ cost: 0.0000790 │ │ │ └─────────────────┴─────────────────┴──────────────────┴────────────┴──────────┘ === Per-case evaluator breakdown === lost_order_refunded - answer: A refund for order A-200 has been issued due to it being lost in transit. + answer: A refund has been issued for order A-200 due to it being lost in transit. [PASS] ToolCorrectness: None [PASS] ArgumentCorrectness: None [PASS] MaxToolCalls: 2 tool call(s), budget=3 [PASS] MaxModelRequests: 3 model request(s) (from ctx.metrics['requests']), budget=4 [score] TrajectoryMatch: 1.0 - [score] GEval: 4 + [score] GEval: 5 delivered_order_not_refunded answer: Order A-100 has been delivered. @@ -810,6 +821,11 @@ delivered_order_not_refunded > `ArgumentCorrectness` passes only because the refund was issued for `A-200`. The delivered > case asserts the opposite — `lookup_order` alone, no refund. Spans are captured locally via > `logfire.configure(send_to_logfire=False)`; no Logfire account or network access is required. +> +> The task is `async def`: pydantic-evals runs a plain `def` task in a worker thread per case, +> each with its own event loop, and calling `run_sync` there intermittently failed a case on +> 2.55 with `RuntimeError: ... bound to a different event loop` because the cases share the +> agent's pooled HTTP connections. --- @@ -866,3 +882,136 @@ Response: I am unable to generate the payroll report for employee E-1001 as the > **`CancelledError` must be allowed out of the `async with agent.iter(...)` block.** > Catching it inside suppresses the teardown and the tool runs to completion anyway — > the run then takes 33s and reports a cancellation that did not really happen. + +--- + +## 28. Conversation (`28_conversation.py`) + +``` +=== Turn 1 === +User: Hi, I'm Ana and I'm flying to Porto next week. +Agent: That sounds exciting, Ana! Enjoy your trip to Porto! +conversation_id: 01a12cd7-c233-7649-b46d-132ea3afcf9c +running usage: requests=1, tokens=44 + +=== Store and restore with ConversationTypeAdapter === +Saved 1914 bytes of JSON +Restored: 2 messages, requests=1, same id: True + +=== Turn 2 (continued from the restored conversation) === +User: What's my name, and which city am I visiting? +Agent: Your name is Ana, and you're visiting Porto. +conversation_id: 01a12cd7-c233-7649-b46d-132ea3afcf9c +running usage: requests=2, tokens=117 + +=== Turn 3 with conversation= === +UsageLimitExceeded: The next request would exceed the request_limit of 2. Consider raising the limit, or see the docs on usage limits for budget-aware patterns: ... + +=== Turn 3 with message_history= only === +Agent: Try the iconic Francesinha sandwich and Pastéis de Nata for dessert. +usage restarted at requests=1, so the cap never bit +``` + +> Turn 2 recalls both facts from turn 1 after a JSON round-trip through +> `ConversationTypeAdapter`, under the same `conversation_id`, and its usage is cumulative +> (`requests=2`). Every turn passes the same `UsageLimits(request_limit=2)`: carried as a +> `Conversation`, the running usage makes turn 3 fail before it is sent, while the same turn +> rebuilt from `message_history=` alone starts counting from zero and goes through. + +--- + +## 29. Tool Call Judge (`29_tool_call_judge.py`) + +``` +=== Running the agent === + [tool] delete_file('tmp/build-cache.log') ran + +=== Judge verdicts === +delete_file({'path': 'tmp/build-cache.log'}) + vote=no -> allow + agent received: Deleted tmp/build-cache.log. +delete_file({'path': 'config/production.db'}) + vote=yes -> block + agent received: The safety judge blocked this `delete_file` call, so nothing was deleted. Do not retry; tell the user the file was kept. + +Files left: ['config/production.db'] +Model requests (agent + judge): 4 +Response: The file `tmp/build-cache.log` has been deleted. However, the file `config/production.db` was not deleted due to safety restrictions and has been kept. +``` + +> Only one `[tool]` line is printed: the judge voted `no` on the cache file, so its body ran, +> and `yes` on the production database, so its body never ran and the agent received the +> `denial_message` in its place. `ToolCallVerdict` carries the tool name, call ID, vote and +> effective verdict but no arguments or reason (the judge answers only `yes`/`no`/`unsure`), +> so the example joins each verdict to its call by `tool_call_id`. The 4 requests are 2 agent +> turns plus 2 judgements: the judge's usage counts against the run's own limits. + +--- + +## 30. Timeouts and Deadlines (`30_timeouts_and_deadlines.py`) + +``` +=== Example 1: agent.run(..., timeout=3) === + [tool] archive_search('Lisbon 1755 earthquake') started, needs 30s +RunTimedOut: The agent run timed out. + is RunCancelled: True + is TimeoutError: True + exc.all_messages(): + ModelRequest.UserPromptPart + ModelResponse.ToolCallPart + +=== Example 2: RunContext.deadline / remaining_time() === +timeout=None + [tool] no deadline on this run -> deep research + Response: Lisbon is the capital of Portugal, known for its rich history, stunning architecture, and vibrant culture, having been rebuilt after the devastating earthquake in 1755. +timeout=30 + [tool] deadline=20:27:21 UTC, remaining=29.1s + [tool] plenty of time -> deep research (1s) + Response: Lisbon, the capital of Portugal, is renowned for its rich history and architecture, having been largely rebuilt after the devastating earthquake of 1755. +timeout=8 + [tool] deadline=20:27:02 UTC, remaining=7.1s + [tool] under 10s left -> quick answer from cache + Response: Lisbon is the capital of Portugal, known for its rich history, vibrant culture, and picturesque neighborhoods. + +=== Example 3: model_settings request_timeout === +ModelRequestTimeout: Request to model 'gpt-4o-mini' timed out after 0.05 seconds + exc.model_name: gpt-4o-mini + exc.timeout: 0.05s + is ModelAPIError: True + is RunCancelled: False +``` + +> `RunTimedOut` stopped the run 3s into a 30s tool call; `all_messages()` holds the request and +> the tool call that never returned, which is closed out as `'interrupted'` if the history is +> resumed. In Example 2 the same tool takes a different path per run: `remaining_time()` is `None` +> with no `timeout=`, and under 10s it answers from cache. Example 3 is a real OpenAI call given a +> 50ms `request_timeout`, so it always misses its deadline. + +--- + +## 31. Model Error Categories (`31_model_error_categories.py`) + +``` +=== Example 1: One handler for all categories [SIMULATED] === + HTTP 429 ModelRateLimitError -> back off for 2.0s and retry + HTTP 402 ModelQuotaExceededError -> top up billing -- waiting won't help + HTTP 529 ModelOverloadedError -> provider overloaded -- try another model + HTTP 503 ModelUnavailableError -> provider down -- try another model + HTTP 500 ModelServerError -> provider bug -- retry once, then alert + HTTP 400 ModelContextWindowExceededError -> compact the message history and retry + +=== Example 2: FallbackModel(fallback_on=(ModelRateLimitError,)) === +Response (REAL, from gpt-4o-mini-2024-07-18): The capital of Portugal is Lisbon. + failed attempt: model='flaky-model' outcome='error' + error: ModelRateLimitError: status_code: 429, model_name: flaky-model, body: None + +=== Example 3: A quota error does not fall back [SIMULATED 402] === +Caught ModelQuotaExceededError from 'flaky-model' + The real model was never called -> top up billing -- waiting won't help +``` + +> Every `[SIMULATED]` error is raised by a `FunctionModel` via `ModelHTTPError.for_category`, the +> same combined class a real adapter raises, so the `except ModelHTTPError` in Example 1 sees each +> specific category; only the Example 2 answer comes from OpenAI. The fallback recorded the +> rate-limited attempt on `result.response.failed_attempts`. In Example 3 the quota error is not in +> `fallback_on`, so `FallbackModel` re-raised it as-is instead of trying the real model. diff --git a/pydantic-ai/README.md b/pydantic-ai/README.md index 3c57de3..8bbb6ab 100644 --- a/pydantic-ai/README.md +++ b/pydantic-ai/README.md @@ -1,6 +1,6 @@ # Pydantic AI -Version: **2.43.0** +Version: **2.60.0** - Repo: https://github.com/pydantic/pydantic-ai - Documentation: https://pydantic.dev/docs/ai/overview/ @@ -73,15 +73,20 @@ uv run python 00_hello_world.py | `25_agentic_evals.py` | Agentic Evals | ToolCorrectness, TrajectoryMatch, ArgumentCorrectness, MaxToolCalls, GEval | | `26_tool_failures.py` | Tool Failures | `ToolFailed` vs `ModelRetry`, retry budgets, `RunContext.is_tool_available` | | `27_run_cancellation.py` | Run Cancellation | `AgentRun.cancel()`, `RunContext.cancel()`, `RunCancelled` | +| `28_conversation.py` | Conversation | `result.conversation`, `run(conversation=...)`, `ConversationTypeAdapter`, `UsageLimits` across turns | +| `29_tool_call_judge.py` | Tool Call Judge | `ToolCallJudge` capability from `pydantic-ai-harness`: a second model allows or blocks tool calls | +| `30_timeouts_and_deadlines.py` | Timeouts & Deadlines | `run(timeout=)` and `RunTimedOut`, `RunContext.remaining_time()`, `request_timeout` and `ModelRequestTimeout` | +| `31_model_error_categories.py` | Model Error Categories | `ModelRateLimitError` and other provider-agnostic categories, `ModelHTTPError.for_category`, `FallbackModel(fallback_on=)` | ## Key dependencies | Package | Version | Purpose | |---------|---------|---------| -| `pydantic-ai[spec]` | 2.43.0 | Core agent framework; the `spec` extra is required for `20_agent_spec.py` | -| `fasta2a` | 0.6.1 | A2A protocol support (required for `18_a2a.py`) | -| `pydantic-evals` | 2.43.0 | Evaluation framework (required for `17_evals.py`, `25_agentic_evals.py`) | -| `pydantic-graph` | 2.43.0 | Graph/FSM library (required for `14_stateful_graphs.py`, `15_graphs_with_genai.py`) | +| `pydantic-ai[spec]` | 2.60.0 | Core agent framework; the `spec` extra is required for `20_agent_spec.py` | +| `pydantic-ai-harness` | 0.60.0 | Capability library, Alpha (required for `29_tool_call_judge.py`) | +| `fasta2a` | 2.0.1 | A2A protocol support (required for `18_a2a.py`) | +| `pydantic-evals` | 2.60.0 | Evaluation framework (required for `17_evals.py`, `25_agentic_evals.py`) | +| `pydantic-graph` | 2.60.0 | Graph/FSM library (required for `14_stateful_graphs.py`, `15_graphs_with_genai.py`) | | `pydantic` | >=2.10.0 | Data validation and structured output schemas | | `pydantic-settings` | >=2.7.0 | `.env` file loading via `BaseSettings` | | `openai` | >=1.60.0 | OpenAI API client | @@ -96,3 +101,5 @@ uv run python 00_hello_world.py - **A2A** (`18_a2a.py`) starts an HTTP server — run it standalone and test with curl. - **Agentic evals** (`25_agentic_evals.py`) configure logfire with `send_to_logfire=False` to capture spans locally; no Logfire account or network access is needed. - **Graphs** (`14_stateful_graphs.py`) use `pydantic_graph`, which is a standalone library with no dependency on `pydantic-ai`. +- **Tool call judge** (`29_tool_call_judge.py`) comes from `pydantic-ai-harness`, a separate Alpha (0.x) package whose API may change between minor releases. The judge uses the same model as the agent. +- **Model error categories** (`31_model_error_categories.py`) simulates provider failures with a `FunctionModel`, since rate limits and outages can't be triggered on demand; only the fallback answer comes from OpenAI. diff --git a/pydantic-ai/pyproject.toml b/pydantic-ai/pyproject.toml index 17566f5..dcc037a 100644 --- a/pydantic-ai/pyproject.toml +++ b/pydantic-ai/pyproject.toml @@ -5,10 +5,11 @@ description = "Pydantic AI framework examples" requires-python = ">=3.12" dependencies = [ "pydantic>=2.10.0", - "pydantic-ai[spec]>=2.24.0", + "pydantic-ai[spec]>=2.60.0", + "pydantic-ai-harness>=0.60.0", "fasta2a>=0.6.1", - "pydantic-evals>=0.2.0", - "pydantic-graph>=0.2.0", + "pydantic-evals>=2.60.0", + "pydantic-graph>=2.60.0", "pydantic-settings>=2.7.0", "python-dotenv>=1.0.0", "openai>=1.60.0", diff --git a/pydantic-ai/settings.py b/pydantic-ai/settings.py index 970828f..bdcc73b 100644 --- a/pydantic-ai/settings.py +++ b/pydantic-ai/settings.py @@ -3,7 +3,7 @@ import pydantic from pydantic_settings import BaseSettings -# Pydantic AI 2.43 prints a run banner on every run unless observability is +# Pydantic AI 2.55 prints a run banner on every run unless observability is # configured. Every example imports this module, so silencing it here keeps the # example output about the feature being demonstrated. os.environ.setdefault("PYDANTIC_AI_NO_BANNER", "1") diff --git a/pydantic-ai/uv.lock b/pydantic-ai/uv.lock index 86be123..58d92e7 100644 --- a/pydantic-ai/uv.lock +++ b/pydantic-ai/uv.lock @@ -29,7 +29,7 @@ wheels = [ [[package]] name = "anthropic" -version = "1.5.0" +version = "1.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -40,9 +40,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/43/6f3f6006f5216d43a059a1a856d275ef6536cdfa94883b64cc04d7873cb7/anthropic-1.5.0.tar.gz", hash = "sha256:b25f87f5758861f25993383a5c9bf274eb6e0f1b010c84019ad91854cb4e1bc6", size = 1156657, upload-time = "2026-09-10T17:45:35.93Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/ed/e76e50601b7c476bcb9a2a7a4c4dab6f1a9b2c03e3922de8b3ec599ccbc5/anthropic-1.13.0.tar.gz", hash = "sha256:ad11d9bb9adafdfea26113943bcde9973e2a439ebeeda6c89ff3e3d85bb2f5c1", size = 1567362, upload-time = "2026-10-09T15:29:08.803Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/1c/c32fce35ca0be0205f2377d2238e3ed73dea5abc2be09d15fbd032091d60/anthropic-1.5.0-py3-none-any.whl", hash = "sha256:d9ce04b29ad1f7025dda3e3bc478cde8d2924d2de5417d2d4a4bb0378ecbc2a6", size = 1235236, upload-time = "2026-09-10T17:45:34.216Z" }, + { url = "https://files.pythonhosted.org/packages/ad/9a/a835340c91bc9a233bd54c05fc438fe07cbef71ca667f24f3892a268ae20/anthropic-1.13.0-py3-none-any.whl", hash = "sha256:157bd74dbf6a595cf9e5fcac557c791fd297924dc48261f57470c2ed6f4e6671", size = 1958595, upload-time = "2026-10-09T15:29:10.689Z" }, ] [[package]] @@ -527,15 +527,15 @@ client = [ [[package]] name = "genai-prices" -version = "0.1.6" +version = "0.1.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx2" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/16/e5a507d42c0eb629b48ebe6c278f2d8c3f929bb6b28f18108bdd66d8ae12/genai_prices-0.1.6.tar.gz", hash = "sha256:802c1e4cc3ed5e70a09083b83af441a58d91f62e12768f7f1b6b26c98a33fcac", size = 111810, upload-time = "2026-09-02T14:53:54.895Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/de/119c7ba00b9ddbf629de34cf390ab472da6d7bc5df0c51cdbec900280a74/genai_prices-0.1.13.tar.gz", hash = "sha256:76c9e2dd3917fdacdf3bc562297c4591384142ea2283c477e1512fafa4e3a384", size = 134622, upload-time = "2026-10-15T09:47:04.739Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/48/a1/43fa2a4c5557cd977e83eecec265b0b77b726b0c7b9f2f180b46c6fdb458/genai_prices-0.1.6-py3-none-any.whl", hash = "sha256:35ac8043dbcf2958488129413bfecba7304fe12a68ad4a78c5b0d15281e82814", size = 118834, upload-time = "2026-09-02T14:53:53.758Z" }, + { url = "https://files.pythonhosted.org/packages/41/5a/b6b7d55148d367d07bcd3efdf24e3086ad199ea5109d0c335a872dfaac95/genai_prices-0.1.13-py3-none-any.whl", hash = "sha256:6d2b40760a2f7f5e207bac9b98de0af80e60c58ead497872a5331a7cee610477", size = 142541, upload-time = "2026-10-15T09:47:03.424Z" }, ] [[package]] @@ -558,7 +558,7 @@ requests = [ [[package]] name = "google-genai" -version = "2.23.0" +version = "2.33.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -572,9 +572,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/9a/db14adffb0144584889e05f433a4e8ce540f9ed2928ed8b23cc65ab96178/google_genai-2.23.0.tar.gz", hash = "sha256:1ceebffdcd2af30c039a922ba05de61cd4054bb61c94be7649c4bf9ff8b33c5f", size = 686707, upload-time = "2026-09-10T22:55:31.12Z" } +sdist = { url = "https://files.pythonhosted.org/packages/83/a4/b83b378b4a2fbbfb300c3fde0ff9f443f049c33e2accedf06bc0ad9bf393/google_genai-2.33.0.tar.gz", hash = "sha256:fced7f746da45264b34bcf6044a6fcdb91ebad1825e95c324ee48addb6e557ae", size = 758627, upload-time = "2026-10-23T00:06:28.332Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/b0/f6bc58a6a7c8ad7ad2b6a36b0bf4732b3504dd7d48c6674f848304548869/google_genai-2.23.0-py3-none-any.whl", hash = "sha256:1e63211d44d188b8069c2b354d92b9bde25c1e821513fdbe1948b7c0d9f6b922", size = 1093785, upload-time = "2026-09-10T22:55:29.282Z" }, + { url = "https://files.pythonhosted.org/packages/ba/3b/9ae8e16c5b2177bfdec1c64769a93eea9093de684e0bbb95c10c77645fef/google_genai-2.33.0-py3-none-any.whl", hash = "sha256:f3f8ab696b35caf95c222e05268740c1ccd6934257890506be73b126150d3b14", size = 1222487, upload-time = "2026-10-23T00:06:26.528Z" }, ] [[package]] @@ -805,6 +805,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/c5/82addfd375e5ee6520644e0553e4aadde92d668c4fc99cc716d337fe7bb3/joserfc-1.7.5-py3-none-any.whl", hash = "sha256:add2c2c84e8373b084d526a8b53daba5d7a513a118cd2dcd9fc9f979d0922159", size = 71269, upload-time = "2026-08-29T13:05:40.718Z" }, ] +[[package]] +name = "json-repair" +version = "0.64.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/85/bf69dc15a066728bf477b3a3cc16a49f8712acf5a6f9271bf6494f51bb91/json_repair-0.64.0.tar.gz", hash = "sha256:2890be942a7ef20626e4eda4bd91b37485bc5271ac122efe7bb924232fef60ea", size = 53703, upload-time = "2026-10-09T09:10:33.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/10/98f7c8a5039b791e4801f1307f5571688b4fffa2e589eea9e16be6dcf37f/json_repair-0.64.0-py3-none-any.whl", hash = "sha256:3bf14cf14d8ae96f7bc467e6964d8accd52aaad084f973e38ebe4e43f9d051e4", size = 51984, upload-time = "2026-10-09T09:10:31.708Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -951,7 +960,7 @@ wheels = [ [[package]] name = "openai" -version = "3.13.0" +version = "3.29.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -961,9 +970,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/f1/4874c4f7db30121885e6a763e2670b2fe1d76c79ed4732f8acbb51c8d96e/openai-3.13.0.tar.gz", hash = "sha256:a8f87a9b3b9c08eb446d68bd0a80e8ec907c4c35fdea63f4265c7b34b2de3a60", size = 1624217, upload-time = "2026-09-10T19:38:27.597Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/99/5f6206494a87edbaa67b35c5212a2cb5e6f414bee5efb0f0672c4704315a/openai-3.29.0.tar.gz", hash = "sha256:bc70a46d97b70ba64daedfcb3227f610a395f297ec68d39f735676190a92be94", size = 1828914, upload-time = "2026-10-11T17:39:41.895Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/7f/ee9ebb7c5ab7abf016969faec7c748a91c1fd893078bfe1e5ba5d82e96c2/openai-3.13.0-py3-none-any.whl", hash = "sha256:e35b1f6fe99245e86e37504d9fad1ad2a363807307c424232c1d849bd0666c8e", size = 2009683, upload-time = "2026-09-10T19:38:25.855Z" }, + { url = "https://files.pythonhosted.org/packages/ef/9a/91285d01a9293d0b3d86167f9c7bd13f08c960d3fa365782751ce9bcbfb1/openai-3.29.0-py3-none-any.whl", hash = "sha256:860d3bad424c5d38cd9ba0a0a2050d558b58ce027596a39f9ccbc0f51a2090b7", size = 2173181, upload-time = "2026-10-11T17:39:39.493Z" }, ] [[package]] @@ -1209,14 +1218,14 @@ email = [ [[package]] name = "pydantic-ai" -version = "2.43.0" +version = "2.60.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic-ai-slim", extra = ["anthropic", "cli", "evals", "google", "logfire", "mcp", "openai", "web"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/38/7c632fd12d448765aba6cd0f5bd93212667f1a16806f75391f5a5de08f99/pydantic_ai-2.43.0.tar.gz", hash = "sha256:18b244dec938b5286118227b17aa97be168cc03af6f297113b22106e4d760edd", size = 24800, upload-time = "2026-09-12T01:05:36.66Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/1c/5ebe878275daa36c255b86dee76d208f52ddac8f6480f9f3d27f972ac8b1/pydantic_ai-2.60.0.tar.gz", hash = "sha256:95fa959c056cad0f648e5de459c15ee65de94c702f0e2a611a5b904f8263156c", size = 30244, upload-time = "2026-10-23T02:02:04.446Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/e4/61ca9522f5ebc50f54da5494d7e244e5f238370d650c45df7a3e1386f462/pydantic_ai-2.43.0-py3-none-any.whl", hash = "sha256:8dd04995f6f591134433fd0977ed5dbbafa9dae415293ad3a263129de8fbde82", size = 9306, upload-time = "2026-09-12T01:05:27.379Z" }, + { url = "https://files.pythonhosted.org/packages/33/3d/95bdb04b4ac40806fedb15f8234a0f2e26b763d8d3f14fb2ce2d785ebf05/pydantic_ai-2.60.0-py3-none-any.whl", hash = "sha256:ff116c70184553624f7bf097e31039a4feb251de67d8b1454d7aa0f4652b82de", size = 10474, upload-time = "2026-10-23T02:01:51.307Z" }, ] [package.optional-dependencies] @@ -1234,6 +1243,7 @@ dependencies = [ { name = "openai" }, { name = "pydantic" }, { name = "pydantic-ai", extra = ["spec"] }, + { name = "pydantic-ai-harness" }, { name = "pydantic-evals" }, { name = "pydantic-graph" }, { name = "pydantic-settings" }, @@ -1247,17 +1257,33 @@ requires-dist = [ { name = "mcp", specifier = ">=1.0.0" }, { name = "openai", specifier = ">=1.60.0" }, { name = "pydantic", specifier = ">=2.10.0" }, - { name = "pydantic-ai", extras = ["spec"], specifier = ">=2.24.0" }, - { name = "pydantic-evals", specifier = ">=0.2.0" }, - { name = "pydantic-graph", specifier = ">=0.2.0" }, + { name = "pydantic-ai", extras = ["spec"], specifier = ">=2.60.0" }, + { name = "pydantic-ai-harness", specifier = ">=0.60.0" }, + { name = "pydantic-evals", specifier = ">=2.60.0" }, + { name = "pydantic-graph", specifier = ">=2.60.0" }, { name = "pydantic-settings", specifier = ">=2.7.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "uvicorn", specifier = ">=0.30.0" }, ] +[[package]] +name = "pydantic-ai-harness" +version = "0.60.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "genai-prices" }, + { name = "httpx" }, + { name = "json-repair" }, + { name = "pydantic-ai-slim" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/3e/de679e739710129588acf5ae86aa43e6a4be51c1a851681a567f7445200e/pydantic_ai_harness-0.60.0.tar.gz", hash = "sha256:61f0d717fa8801998658fc94b78178106c0655b98a7d9cb1ac19e3d4d3be881f", size = 1247049, upload-time = "2026-10-23T02:02:06.401Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/84/49689c53efd7cdd6c4fef045ff7b21b856cd220bbffcd0268834ee1273c0/pydantic_ai_harness-0.60.0-py3-none-any.whl", hash = "sha256:5b8c870c76c26d1575f6b87267006a252c7c6a44ba58144f5ebcbee8a7a4256c", size = 1537743, upload-time = "2026-10-23T02:01:54.789Z" }, +] + [[package]] name = "pydantic-ai-slim" -version = "2.43.0" +version = "2.60.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1269,9 +1295,9 @@ dependencies = [ { name = "pydantic-graph" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6e/09/6672706de85503112ad76820d6e8444742f08610e5be868f75f0a37ea19a/pydantic_ai_slim-2.43.0.tar.gz", hash = "sha256:72c123024b8ecc1c6358a34dc9b88e5e7902232b16cb05a82f7a457861a891dc", size = 1464836, upload-time = "2026-09-12T01:05:38.532Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/77/738b4f89879801e7fa678e61a0cf93e0ec6f117db69a6ab92df1258cab1a/pydantic_ai_slim-2.60.0.tar.gz", hash = "sha256:cf0b2f1b8ef4ea288816d970ded00ac8718187b8bfeb66645e06ad15c2210283", size = 2135346, upload-time = "2026-10-23T02:02:07.689Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/c0/8631a038946d3782de9370ba1581d6780f276ca462b94d3e9c1d8d9e1f85/pydantic_ai_slim-2.43.0-py3-none-any.whl", hash = "sha256:b7843c2b8a8499beeb7dc82b143951bb47765256fa21732f1d4dcbfd7b84eacc", size = 1726087, upload-time = "2026-09-12T01:05:30.754Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2e/6cdaee12b671e4d96cf12eeab4b2fdac0a36baceb03dd78dd1653739a746/pydantic_ai_slim-2.60.0-py3-none-any.whl", hash = "sha256:071f93d1cb3744c46aaaa0c36eef7a11d114319cdaf5e656725c94e21ad4b693", size = 2474208, upload-time = "2026-10-23T02:01:56.552Z" }, ] [package.optional-dependencies] @@ -1388,7 +1414,7 @@ wheels = [ [[package]] name = "pydantic-evals" -version = "2.43.0" +version = "2.60.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1398,14 +1424,14 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a0/19/06010008852f19460f55064a83a2742be011bf98ae25ac0a91d2159e75b7/pydantic_evals-2.43.0.tar.gz", hash = "sha256:33e5d0b23a23c1169fda0b4b33ed3bf285dfce6eaea3758aacc190635f455759", size = 88627, upload-time = "2026-09-12T01:05:39.84Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/71/842467aa8d785dd10acb54d9e2ada75d8384e67f37fa50b002ccd764ca85/pydantic_evals-2.60.0.tar.gz", hash = "sha256:8ec8941560dc32104d41f3c2fcf061b7f88e2fa4024c8086f728b951dbcfb731", size = 92183, upload-time = "2026-10-23T02:02:10.744Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/94/0ae784371fcfad1ffc9b35054661e4d85f646e73e1d3c93380880a8320b8/pydantic_evals-2.43.0-py3-none-any.whl", hash = "sha256:0d2b599d40d8972042490485de5e8d54bfe59422922a0c669826183d8a3e1eea", size = 103905, upload-time = "2026-09-12T01:05:32.838Z" }, + { url = "https://files.pythonhosted.org/packages/be/42/ae33e0ea75c71e939c1f3be40f0db11e6840a0d2fe777f4807c6f9e8c9b7/pydantic_evals-2.60.0-py3-none-any.whl", hash = "sha256:2e42277e0111ec055304b913d867bd86ad133226d6e4df91a0faafb8ecf85ebc", size = 107193, upload-time = "2026-10-23T02:02:00.552Z" }, ] [[package]] name = "pydantic-graph" -version = "2.43.0" +version = "2.60.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1413,9 +1439,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/21/c9/cf80cd8d7b114ab6eeb87bfb786bb9a761aa51e91025e3e3bf419c2b1a9b/pydantic_graph-2.43.0.tar.gz", hash = "sha256:31ce4900f99c1c5eab7daeed07ec6f2b4545517562fe0394c5d5385a0a1528f0", size = 45189, upload-time = "2026-09-12T01:05:40.903Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/88/1a77a29a73f4cdaaea7c84d90eac381842135c401e05d85d33cdca831a0f/pydantic_graph-2.60.0.tar.gz", hash = "sha256:61b5f0bedbd77dc5526b0b70e77974b0b918ca949f228719530c0cb6b5f1200d", size = 46084, upload-time = "2026-10-23T02:02:11.697Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/f9/5d7979da026d09ca320b844e5080ef393770a7efbe14acf5e8eb92a67fdb/pydantic_graph-2.43.0-py3-none-any.whl", hash = "sha256:70ce81b2aa69440d30e5a0aa9082f13fff45b5bf4e44f06a63551e90b65f41a4", size = 52700, upload-time = "2026-09-12T01:05:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/15/2b/8ce3a473815d00e147faaffa4873417f561c39d274b7f71716b130bd74ac/pydantic_graph-2.60.0-py3-none-any.whl", hash = "sha256:88eed2878fee34d8bca35e504115f0c4509def9df6ac5087374f5bed11c4ebe3", size = 53276, upload-time = "2026-10-23T02:02:02.249Z" }, ] [[package]]