Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ This repository provides a comprehensive, hands-on comparison of modern AI agent
<img src="res/pydantic-ai.svg" alt="Pydantic-AI" width="120" style="vertical-align: middle;">
</picture>
</td>
<td><code>2.43.0</code></td>
<td><code>2.60.0</code></td>
<td>
<a href="https://ai.pydantic.dev/">
<picture>
Expand Down
7 changes: 5 additions & 2 deletions pydantic-ai/25_agentic_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,12 @@ def issue_refund(order_id: str, reason: str) -> str:
return f"Refund issued for {order_id} ({reason})"


def handle_ticket(ticket: str) -> str:
# Async so all cases share one event loop: a sync task runs each case in its own
# thread and loop, and the agent's pooled HTTP connections cannot cross loops.
async def handle_ticket(ticket: str) -> str:
"""Task under evaluation: run the agent on one support ticket."""
return agent.run_sync(ticket).output.strip()
result = await agent.run(ticket)
return result.output.strip()


# --- 3. Cases carry their own trajectory expectations ---
Expand Down
91 changes: 91 additions & 0 deletions pydantic-ai/28_conversation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import asyncio

from dotenv import load_dotenv

from pydantic_ai import Agent, AgentRunResult, ConversationTypeAdapter, UsageLimits
from pydantic_ai.exceptions import UsageLimitExceeded

from settings import settings

load_dotenv()

"""
-----------------------------------------------------------------------
In this example, we explore Pydantic AI with the following features:
- result.conversation: messages, running usage and conversation_id in one object
- agent.run(..., conversation=...) to continue a conversation whole
- ConversationTypeAdapter to store a conversation as JSON and restore it
- UsageLimits that apply across every turn, not just the current one

message_history= carries only the messages, so a conversation reassembled
from it silently loses its running usage: every turn's UsageLimits budget
starts over from zero and a conversation-wide cap is never reached. A
Conversation carries the messages, the usage and the conversation_id (plus
any pending deferred tool calls) together; passing it alongside
message_history=, usage= or conversation_id= raises UserError.

For more details, visit:
https://pydantic.dev/docs/ai/core-concepts/message-history/#carrying-a-conversation-whole
-----------------------------------------------------------------------
"""

agent = Agent(
model=settings.OPENAI_MODEL_NAME,
instructions="Be concise. Reply in one short sentence.",
)

# The same budget is passed to every turn: two model requests per conversation
BUDGET = UsageLimits(request_limit=2)


def show_turn(prompt: str, result: AgentRunResult[str]) -> None:
usage = result.conversation.usage
print(f"User: {prompt}")
print(f"Agent: {result.output}")
print(f"conversation_id: {result.conversation_id}")
print(f"running usage: requests={usage.requests}, tokens={usage.total_tokens}")


async def main() -> None:
# --- 1. First turn starts a conversation ---
print("=== Turn 1 ===")
prompt = "Hi, I'm Ana and I'm flying to Porto next week."
first = await agent.run(prompt, usage_limits=BUDGET)
show_turn(prompt, first)

# --- 2. Store the whole conversation and restore it ---
print("\n=== Store and restore with ConversationTypeAdapter ===")
stored = ConversationTypeAdapter.dump_json(first.conversation)
restored = ConversationTypeAdapter.validate_json(stored)
print(f"Saved {len(stored)} bytes of JSON")
print(
f"Restored: {len(restored.messages)} messages, "
f"requests={restored.usage.requests}, "
f"same id: {restored.conversation_id == first.conversation_id}"
)

# --- 3. Continue from the restored copy: the agent recalls turn 1 ---
print("\n=== Turn 2 (continued from the restored conversation) ===")
prompt = "What's my name, and which city am I visiting?"
second = await agent.run(prompt, conversation=restored, usage_limits=BUDGET)
show_turn(prompt, second)

# --- 4. The budget counts every turn, so the third one is refused ---
print("\n=== Turn 3 with conversation= ===")
prompt = "What should I eat there?"
try:
await agent.run(prompt, conversation=second.conversation, usage_limits=BUDGET)
except UsageLimitExceeded as exc:
print(f"UsageLimitExceeded: {exc}")

# --- 5. Contrast: message_history= alone drops the running usage ---
print("\n=== Turn 3 with message_history= only ===")
third = await agent.run(
prompt, message_history=second.all_messages(), usage_limits=BUDGET
)
print(f"Agent: {third.output}")
print(f"usage restarted at requests={third.usage.requests}, so the cap never bit")


if __name__ == "__main__":
asyncio.run(main())
100 changes: 100 additions & 0 deletions pydantic-ai/29_tool_call_judge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import asyncio

from dotenv import load_dotenv

from pydantic_ai import Agent, ToolCallPart, ToolReturnPart
from pydantic_ai_harness.tool_call_judge import ToolCallJudge, ToolCallVerdict

from settings import settings

load_dotenv()

"""
-----------------------------------------------------------------------
In this example, we explore Pydantic AI with the following features:
- ToolCallJudge (pydantic-ai-harness): a second model votes on tool calls
- tools=[...] to judge only the risky tools
- A custom denial_message that the agent receives instead of the result
- on_verdict to observe every decision from application code

Whether a tool call is safe often depends on its arguments rather than
the tool: deleting a cache file is fine, deleting the production database
is not. ToolCallJudge asks a second model one risk question about each
selected call after its arguments are validated and before the tool body
runs. `yes` blocks the call, `no` lets it run, and `unsure` follows
on_uncertain ('block' by default). A blocked call is not an error: the
agent reads the denial message and carries on.

NOTE: pydantic-ai-harness is Alpha (0.x), so its API may change between
minor releases. The judge is a filter, not a security boundary.

For more details, visit:
https://pydantic.dev/docs/ai/harness/tool-call-judge/
-----------------------------------------------------------------------
"""

# Simulated file system
FILES = {
"tmp/build-cache.log": "stale build output",
"config/production.db": "live customer records",
}

# --- 1. Record each verdict, keyed by the call it judged ---
verdicts: dict[str, ToolCallVerdict] = {}


def record_verdict(verdict: ToolCallVerdict) -> None:
verdicts[verdict.tool_call_id] = verdict


# --- 2. Put a judge in front of delete_file ---
judge = ToolCallJudge(
settings.OPENAI_MODEL_NAME,
tools=["delete_file"],
question="Would this delete production data, customer records or configuration?",
denial_message=(
"The safety judge blocked this `{tool_name}` call, so nothing was deleted. "
"Do not retry; tell the user the file was kept."
),
on_verdict=record_verdict,
)

agent = Agent(
model=settings.OPENAI_MODEL_NAME,
instructions="Call delete_file once for each file the user names. Be concise.",
capabilities=[judge],
)


@agent.tool_plain
def delete_file(path: str) -> str:
"""Delete a file by its path."""
FILES.pop(path, None)
print(f" [tool] delete_file({path!r}) ran")
return f"Deleted {path}."


async def main() -> None:
# --- 3. One harmless deletion and one destructive one ---
print("=== Running the agent ===")
result = await agent.run("Delete tmp/build-cache.log and config/production.db.")

# --- 4. Match each verdict to its arguments and to what the agent was told ---
print("\n=== Judge verdicts ===")
parts = [part for message in result.all_messages() for part in message.parts]
returns = {p.tool_call_id: p.content for p in parts if isinstance(p, ToolReturnPart)}
for call in parts:
if isinstance(call, ToolCallPart) and call.tool_call_id in verdicts:
verdict = verdicts[call.tool_call_id]
print(f"{call.tool_name}({call.args_as_dict()})")
print(f" vote={verdict.answer} -> {verdict.verdict}")
print(f" agent received: {returns[call.tool_call_id]}")

print(f"\nFiles left: {sorted(FILES)}")
# The judge's own model requests are added to the run's usage
print(f"Model requests (agent + judge): {result.usage.requests}")
print(f"Response: {result.output}")


if __name__ == "__main__":
asyncio.run(main())
136 changes: 136 additions & 0 deletions pydantic-ai/30_timeouts_and_deadlines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import asyncio

from dotenv import load_dotenv

from pydantic_ai import (
Agent,
ModelAPIError,
ModelRequestTimeout,
RunCancelled,
RunContext,
RunTimedOut,
)

from settings import settings

load_dotenv()

"""
-----------------------------------------------------------------------
In this example, we explore Pydantic AI with the following features:
- agent.run(..., timeout=N) to bound the wall-clock time of a whole run
- RunTimedOut, which keeps everything the run completed in all_messages()
- RunContext.deadline / remaining_time() so a tool can fit its work to the time left
- model_settings={"request_timeout": ..., "max_retries": 0} and ModelRequestTimeout

Two different clocks are at play. `timeout=` bounds the whole run (every
model request, tool call and hook) and raises RunTimedOut, a RunCancelled
that is also a TimeoutError. `request_timeout` bounds a single model
request across all of the provider SDK's retries and raises
ModelRequestTimeout, a ModelAPIError -- so a FallbackModel can move on to
another model when one is too slow, while a run timeout never falls back.

For more details, visit:
https://pydantic.dev/docs/ai/core-concepts/timeouts/
-----------------------------------------------------------------------
"""


# --- 1. An agent whose only tool is far slower than the run's budget ---
slow_agent = Agent(
model=settings.OPENAI_MODEL_NAME,
instructions="Always use the archive_search tool before answering. Be brief.",
)


@slow_agent.tool_plain
async def archive_search(query: str) -> str:
"""Search the historical archive. Slow."""
print(f" [tool] archive_search({query!r}) started, needs 30s")
await asyncio.sleep(30)
return "Archive results."


# --- 2. An agent with a tool that reads the run's deadline ---
budget_agent = Agent(
model=settings.OPENAI_MODEL_NAME,
instructions="Always call the research tool, then answer in one sentence.",
)


@budget_agent.tool
async def research(ctx: RunContext, topic: str) -> str:
"""Research a topic, as deeply as the time left allows."""
remaining = ctx.remaining_time()
if remaining is None:
print(" [tool] no deadline on this run -> deep research")
return f"(deep) {topic}: capital of Portugal, rebuilt after the 1755 earthquake."

print(f" [tool] deadline={ctx.deadline:%H:%M:%S} UTC, remaining={remaining:.1f}s")
if remaining < 10:
print(" [tool] under 10s left -> quick answer from cache")
return f"(cached) {topic}: capital of Portugal."

print(" [tool] plenty of time -> deep research (1s)")
await asyncio.sleep(1)
return f"(deep) {topic}: capital of Portugal, rebuilt after the 1755 earthquake."


async def main() -> None:

# ------------------------------------------------------------------
# Example 1: timeout= on the run raises RunTimedOut
# ------------------------------------------------------------------
print("=== Example 1: agent.run(..., timeout=3) ===")

try:
await slow_agent.run("Find old records about Lisbon's 1755 earthquake.", timeout=3)
except RunTimedOut as exc:
print(f"RunTimedOut: {exc}")
# One except clause covers both cancellation and asyncio-style timeouts
print(f" is RunCancelled: {isinstance(exc, RunCancelled)}")
print(f" is TimeoutError: {isinstance(exc, TimeoutError)}")
# The completed work survives. The tool call that never returned is closed
# out as 'interrupted' if this history is passed as message_history to resume.
print(" exc.all_messages():")
for message in exc.all_messages():
for part in message.parts:
print(f" {type(message).__name__}.{type(part).__name__}")
print()

# ------------------------------------------------------------------
# Example 2: a tool sizes its work with ctx.remaining_time()
# ------------------------------------------------------------------
print("=== Example 2: RunContext.deadline / remaining_time() ===")

for timeout in (None, 30, 8):
print(f"timeout={timeout}")
result = await budget_agent.run("Tell me about Lisbon.", timeout=timeout)
print(f" Response: {result.output}")
print()

# ------------------------------------------------------------------
# Example 3: request_timeout raises ModelRequestTimeout
# ------------------------------------------------------------------
print("=== Example 3: model_settings request_timeout ===")

# 50ms is far below real API latency, so the request is guaranteed to miss
# its deadline. max_retries=0 makes the OpenAI SDK try exactly once.
impatient_agent = Agent(
model=settings.OPENAI_MODEL_NAME,
model_settings={"request_timeout": 0.05, "max_retries": 0},
)

try:
await impatient_agent.run("What is the capital of France?")
except ModelRequestTimeout as exc:
print(f"ModelRequestTimeout: {exc}")
print(f" exc.model_name: {exc.model_name}")
print(f" exc.timeout: {exc.timeout}s")
# A model error, not a cancellation: FallbackModel falls back on it
print(f" is ModelAPIError: {isinstance(exc, ModelAPIError)}")
print(f" is RunCancelled: {isinstance(exc, RunCancelled)}")


if __name__ == "__main__":
asyncio.run(main())
Loading
Loading