582 lines
22 KiB
Python
582 lines
22 KiB
Python
"""Unit tests for autonomy_paused event + timeout/failure tracking (IQ-Boost U3, R10).
|
|
|
|
Verifies:
|
|
- Timeout triggers autonomy_paused with reason="timeout"
|
|
- Consecutive failures (3x) trigger autonomy_paused with reason="consecutive_failures"
|
|
- resume_handler returns True → counters reset, execution continues
|
|
- resume_handler returns False → execution stops (cancel)
|
|
- resume_handler=None → auto-resume (non-blocking, for tests/REST)
|
|
- Non-autonomy mode → no pause (gate is a no-op)
|
|
- Disabled thresholds (0) → no pause
|
|
- _track_tool_result_for_autonomy increments on error, resets on success
|
|
- U6: resume Future pre-registration race fix
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from agentkit.core.react import ReActEngine, ReActEvent
|
|
from agentkit.server.config import DangerousToolsConfig
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper: simulates the caller pattern (detect → yield event → await resume)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def _detect_and_await_pause(
|
|
engine: ReActEngine,
|
|
step: int,
|
|
progress: dict,
|
|
resume_handler=None,
|
|
) -> tuple[bool, list[ReActEvent]]:
|
|
"""Simulate the react.py caller pattern for autonomy pause.
|
|
|
|
Mirrors the production code: detect (non-blocking) → build event →
|
|
yield event → await resume (blocking). Returns (should_continue, events).
|
|
"""
|
|
pause_info = engine._detect_autonomy_pause(step, progress)
|
|
if pause_info is None:
|
|
return True, []
|
|
reason, token, event_data = pause_info
|
|
event = ReActEvent(event_type="autonomy_paused", step=step, data=event_data)
|
|
should_continue = await engine._await_autonomy_resume(token, reason, resume_handler)
|
|
return should_continue, [event]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _track_tool_result_for_autonomy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTrackToolResult:
|
|
"""Failure tracking increments on error, resets on success."""
|
|
|
|
def test_success_resets_counter(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 2
|
|
engine._track_tool_result_for_autonomy({"output": "ok"})
|
|
assert engine._consecutive_failures == 0
|
|
|
|
def test_error_increments_counter(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 1
|
|
engine._track_tool_result_for_autonomy({"error": "boom"})
|
|
assert engine._consecutive_failures == 2
|
|
|
|
def test_is_error_flag_increments(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._track_tool_result_for_autonomy({"is_error": True, "output": ""})
|
|
assert engine._consecutive_failures == 1
|
|
|
|
def test_error_type_increments(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._track_tool_result_for_autonomy({"error_type": "permission_denied"})
|
|
assert engine._consecutive_failures == 1
|
|
|
|
def test_non_dict_result_resets(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 2
|
|
engine._track_tool_result_for_autonomy("plain string result")
|
|
assert engine._consecutive_failures == 0
|
|
|
|
def test_no_tracking_when_not_autonomy(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=False,
|
|
)
|
|
engine._consecutive_failures = 0
|
|
engine._track_tool_result_for_autonomy({"error": "boom"})
|
|
assert engine._consecutive_failures == 0 # Unchanged
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — no pause conditions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseNoTrigger:
|
|
"""When conditions are not met, no pause is triggered (should_continue=True)."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_autonomy_mode_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=False,
|
|
)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_config_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=None,
|
|
autonomy_mode=True,
|
|
)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_below_thresholds_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=30,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() # Just started
|
|
engine._consecutive_failures = 1 # Below threshold
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_disabled_thresholds_no_pause(self):
|
|
"""0 = disabled for both thresholds."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=0,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
# Even with expired time + many failures, disabled = no pause.
|
|
engine._autonomy_started_at = time.time() - 999999
|
|
engine._consecutive_failures = 99
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — timeout trigger
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseTimeout:
|
|
"""Timeout triggers autonomy_paused with reason='timeout'."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_auto_resume(self):
|
|
"""resume_handler=None → auto-resume (non-blocking)."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1, # 1 minute
|
|
max_consecutive_failures=99, # Disabled for this test
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
# Simulate started 2 minutes ago (exceeds 1-min timeout).
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=5, progress={"step": 5}, resume_handler=None
|
|
)
|
|
|
|
# Auto-resume → should_continue=True
|
|
assert should_continue is True
|
|
assert len(events) == 1
|
|
assert events[0].event_type == "autonomy_paused"
|
|
assert events[0].data["reason"] == "timeout"
|
|
assert "resume_token" in events[0].data
|
|
assert events[0].data["elapsed_seconds"] >= 120
|
|
# Counters reset after auto-resume
|
|
assert engine._consecutive_failures == 0
|
|
assert engine._autonomy_started_at > time.time() - 1 # Fresh timer
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_user_resumes(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1,
|
|
max_consecutive_failures=99,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
resume_handler = AsyncMock(return_value=True)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=3, progress={}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is True
|
|
assert events[0].data["reason"] == "timeout"
|
|
resume_handler.assert_awaited_once()
|
|
# Counters reset
|
|
assert engine._consecutive_failures == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_user_cancels(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1,
|
|
max_consecutive_failures=99,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
resume_handler = AsyncMock(return_value=False)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=3, progress={}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is False # Cancelled
|
|
assert events[0].data["reason"] == "timeout"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — consecutive failures trigger
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseConsecutiveFailures:
|
|
"""3 consecutive failures trigger autonomy_paused."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_consecutive_failures_triggers_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0, # Disabled
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time()
|
|
engine._consecutive_failures = 3 # At threshold
|
|
|
|
resume_handler = AsyncMock(return_value=True)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=5, progress={"step": 5}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is True # Resumed
|
|
assert len(events) == 1
|
|
assert events[0].event_type == "autonomy_paused"
|
|
assert events[0].data["reason"] == "consecutive_failures"
|
|
assert events[0].data["consecutive_failures"] == 3
|
|
# Reset after resume
|
|
assert engine._consecutive_failures == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_below_failure_threshold_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time()
|
|
engine._consecutive_failures = 2 # Below threshold
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_handler_exception_cancels(self):
|
|
"""If resume_handler raises, treat as cancel (should_continue=False)."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 3
|
|
|
|
async def failing_handler(token, reason):
|
|
raise RuntimeError("handler crashed")
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=failing_handler
|
|
)
|
|
|
|
assert should_continue is False # Cancelled due to exception
|
|
assert events[0].data["reason"] == "consecutive_failures"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyConfigParsing:
|
|
"""DangerousToolsConfig parses autonomy thresholds from dict."""
|
|
|
|
def test_defaults(self):
|
|
cfg = DangerousToolsConfig()
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|
|
|
|
def test_custom_values_from_dict(self):
|
|
cfg = DangerousToolsConfig.from_dict(
|
|
{
|
|
"enabled": True,
|
|
"autonomy_timeout_minutes": 60,
|
|
"max_consecutive_failures": 5,
|
|
}
|
|
)
|
|
assert cfg.autonomy_timeout_minutes == 60
|
|
assert cfg.max_consecutive_failures == 5
|
|
|
|
def test_disabled_values_from_dict(self):
|
|
cfg = DangerousToolsConfig.from_dict(
|
|
{
|
|
"autonomy_timeout_minutes": 0,
|
|
"max_consecutive_failures": 0,
|
|
}
|
|
)
|
|
assert cfg.autonomy_timeout_minutes == 0
|
|
assert cfg.max_consecutive_failures == 0
|
|
|
|
def test_partial_config_uses_defaults(self):
|
|
cfg = DangerousToolsConfig.from_dict({"enabled": True})
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|
|
|
|
def test_empty_dict_uses_defaults(self):
|
|
cfg = DangerousToolsConfig.from_dict({})
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# U6: resume Future pre-registration race fix
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestResumeFutureRaceFix:
|
|
"""U6: Future is pre-registered before send_json so the WebSocket loop
|
|
can resolve it even if the resume message arrives before _resume_handler
|
|
is called by the engine.
|
|
|
|
These tests mirror the chat.py ``autonomy_paused`` event handler +
|
|
``_resume_handler`` closure interaction without spinning up a full
|
|
WebSocket server.
|
|
"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_future_registered_before_send(self):
|
|
"""Future exists in pending_autonomy_resumes before send_json fires.
|
|
|
|
Simulates the autonomy_paused event handler: the pre-registration
|
|
step (U6) must insert the Future before the event is forwarded to
|
|
the frontend.
|
|
"""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
event_data = {
|
|
"resume_token": "autonomy_pause:timeout:1",
|
|
"reason": "timeout",
|
|
}
|
|
# Mirror chat.py: autonomy_paused event handler pre-registration.
|
|
resume_token = event_data.get("resume_token")
|
|
if resume_token and resume_token not in pending:
|
|
loop = asyncio.get_running_loop()
|
|
pending[resume_token] = loop.create_future()
|
|
# At this point send_json has NOT run yet — Future is already
|
|
# registered, so a concurrent resume message can resolve it.
|
|
assert resume_token in pending
|
|
assert not pending[resume_token].done()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_message_not_lost_when_arriving_before_handler(self):
|
|
"""Resume message arriving before _resume_handler is awaited still
|
|
resolves the Future (no lost message).
|
|
|
|
This is the core race the fix addresses: pre-registration lets the
|
|
WebSocket loop find the Future even if the engine has not yet
|
|
called _resume_handler.
|
|
"""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:timeout:1"
|
|
|
|
# Step 1: autonomy_paused event handler pre-registers Future.
|
|
loop = asyncio.get_running_loop()
|
|
pre_future: asyncio.Future[bool] = loop.create_future()
|
|
pending[resume_token] = pre_future
|
|
|
|
# Step 2: WebSocket loop receives resume BEFORE engine calls
|
|
# _resume_handler (the race window the fix closes).
|
|
assert resume_token in pending
|
|
fut = pending[resume_token]
|
|
if not fut.done():
|
|
fut.set_result(True)
|
|
|
|
# Step 3: engine finally calls _resume_handler, which awaits the
|
|
# already-resolved Future — no deadlock.
|
|
assert pre_future.done()
|
|
assert pre_future.result() is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_handler_uses_pre_registered_future(self):
|
|
"""_resume_handler finds and awaits the pre-registered Future
|
|
instead of creating a new one (which would orphan the pre-registered
|
|
Future and lose the resume message)."""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:consecutive_failures:3"
|
|
|
|
# Pre-register (autonomy_paused event handler).
|
|
loop = asyncio.get_running_loop()
|
|
pre_future: asyncio.Future[bool] = loop.create_future()
|
|
pending[resume_token] = pre_future
|
|
|
|
# Mirror _resume_handler: look up existing Future first.
|
|
future = pending.get(resume_token)
|
|
assert future is pre_future # Same object — no orphaned Future.
|
|
|
|
# Resolve and await.
|
|
pre_future.set_result(True)
|
|
result = await asyncio.wait_for(future, timeout=1.0)
|
|
assert result is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_handler_fallback_creates_future(self):
|
|
"""When no pre-registered Future exists (non-WS caller or event not
|
|
seen), _resume_handler creates one as a fallback."""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:timeout:1"
|
|
|
|
# Mirror _resume_handler fallback path.
|
|
future = pending.get(resume_token)
|
|
if future is None:
|
|
loop = asyncio.get_running_loop()
|
|
future = loop.create_future()
|
|
pending[resume_token] = future
|
|
|
|
assert resume_token in pending
|
|
assert pending[resume_token] is future
|
|
assert not future.done()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_double_resolve_guard(self):
|
|
"""Duplicate resume messages do not crash — the second is ignored
|
|
because the Future is already done.
|
|
|
|
Mirrors the WebSocket loop's double-resolve guard
|
|
(``if not fut.done(): fut.set_result(...)``).
|
|
"""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:timeout:1"
|
|
loop = asyncio.get_running_loop()
|
|
future: asyncio.Future[bool] = loop.create_future()
|
|
pending[resume_token] = future
|
|
|
|
# First resume resolves the Future.
|
|
assert not future.done()
|
|
future.set_result(True)
|
|
|
|
# Second resume: guard prevents double-resolve (would raise
|
|
# InvalidStateError without the done() check).
|
|
if not future.done():
|
|
future.set_result(True) # Not reached — guard holds.
|
|
|
|
assert future.done()
|
|
assert future.result() is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pre_registration_does_not_overwrite_existing(self):
|
|
"""Pre-registration skips when a Future already exists (idempotent).
|
|
|
|
Guards against the edge case where the engine emits autonomy_paused
|
|
twice with the same token (e.g. retry) — the second pre-registration
|
|
must not replace the first Future.
|
|
"""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:timeout:1"
|
|
loop = asyncio.get_running_loop()
|
|
original: asyncio.Future[bool] = loop.create_future()
|
|
pending[resume_token] = original
|
|
|
|
# Mirror chat.py: pre-registration checks ``not in``.
|
|
if resume_token and resume_token not in pending:
|
|
pending[resume_token] = loop.create_future()
|
|
|
|
assert pending[resume_token] is original # Unchanged.
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_full_flow_pre_register_then_handler_awaits(self):
|
|
"""End-to-end: pre-register → resolve from WS loop → handler awaits
|
|
and returns True.
|
|
|
|
Validates the complete U6 fix: Future is pre-registered before the
|
|
event is sent, the WS loop resolves it, and _resume_handler returns
|
|
the resolved value without creating a new Future.
|
|
"""
|
|
pending: dict[str, asyncio.Future] = {}
|
|
resume_token = "autonomy_pause:timeout:5"
|
|
event_data = {"resume_token": resume_token, "reason": "timeout"}
|
|
|
|
# 1. autonomy_paused event handler: pre-register BEFORE send_json.
|
|
rt = event_data.get("resume_token")
|
|
if rt and rt not in pending:
|
|
loop = asyncio.get_running_loop()
|
|
pending[rt] = loop.create_future()
|
|
|
|
# 2. WS loop resolves the Future (resume message arrives).
|
|
fut = pending[rt]
|
|
if not fut.done():
|
|
fut.set_result(True)
|
|
|
|
# 3. _resume_handler: look up pre-registered Future, await it.
|
|
handler_future = pending.get(rt)
|
|
assert handler_future is not None
|
|
result = await asyncio.wait_for(handler_future, timeout=1.0)
|
|
|
|
assert result is True
|
|
# 4. Cleanup (finally block in _resume_handler).
|
|
pending.pop(rt, None)
|
|
assert rt not in pending
|