394 lines
14 KiB
Python
394 lines
14 KiB
Python
"""Unit tests for autonomy_paused event + timeout/failure tracking (IQ-Boost U3, R10).
|
|
|
|
Verifies:
|
|
- Timeout triggers autonomy_paused with reason="timeout"
|
|
- Consecutive failures (3x) trigger autonomy_paused with reason="consecutive_failures"
|
|
- resume_handler returns True → counters reset, execution continues
|
|
- resume_handler returns False → execution stops (cancel)
|
|
- resume_handler=None → auto-resume (non-blocking, for tests/REST)
|
|
- Non-autonomy mode → no pause (gate is a no-op)
|
|
- Disabled thresholds (0) → no pause
|
|
- _track_tool_result_for_autonomy increments on error, resets on success
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from agentkit.core.react import ReActEngine, ReActEvent
|
|
from agentkit.server.config import DangerousToolsConfig
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper: simulates the caller pattern (detect → yield event → await resume)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def _detect_and_await_pause(
|
|
engine: ReActEngine,
|
|
step: int,
|
|
progress: dict,
|
|
resume_handler=None,
|
|
) -> tuple[bool, list[ReActEvent]]:
|
|
"""Simulate the react.py caller pattern for autonomy pause.
|
|
|
|
Mirrors the production code: detect (non-blocking) → build event →
|
|
yield event → await resume (blocking). Returns (should_continue, events).
|
|
"""
|
|
pause_info = engine._detect_autonomy_pause(step, progress)
|
|
if pause_info is None:
|
|
return True, []
|
|
reason, token, event_data = pause_info
|
|
event = ReActEvent(event_type="autonomy_paused", step=step, data=event_data)
|
|
should_continue = await engine._await_autonomy_resume(
|
|
token, reason, resume_handler
|
|
)
|
|
return should_continue, [event]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _track_tool_result_for_autonomy
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTrackToolResult:
|
|
"""Failure tracking increments on error, resets on success."""
|
|
|
|
def test_success_resets_counter(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 2
|
|
engine._track_tool_result_for_autonomy({"output": "ok"})
|
|
assert engine._consecutive_failures == 0
|
|
|
|
def test_error_increments_counter(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 1
|
|
engine._track_tool_result_for_autonomy({"error": "boom"})
|
|
assert engine._consecutive_failures == 2
|
|
|
|
def test_is_error_flag_increments(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._track_tool_result_for_autonomy({"is_error": True, "output": ""})
|
|
assert engine._consecutive_failures == 1
|
|
|
|
def test_error_type_increments(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._track_tool_result_for_autonomy({"error_type": "permission_denied"})
|
|
assert engine._consecutive_failures == 1
|
|
|
|
def test_non_dict_result_resets(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 2
|
|
engine._track_tool_result_for_autonomy("plain string result")
|
|
assert engine._consecutive_failures == 0
|
|
|
|
def test_no_tracking_when_not_autonomy(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=False,
|
|
)
|
|
engine._consecutive_failures = 0
|
|
engine._track_tool_result_for_autonomy({"error": "boom"})
|
|
assert engine._consecutive_failures == 0 # Unchanged
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — no pause conditions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseNoTrigger:
|
|
"""When conditions are not met, no pause is triggered (should_continue=True)."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_autonomy_mode_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(),
|
|
autonomy_mode=False,
|
|
)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_config_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=None,
|
|
autonomy_mode=True,
|
|
)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_below_thresholds_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=30,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() # Just started
|
|
engine._consecutive_failures = 1 # Below threshold
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_disabled_thresholds_no_pause(self):
|
|
"""0 = disabled for both thresholds."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=0,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
# Even with expired time + many failures, disabled = no pause.
|
|
engine._autonomy_started_at = time.time() - 999999
|
|
engine._consecutive_failures = 99
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — timeout trigger
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseTimeout:
|
|
"""Timeout triggers autonomy_paused with reason='timeout'."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_auto_resume(self):
|
|
"""resume_handler=None → auto-resume (non-blocking)."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1, # 1 minute
|
|
max_consecutive_failures=99, # Disabled for this test
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
# Simulate started 2 minutes ago (exceeds 1-min timeout).
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=5, progress={"step": 5}, resume_handler=None
|
|
)
|
|
|
|
# Auto-resume → should_continue=True
|
|
assert should_continue is True
|
|
assert len(events) == 1
|
|
assert events[0].event_type == "autonomy_paused"
|
|
assert events[0].data["reason"] == "timeout"
|
|
assert "resume_token" in events[0].data
|
|
assert events[0].data["elapsed_seconds"] >= 120
|
|
# Counters reset after auto-resume
|
|
assert engine._consecutive_failures == 0
|
|
assert engine._autonomy_started_at > time.time() - 1 # Fresh timer
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_user_resumes(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1,
|
|
max_consecutive_failures=99,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
resume_handler = AsyncMock(return_value=True)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=3, progress={}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is True
|
|
assert events[0].data["reason"] == "timeout"
|
|
resume_handler.assert_awaited_once()
|
|
# Counters reset
|
|
assert engine._consecutive_failures == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_triggers_pause_user_cancels(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=1,
|
|
max_consecutive_failures=99,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time() - 120
|
|
|
|
resume_handler = AsyncMock(return_value=False)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=3, progress={}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is False # Cancelled
|
|
assert events[0].data["reason"] == "timeout"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _detect_autonomy_pause + _await_autonomy_resume — consecutive failures trigger
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyPauseConsecutiveFailures:
|
|
"""3 consecutive failures trigger autonomy_paused."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_consecutive_failures_triggers_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0, # Disabled
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time()
|
|
engine._consecutive_failures = 3 # At threshold
|
|
|
|
resume_handler = AsyncMock(return_value=True)
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=5, progress={"step": 5}, resume_handler=resume_handler
|
|
)
|
|
|
|
assert should_continue is True # Resumed
|
|
assert len(events) == 1
|
|
assert events[0].event_type == "autonomy_paused"
|
|
assert events[0].data["reason"] == "consecutive_failures"
|
|
assert events[0].data["consecutive_failures"] == 3
|
|
# Reset after resume
|
|
assert engine._consecutive_failures == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_below_failure_threshold_no_pause(self):
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._autonomy_started_at = time.time()
|
|
engine._consecutive_failures = 2 # Below threshold
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=None
|
|
)
|
|
|
|
assert should_continue is True
|
|
assert events == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resume_handler_exception_cancels(self):
|
|
"""If resume_handler raises, treat as cancel (should_continue=False)."""
|
|
engine = ReActEngine(
|
|
llm_gateway=MagicMock(),
|
|
dangerous_tools_config=DangerousToolsConfig(
|
|
autonomy_timeout_minutes=0,
|
|
max_consecutive_failures=3,
|
|
),
|
|
autonomy_mode=True,
|
|
)
|
|
engine._consecutive_failures = 3
|
|
|
|
async def failing_handler(token, reason):
|
|
raise RuntimeError("handler crashed")
|
|
|
|
should_continue, events = await _detect_and_await_pause(
|
|
engine, step=1, progress={}, resume_handler=failing_handler
|
|
)
|
|
|
|
assert should_continue is False # Cancelled due to exception
|
|
assert events[0].data["reason"] == "consecutive_failures"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAutonomyConfigParsing:
|
|
"""DangerousToolsConfig parses autonomy thresholds from dict."""
|
|
|
|
def test_defaults(self):
|
|
cfg = DangerousToolsConfig()
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|
|
|
|
def test_custom_values_from_dict(self):
|
|
cfg = DangerousToolsConfig.from_dict(
|
|
{
|
|
"enabled": True,
|
|
"autonomy_timeout_minutes": 60,
|
|
"max_consecutive_failures": 5,
|
|
}
|
|
)
|
|
assert cfg.autonomy_timeout_minutes == 60
|
|
assert cfg.max_consecutive_failures == 5
|
|
|
|
def test_disabled_values_from_dict(self):
|
|
cfg = DangerousToolsConfig.from_dict(
|
|
{
|
|
"autonomy_timeout_minutes": 0,
|
|
"max_consecutive_failures": 0,
|
|
}
|
|
)
|
|
assert cfg.autonomy_timeout_minutes == 0
|
|
assert cfg.max_consecutive_failures == 0
|
|
|
|
def test_partial_config_uses_defaults(self):
|
|
cfg = DangerousToolsConfig.from_dict({"enabled": True})
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|
|
|
|
def test_empty_dict_uses_defaults(self):
|
|
cfg = DangerousToolsConfig.from_dict({})
|
|
assert cfg.autonomy_timeout_minutes == 30
|
|
assert cfg.max_consecutive_failures == 3
|