"""Unit tests for autonomy_paused event + timeout/failure tracking (IQ-Boost U3, R10). Verifies: - Timeout triggers autonomy_paused with reason="timeout" - Consecutive failures (3x) trigger autonomy_paused with reason="consecutive_failures" - resume_handler returns True → counters reset, execution continues - resume_handler returns False → execution stops (cancel) - resume_handler=None → auto-resume (non-blocking, for tests/REST) - Non-autonomy mode → no pause (gate is a no-op) - Disabled thresholds (0) → no pause - _track_tool_result_for_autonomy increments on error, resets on success """ from __future__ import annotations import time from unittest.mock import AsyncMock, MagicMock import pytest from agentkit.core.react import ReActEngine, ReActEvent from agentkit.server.config import DangerousToolsConfig # --------------------------------------------------------------------------- # Helper: simulates the caller pattern (detect → yield event → await resume) # --------------------------------------------------------------------------- async def _detect_and_await_pause( engine: ReActEngine, step: int, progress: dict, resume_handler=None, ) -> tuple[bool, list[ReActEvent]]: """Simulate the react.py caller pattern for autonomy pause. Mirrors the production code: detect (non-blocking) → build event → yield event → await resume (blocking). Returns (should_continue, events). """ pause_info = engine._detect_autonomy_pause(step, progress) if pause_info is None: return True, [] reason, token, event_data = pause_info event = ReActEvent(event_type="autonomy_paused", step=step, data=event_data) should_continue = await engine._await_autonomy_resume( token, reason, resume_handler ) return should_continue, [event] # --------------------------------------------------------------------------- # _track_tool_result_for_autonomy # --------------------------------------------------------------------------- class TestTrackToolResult: """Failure tracking increments on error, resets on success.""" def test_success_resets_counter(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=True, ) engine._consecutive_failures = 2 engine._track_tool_result_for_autonomy({"output": "ok"}) assert engine._consecutive_failures == 0 def test_error_increments_counter(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=True, ) engine._consecutive_failures = 1 engine._track_tool_result_for_autonomy({"error": "boom"}) assert engine._consecutive_failures == 2 def test_is_error_flag_increments(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=True, ) engine._track_tool_result_for_autonomy({"is_error": True, "output": ""}) assert engine._consecutive_failures == 1 def test_error_type_increments(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=True, ) engine._track_tool_result_for_autonomy({"error_type": "permission_denied"}) assert engine._consecutive_failures == 1 def test_non_dict_result_resets(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=True, ) engine._consecutive_failures = 2 engine._track_tool_result_for_autonomy("plain string result") assert engine._consecutive_failures == 0 def test_no_tracking_when_not_autonomy(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=False, ) engine._consecutive_failures = 0 engine._track_tool_result_for_autonomy({"error": "boom"}) assert engine._consecutive_failures == 0 # Unchanged # --------------------------------------------------------------------------- # _detect_autonomy_pause + _await_autonomy_resume — no pause conditions # --------------------------------------------------------------------------- class TestAutonomyPauseNoTrigger: """When conditions are not met, no pause is triggered (should_continue=True).""" @pytest.mark.asyncio async def test_no_autonomy_mode_no_pause(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig(), autonomy_mode=False, ) should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=None ) assert should_continue is True assert events == [] @pytest.mark.asyncio async def test_no_config_no_pause(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=None, autonomy_mode=True, ) should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=None ) assert should_continue is True assert events == [] @pytest.mark.asyncio async def test_below_thresholds_no_pause(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=30, max_consecutive_failures=3, ), autonomy_mode=True, ) engine._autonomy_started_at = time.time() # Just started engine._consecutive_failures = 1 # Below threshold should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=None ) assert should_continue is True assert events == [] @pytest.mark.asyncio async def test_disabled_thresholds_no_pause(self): """0 = disabled for both thresholds.""" engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=0, max_consecutive_failures=0, ), autonomy_mode=True, ) # Even with expired time + many failures, disabled = no pause. engine._autonomy_started_at = time.time() - 999999 engine._consecutive_failures = 99 should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=None ) assert should_continue is True assert events == [] # --------------------------------------------------------------------------- # _detect_autonomy_pause + _await_autonomy_resume — timeout trigger # --------------------------------------------------------------------------- class TestAutonomyPauseTimeout: """Timeout triggers autonomy_paused with reason='timeout'.""" @pytest.mark.asyncio async def test_timeout_triggers_pause_auto_resume(self): """resume_handler=None → auto-resume (non-blocking).""" engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=1, # 1 minute max_consecutive_failures=99, # Disabled for this test ), autonomy_mode=True, ) # Simulate started 2 minutes ago (exceeds 1-min timeout). engine._autonomy_started_at = time.time() - 120 should_continue, events = await _detect_and_await_pause( engine, step=5, progress={"step": 5}, resume_handler=None ) # Auto-resume → should_continue=True assert should_continue is True assert len(events) == 1 assert events[0].event_type == "autonomy_paused" assert events[0].data["reason"] == "timeout" assert "resume_token" in events[0].data assert events[0].data["elapsed_seconds"] >= 120 # Counters reset after auto-resume assert engine._consecutive_failures == 0 assert engine._autonomy_started_at > time.time() - 1 # Fresh timer @pytest.mark.asyncio async def test_timeout_triggers_pause_user_resumes(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=1, max_consecutive_failures=99, ), autonomy_mode=True, ) engine._autonomy_started_at = time.time() - 120 resume_handler = AsyncMock(return_value=True) should_continue, events = await _detect_and_await_pause( engine, step=3, progress={}, resume_handler=resume_handler ) assert should_continue is True assert events[0].data["reason"] == "timeout" resume_handler.assert_awaited_once() # Counters reset assert engine._consecutive_failures == 0 @pytest.mark.asyncio async def test_timeout_triggers_pause_user_cancels(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=1, max_consecutive_failures=99, ), autonomy_mode=True, ) engine._autonomy_started_at = time.time() - 120 resume_handler = AsyncMock(return_value=False) should_continue, events = await _detect_and_await_pause( engine, step=3, progress={}, resume_handler=resume_handler ) assert should_continue is False # Cancelled assert events[0].data["reason"] == "timeout" # --------------------------------------------------------------------------- # _detect_autonomy_pause + _await_autonomy_resume — consecutive failures trigger # --------------------------------------------------------------------------- class TestAutonomyPauseConsecutiveFailures: """3 consecutive failures trigger autonomy_paused.""" @pytest.mark.asyncio async def test_consecutive_failures_triggers_pause(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=0, # Disabled max_consecutive_failures=3, ), autonomy_mode=True, ) engine._autonomy_started_at = time.time() engine._consecutive_failures = 3 # At threshold resume_handler = AsyncMock(return_value=True) should_continue, events = await _detect_and_await_pause( engine, step=5, progress={"step": 5}, resume_handler=resume_handler ) assert should_continue is True # Resumed assert len(events) == 1 assert events[0].event_type == "autonomy_paused" assert events[0].data["reason"] == "consecutive_failures" assert events[0].data["consecutive_failures"] == 3 # Reset after resume assert engine._consecutive_failures == 0 @pytest.mark.asyncio async def test_below_failure_threshold_no_pause(self): engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=0, max_consecutive_failures=3, ), autonomy_mode=True, ) engine._autonomy_started_at = time.time() engine._consecutive_failures = 2 # Below threshold should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=None ) assert should_continue is True assert events == [] @pytest.mark.asyncio async def test_resume_handler_exception_cancels(self): """If resume_handler raises, treat as cancel (should_continue=False).""" engine = ReActEngine( llm_gateway=MagicMock(), dangerous_tools_config=DangerousToolsConfig( autonomy_timeout_minutes=0, max_consecutive_failures=3, ), autonomy_mode=True, ) engine._consecutive_failures = 3 async def failing_handler(token, reason): raise RuntimeError("handler crashed") should_continue, events = await _detect_and_await_pause( engine, step=1, progress={}, resume_handler=failing_handler ) assert should_continue is False # Cancelled due to exception assert events[0].data["reason"] == "consecutive_failures" # --------------------------------------------------------------------------- # Config parsing # --------------------------------------------------------------------------- class TestAutonomyConfigParsing: """DangerousToolsConfig parses autonomy thresholds from dict.""" def test_defaults(self): cfg = DangerousToolsConfig() assert cfg.autonomy_timeout_minutes == 30 assert cfg.max_consecutive_failures == 3 def test_custom_values_from_dict(self): cfg = DangerousToolsConfig.from_dict( { "enabled": True, "autonomy_timeout_minutes": 60, "max_consecutive_failures": 5, } ) assert cfg.autonomy_timeout_minutes == 60 assert cfg.max_consecutive_failures == 5 def test_disabled_values_from_dict(self): cfg = DangerousToolsConfig.from_dict( { "autonomy_timeout_minutes": 0, "max_consecutive_failures": 0, } ) assert cfg.autonomy_timeout_minutes == 0 assert cfg.max_consecutive_failures == 0 def test_partial_config_uses_defaults(self): cfg = DangerousToolsConfig.from_dict({"enabled": True}) assert cfg.autonomy_timeout_minutes == 30 assert cfg.max_consecutive_failures == 3 def test_empty_dict_uses_defaults(self): cfg = DangerousToolsConfig.from_dict({}) assert cfg.autonomy_timeout_minutes == 30 assert cfg.max_consecutive_failures == 3