"""Tests for Evolution Dashboard API routes""" from __future__ import annotations import json from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient from agentkit.llm.gateway import LLMGateway from agentkit.server.app import create_app from agentkit.server.routes.evolution_dashboard import ( DashboardExperience, DashboardOptimization, _experiences, _optimizations, ) from agentkit.skills.registry import SkillRegistry from agentkit.tools.registry import ToolRegistry # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture def mock_llm_gateway(): return LLMGateway() @pytest.fixture def skill_registry(): return SkillRegistry() @pytest.fixture def tool_registry(): return ToolRegistry() @pytest.fixture def app(mock_llm_gateway, skill_registry, tool_registry): return create_app( llm_gateway=mock_llm_gateway, skill_registry=skill_registry, tool_registry=tool_registry, ) @pytest.fixture def client(app): return TestClient(app) @pytest.fixture(autouse=True) def clear_in_memory_stores(): """Clear in-memory stores before each test""" _experiences.clear() _optimizations.clear() yield _experiences.clear() _optimizations.clear() def _add_experience( task_type: str = "code_review", goal: str = "Review PR #123", outcome: str = "success", duration: float = 30.0, ): """Helper to add an experience to the in-memory store""" exp = DashboardExperience( id=f"exp-{len(_experiences)}", task_type=task_type, goal=goal, outcome=outcome, duration_seconds=duration, created_at=datetime.now(timezone.utc), ) _experiences.append(exp) return exp def _add_optimization( task_type: str = "code_review", previous_path: list[str] | None = None, current_path: list[str] | None = None, improvement: float = 0.15, ): """Helper to add an optimization to the in-memory store""" opt = DashboardOptimization( id=f"opt-{len(_optimizations)}", task_type=task_type, previous_path=previous_path or ["step1", "step2", "step3"], current_path=current_path or ["step1", "step3"], improvement=improvement, updated_at=datetime.now(timezone.utc), ) _optimizations.append(opt) return opt # --------------------------------------------------------------------------- # GET /evolution-dashboard/experiences # --------------------------------------------------------------------------- class TestListExperiences: def test_list_experiences_empty(self, client): response = client.get("/api/v1/evolution-dashboard/experiences") assert response.status_code == 200 data = response.json() assert "experiences" in data assert "total" in data assert data["total"] == 0 assert data["experiences"] == [] def test_list_experiences_with_data(self, client): _add_experience(goal="Review PR #1", outcome="success") _add_experience(goal="Review PR #2", outcome="failure") response = client.get("/api/v1/evolution-dashboard/experiences") assert response.status_code == 200 data = response.json() assert data["total"] == 2 assert len(data["experiences"]) == 2 def test_list_experiences_filter_by_task_type(self, client): _add_experience(task_type="code_review", goal="Review PR") _add_experience(task_type="data_analysis", goal="Analyze data") response = client.get( "/api/v1/evolution-dashboard/experiences?task_type=code_review" ) assert response.status_code == 200 data = response.json() assert data["total"] == 1 assert data["experiences"][0]["task_type"] == "code_review" def test_list_experiences_filter_by_outcome(self, client): _add_experience(outcome="success", goal="Success task") _add_experience(outcome="failure", goal="Failed task") response = client.get( "/api/v1/evolution-dashboard/experiences?outcome=success" ) assert response.status_code == 200 data = response.json() assert data["total"] == 1 assert data["experiences"][0]["outcome"] == "success" def test_list_experiences_limit(self, client): for i in range(10): _add_experience(goal=f"Task {i}") response = client.get( "/api/v1/evolution-dashboard/experiences?limit=3" ) assert response.status_code == 200 data = response.json() assert len(data["experiences"]) == 3 def test_experience_structure(self, client): _add_experience() response = client.get("/api/v1/evolution-dashboard/experiences") data = response.json() exp = data["experiences"][0] assert "id" in exp assert "task_type" in exp assert "goal" in exp assert "outcome" in exp assert "duration" in exp assert "created_at" in exp # --------------------------------------------------------------------------- # GET /evolution-dashboard/metrics # --------------------------------------------------------------------------- class TestGetMetrics: def test_metrics_empty(self, client): response = client.get("/api/v1/evolution-dashboard/metrics") assert response.status_code == 200 data = response.json() assert "metrics" in data assert "trends" in data metrics = data["metrics"] assert metrics["total_tasks"] == 0 assert metrics["success_rate"] == 0.0 def test_metrics_with_data(self, client): _add_experience(outcome="success", duration=10.0) _add_experience(outcome="success", duration=20.0) _add_experience(outcome="failure", duration=30.0) response = client.get("/api/v1/evolution-dashboard/metrics") assert response.status_code == 200 data = response.json() metrics = data["metrics"] assert metrics["total_tasks"] == 3 assert metrics["success_rate"] == pytest.approx(2 / 3, abs=0.01) assert metrics["avg_duration"] == pytest.approx(20.0, abs=0.1) def test_metrics_period_7d(self, client): response = client.get("/api/v1/evolution-dashboard/metrics?period=7d") assert response.status_code == 200 data = response.json() assert len(data["trends"]) == 7 def test_metrics_period_30d(self, client): response = client.get("/api/v1/evolution-dashboard/metrics?period=30d") assert response.status_code == 200 data = response.json() assert len(data["trends"]) == 30 def test_metrics_trends_structure(self, client): response = client.get("/api/v1/evolution-dashboard/metrics?period=7d") data = response.json() for trend in data["trends"]: assert "date" in trend assert "success_rate" in trend assert "avg_duration" in trend assert "retry_rate" in trend def test_metrics_period_all(self, client): response = client.get("/api/v1/evolution-dashboard/metrics?period=all") assert response.status_code == 200 data = response.json() assert len(data["trends"]) == 30 # --------------------------------------------------------------------------- # GET /evolution-dashboard/pitfalls # --------------------------------------------------------------------------- class TestCheckPitfalls: def test_pitfalls_no_detector(self, client): """When pitfall_detector is not configured, return empty warnings""" response = client.get( "/api/v1/evolution-dashboard/pitfalls?task_type=code_review&steps=step1,step2" ) assert response.status_code == 200 data = response.json() assert "warnings" in data assert data["warnings"] == [] def test_pitfalls_no_steps(self, client): """When no steps provided, return empty warnings""" response = client.get( "/api/v1/evolution-dashboard/pitfalls?task_type=code_review" ) assert response.status_code == 200 data = response.json() assert data["warnings"] == [] def test_pitfalls_with_steps(self, client): """When steps are provided but no detector, return empty warnings""" response = client.get( "/api/v1/evolution-dashboard/pitfalls?task_type=code_review&steps=analyze,review,approve" ) assert response.status_code == 200 data = response.json() assert "warnings" in data def test_pitfalls_missing_task_type(self, client): """When task_type is missing, should return 422""" response = client.get("/api/v1/evolution-dashboard/pitfalls") assert response.status_code == 422 # --------------------------------------------------------------------------- # GET /evolution-dashboard/path-optimizations # --------------------------------------------------------------------------- class TestListPathOptimizations: def test_optimizations_empty(self, client): response = client.get("/api/v1/evolution-dashboard/path-optimizations") assert response.status_code == 200 data = response.json() assert "optimizations" in data assert data["optimizations"] == [] def test_optimizations_with_data(self, client): _add_optimization( task_type="code_review", previous_path=["analyze", "review", "approve"], current_path=["analyze", "approve"], improvement=0.2, ) response = client.get("/api/v1/evolution-dashboard/path-optimizations") assert response.status_code == 200 data = response.json() assert len(data["optimizations"]) == 1 opt = data["optimizations"][0] assert opt["task_type"] == "code_review" assert opt["improvement"] == 0.2 assert opt["previous_path"] == ["analyze", "review", "approve"] assert opt["current_path"] == ["analyze", "approve"] def test_optimizations_filter_by_task_type(self, client): _add_optimization(task_type="code_review") _add_optimization(task_type="data_analysis") response = client.get( "/api/v1/evolution-dashboard/path-optimizations?task_type=code_review" ) assert response.status_code == 200 data = response.json() assert len(data["optimizations"]) == 1 assert data["optimizations"][0]["task_type"] == "code_review" def test_optimizations_limit(self, client): for i in range(10): _add_optimization(task_type=f"task_{i}") response = client.get( "/api/v1/evolution-dashboard/path-optimizations?limit=3" ) assert response.status_code == 200 data = response.json() assert len(data["optimizations"]) <= 3 def test_optimization_structure(self, client): _add_optimization() response = client.get("/api/v1/evolution-dashboard/path-optimizations") data = response.json() opt = data["optimizations"][0] assert "id" in opt assert "task_type" in opt assert "previous_path" in opt assert "current_path" in opt assert "improvement" in opt assert "updated_at" in opt # --------------------------------------------------------------------------- # WebSocket /evolution-dashboard/ws # --------------------------------------------------------------------------- class TestEvolutionDashboardWebSocket: def test_ws_connect(self, client): with client.websocket_connect("/api/v1/evolution-dashboard/ws") as ws: data = ws.receive_json() assert data["type"] == "connected" def test_ws_ping_pong(self, client): with client.websocket_connect("/api/v1/evolution-dashboard/ws") as ws: # Receive connected message connected = ws.receive_json() assert connected["type"] == "connected" # Send ping ws.send_json({"type": "ping"}) pong = ws.receive_json() assert pong["type"] == "pong" def test_ws_subscribe(self, client): with client.websocket_connect("/api/v1/evolution-dashboard/ws") as ws: connected = ws.receive_json() assert connected["type"] == "connected" ws.send_json({"type": "subscribe", "channels": ["experiences"]}) sub = ws.receive_json() assert sub["type"] == "subscribed" # --------------------------------------------------------------------------- # With experience_store configured # --------------------------------------------------------------------------- class TestWithExperienceStore: def test_experiences_with_store(self, app, client): """Test that experiences endpoint works when experience_store is configured""" mock_store = AsyncMock() mock_store.search = AsyncMock(return_value=[]) app.state.experience_store = mock_store response = client.get("/api/v1/evolution-dashboard/experiences") assert response.status_code == 200 data = response.json() assert "experiences" in data # Clean up app.state.experience_store = None def test_metrics_with_store(self, app, client): """Test that metrics endpoint works when experience_store is configured""" from agentkit.evolution.experience_schema import EvolutionMetrics mock_store = AsyncMock() mock_metrics = EvolutionMetrics( task_type="code_review", time_window="7d", completion_rate=0.85, avg_duration=25.0, retry_rate=0.1, sample_count=100, window_start=datetime.now(timezone.utc), window_end=datetime.now(timezone.utc), ) mock_store.get_metrics = AsyncMock(return_value=[mock_metrics]) app.state.experience_store = mock_store response = client.get("/api/v1/evolution-dashboard/metrics?period=7d") assert response.status_code == 200 data = response.json() assert data["metrics"]["total_tasks"] == 100 assert data["metrics"]["success_rate"] == 0.85 # Clean up app.state.experience_store = None def test_pitfalls_with_detector(self, app, client): """Test that pitfalls endpoint works when pitfall_detector is configured""" from agentkit.evolution.pitfall_detector import PitfallWarning, WarningLevel mock_detector = AsyncMock() mock_detector.check_pitfalls = AsyncMock( return_value=[ PitfallWarning( step_name="deploy", warning_level=WarningLevel.HIGH, failure_rate=0.6, historical_failures=["timeout", "config error"], suggestion="该步骤历史失败率高达 60%,建议重点关注", ) ] ) app.state.pitfall_detector = mock_detector response = client.get( "/api/v1/evolution-dashboard/pitfalls?task_type=deployment&steps=build,deploy,verify" ) assert response.status_code == 200 data = response.json() assert len(data["warnings"]) == 1 assert data["warnings"][0]["step"] == "deploy" assert data["warnings"][0]["risk_level"] == "high" assert data["warnings"][0]["historical_failure_rate"] == 0.6 # Clean up app.state.pitfall_detector = None def test_optimizations_with_optimizer(self, app, client): """Test that path-optimizations endpoint works when path_optimizer is configured""" from agentkit.evolution.path_optimizer import ExecutionPath mock_optimizer = MagicMock() recommended_path = ExecutionPath( path_id="path-001", task_type="code_review", steps=["analyze", "approve"], total_duration=15.0, success_rate=0.9, sample_count=10, is_recommended=True, created_at=datetime.now(timezone.utc), ) mock_optimizer.get_recommended_path = MagicMock(return_value=recommended_path) mock_optimizer._recommended_paths = {"code_review": recommended_path} app.state.path_optimizer = mock_optimizer response = client.get("/api/v1/evolution-dashboard/path-optimizations") assert response.status_code == 200 data = response.json() assert len(data["optimizations"]) >= 1 # Clean up app.state.path_optimizer = None # --------------------------------------------------------------------------- # Without experience_store configured (graceful degradation) # --------------------------------------------------------------------------- class TestWithoutExperienceStore: def test_experiences_graceful(self, client): """When experience_store is None, should still return valid response""" response = client.get("/api/v1/evolution-dashboard/experiences") assert response.status_code == 200 assert "experiences" in response.json() def test_metrics_graceful(self, client): """When experience_store is None, should still return valid response""" response = client.get("/api/v1/evolution-dashboard/metrics") assert response.status_code == 200 assert "metrics" in response.json() assert "trends" in response.json() def test_pitfalls_graceful(self, client): """When pitfall_detector is None, should return empty warnings""" response = client.get( "/api/v1/evolution-dashboard/pitfalls?task_type=test&steps=step1" ) assert response.status_code == 200 assert response.json()["warnings"] == [] def test_optimizations_graceful(self, client): """When path_optimizer is None, should still return valid response""" response = client.get("/api/v1/evolution-dashboard/path-optimizations") assert response.status_code == 200 assert "optimizations" in response.json()