334 lines
11 KiB
Python
334 lines
11 KiB
Python
"""Unit tests for Evolution API routes"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from agentkit.evolution.evolution_store import InMemoryEvolutionStore
|
|
from agentkit.llm.gateway import LLMGateway
|
|
from agentkit.llm.protocol import LLMResponse, TokenUsage
|
|
from agentkit.skills.registry import SkillRegistry
|
|
from agentkit.tools.registry import ToolRegistry
|
|
from agentkit.server.app import create_app
|
|
from unittest.mock import AsyncMock
|
|
|
|
|
|
def _run_async(coro):
|
|
"""Run an async coroutine synchronously (works on Python 3.14+)."""
|
|
try:
|
|
loop = asyncio.get_running_loop()
|
|
except RuntimeError:
|
|
loop = None
|
|
if loop and loop.is_running():
|
|
# Already in an async context — use nest_asyncio or a new thread
|
|
import concurrent.futures
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
return pool.submit(asyncio.run, coro).result()
|
|
return asyncio.run(coro)
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_llm_gateway():
|
|
gateway = LLMGateway()
|
|
mock_provider = AsyncMock()
|
|
mock_provider.chat.return_value = LLMResponse(
|
|
content='{"result": "mocked"}',
|
|
model="test-model",
|
|
usage=TokenUsage(prompt_tokens=10, completion_tokens=20),
|
|
)
|
|
gateway.register_provider("test", mock_provider)
|
|
return gateway
|
|
|
|
|
|
@pytest.fixture
|
|
def evolution_store():
|
|
return InMemoryEvolutionStore()
|
|
|
|
|
|
@pytest.fixture
|
|
def app(mock_llm_gateway, evolution_store):
|
|
app = create_app(
|
|
llm_gateway=mock_llm_gateway,
|
|
skill_registry=SkillRegistry(),
|
|
tool_registry=ToolRegistry(),
|
|
)
|
|
app.state.evolution_store = evolution_store
|
|
return app
|
|
|
|
|
|
@pytest.fixture
|
|
def client(app):
|
|
return TestClient(app)
|
|
|
|
|
|
class TestListEvolutionEvents:
|
|
"""GET /api/v1/evolution/events"""
|
|
|
|
def test_returns_empty_list(self, client):
|
|
response = client.get("/api/v1/evolution/events")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["items"] == []
|
|
assert data["total"] == 0
|
|
|
|
def test_returns_events_after_record(self, client, evolution_store):
|
|
from agentkit.core.protocol import EvolutionEvent
|
|
|
|
event = EvolutionEvent(
|
|
agent_name="test_agent",
|
|
change_type="prompt",
|
|
before={"old": "value"},
|
|
after={"new": "value"},
|
|
metrics={"quality_score": 0.9},
|
|
)
|
|
_run_async(evolution_store.record(event))
|
|
|
|
response = client.get("/api/v1/evolution/events")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 1
|
|
assert data["items"][0]["agent_name"] == "test_agent"
|
|
assert data["items"][0]["change_type"] == "prompt"
|
|
|
|
def test_filter_by_agent_name(self, client, evolution_store):
|
|
from agentkit.core.protocol import EvolutionEvent
|
|
|
|
event1 = EvolutionEvent(
|
|
agent_name="agent_a",
|
|
change_type="prompt",
|
|
before={},
|
|
after={},
|
|
)
|
|
event2 = EvolutionEvent(
|
|
agent_name="agent_b",
|
|
change_type="strategy",
|
|
before={},
|
|
after={},
|
|
)
|
|
_run_async(evolution_store.record(event1))
|
|
_run_async(evolution_store.record(event2))
|
|
|
|
response = client.get("/api/v1/evolution/events?agent_name=agent_a")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 1
|
|
assert data["items"][0]["agent_name"] == "agent_a"
|
|
|
|
def test_filter_by_event_type(self, client, evolution_store):
|
|
from agentkit.core.protocol import EvolutionEvent
|
|
|
|
event1 = EvolutionEvent(
|
|
agent_name="agent_a",
|
|
change_type="prompt",
|
|
before={},
|
|
after={},
|
|
)
|
|
event2 = EvolutionEvent(
|
|
agent_name="agent_a",
|
|
change_type="strategy",
|
|
before={},
|
|
after={},
|
|
)
|
|
_run_async(evolution_store.record(event1))
|
|
_run_async(evolution_store.record(event2))
|
|
|
|
response = client.get("/api/v1/evolution/events?event_type=strategy")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 1
|
|
assert data["items"][0]["change_type"] == "strategy"
|
|
|
|
def test_pagination(self, client, evolution_store):
|
|
from agentkit.core.protocol import EvolutionEvent
|
|
|
|
for i in range(5):
|
|
event = EvolutionEvent(
|
|
agent_name=f"agent_{i}",
|
|
change_type="prompt",
|
|
before={},
|
|
after={},
|
|
)
|
|
_run_async(evolution_store.record(event))
|
|
|
|
response = client.get("/api/v1/evolution/events?limit=2&offset=0")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert len(data["items"]) == 2
|
|
assert data["total"] == 5
|
|
|
|
def test_returns_503_when_store_not_configured(self, mock_llm_gateway):
|
|
app = create_app(
|
|
llm_gateway=mock_llm_gateway,
|
|
skill_registry=SkillRegistry(),
|
|
tool_registry=ToolRegistry(),
|
|
)
|
|
app.state.evolution_store = None
|
|
client = TestClient(app)
|
|
response = client.get("/api/v1/evolution/events")
|
|
assert response.status_code == 503
|
|
|
|
|
|
class TestGetSkillVersions:
|
|
"""GET /api/v1/evolution/skills/{skill_name}/versions"""
|
|
|
|
def test_returns_empty_versions(self, client):
|
|
response = client.get("/api/v1/evolution/skills/unknown_skill/versions")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["skill_name"] == "unknown_skill"
|
|
assert data["versions"] == []
|
|
|
|
def test_returns_versions_after_record(self, client, evolution_store):
|
|
_run_async(
|
|
evolution_store.record_skill_version(
|
|
skill_name="my_skill",
|
|
version="1.0.0",
|
|
content='{"prompt": "hello"}',
|
|
)
|
|
)
|
|
_run_async(
|
|
evolution_store.record_skill_version(
|
|
skill_name="my_skill",
|
|
version="2.0.0",
|
|
content='{"prompt": "world"}',
|
|
parent_version="1.0.0",
|
|
)
|
|
)
|
|
|
|
response = client.get("/api/v1/evolution/skills/my_skill/versions")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["skill_name"] == "my_skill"
|
|
assert len(data["versions"]) == 2
|
|
# Most recent first
|
|
assert data["versions"][0]["version"] == "2.0.0"
|
|
assert data["versions"][0]["parent_version"] == "1.0.0"
|
|
|
|
def test_returns_503_when_store_not_configured(self, mock_llm_gateway):
|
|
app = create_app(
|
|
llm_gateway=mock_llm_gateway,
|
|
skill_registry=SkillRegistry(),
|
|
tool_registry=ToolRegistry(),
|
|
)
|
|
app.state.evolution_store = None
|
|
client = TestClient(app)
|
|
response = client.get("/api/v1/evolution/skills/test/versions")
|
|
assert response.status_code == 503
|
|
|
|
|
|
class TestTriggerEvolution:
|
|
"""POST /api/v1/evolution/trigger"""
|
|
|
|
def test_trigger_returns_404_for_unknown_agent(self, client):
|
|
response = client.post(
|
|
"/api/v1/evolution/trigger",
|
|
json={"agent_name": "nonexistent"},
|
|
)
|
|
assert response.status_code == 404
|
|
|
|
def test_trigger_records_event(self, client, evolution_store):
|
|
from agentkit.skills.base import Skill, SkillConfig
|
|
|
|
# Register a skill and create an agent
|
|
skill_config = SkillConfig(
|
|
name="evo_skill",
|
|
agent_type="evo_type",
|
|
task_mode="llm_generate",
|
|
prompt={"identity": "Evo Agent"},
|
|
)
|
|
skill = Skill(config=skill_config)
|
|
client.app.state.skill_registry.register(skill)
|
|
client.post("/api/v1/agents", json={"skill_name": "evo_skill"})
|
|
|
|
response = client.post(
|
|
"/api/v1/evolution/trigger",
|
|
json={"agent_name": "evo_skill", "skill_name": "evo_skill"},
|
|
)
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["agent_name"] == "evo_skill"
|
|
assert data["status"] == "triggered"
|
|
assert "event_id" in data
|
|
|
|
def test_returns_503_when_store_not_configured(self, mock_llm_gateway):
|
|
app = create_app(
|
|
llm_gateway=mock_llm_gateway,
|
|
skill_registry=SkillRegistry(),
|
|
tool_registry=ToolRegistry(),
|
|
)
|
|
app.state.evolution_store = None
|
|
client = TestClient(app)
|
|
response = client.post(
|
|
"/api/v1/evolution/trigger",
|
|
json={"agent_name": "test"},
|
|
)
|
|
assert response.status_code == 503
|
|
|
|
|
|
class TestListABTests:
|
|
"""GET /api/v1/evolution/ab-tests"""
|
|
|
|
def test_returns_empty_list(self, client):
|
|
response = client.get("/api/v1/evolution/ab-tests")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["items"] == []
|
|
assert data["total"] == 0
|
|
|
|
def test_returns_ab_test_results(self, client, evolution_store):
|
|
_run_async(
|
|
evolution_store.record_ab_test_result(
|
|
test_id="test_1",
|
|
variant="control",
|
|
score=0.8,
|
|
sample_count=10,
|
|
)
|
|
)
|
|
_run_async(
|
|
evolution_store.record_ab_test_result(
|
|
test_id="test_1",
|
|
variant="experiment",
|
|
score=0.9,
|
|
sample_count=10,
|
|
)
|
|
)
|
|
|
|
response = client.get("/api/v1/evolution/ab-tests")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 2
|
|
|
|
def test_filter_by_status(self, client, evolution_store):
|
|
_run_async(
|
|
evolution_store.record_ab_test_result(
|
|
test_id="test_1",
|
|
variant="control",
|
|
score=0.8,
|
|
)
|
|
)
|
|
_run_async(
|
|
evolution_store.record_ab_test_result(
|
|
test_id="test_2",
|
|
variant="experiment",
|
|
score=0.9,
|
|
)
|
|
)
|
|
|
|
response = client.get("/api/v1/evolution/ab-tests?status=control")
|
|
assert response.status_code == 200
|
|
data = response.json()
|
|
assert data["total"] == 1
|
|
assert data["items"][0]["variant"] == "control"
|
|
|
|
def test_returns_503_when_store_not_configured(self, mock_llm_gateway):
|
|
app = create_app(
|
|
llm_gateway=mock_llm_gateway,
|
|
skill_registry=SkillRegistry(),
|
|
tool_registry=ToolRegistry(),
|
|
)
|
|
app.state.evolution_store = None
|
|
client = TestClient(app)
|
|
response = client.get("/api/v1/evolution/ab-tests")
|
|
assert response.status_code == 503
|