198 lines
7.2 KiB
Python
198 lines
7.2 KiB
Python
"""U7: ABTester prompt-version offline comparison (R14).
|
|
|
|
Covers:
|
|
- compare_prompt_versions(): no episodic_memory → "no_data"
|
|
- Multiple versions sorted by score descending
|
|
- Single version → that version is best_version
|
|
- No matching versions → "no_data"
|
|
- Retrieval failure → "no_data" (non-blocking)
|
|
- best_version is the highest-scored
|
|
- Low-score versions included (no cleanup in compare; recommendation only)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from agentkit.evolution.ab_tester import ABTester
|
|
from agentkit.memory.base import MemoryItem
|
|
|
|
|
|
def _make_item(
|
|
version: int,
|
|
score: float,
|
|
task_hash: str = "abc123",
|
|
reflection: str = "reflection text",
|
|
improved_prompt: str = "improved prompt",
|
|
created_at: datetime | None = None,
|
|
) -> MemoryItem:
|
|
"""Build a MemoryItem mimicking list_prompt_reflections_by_hash output."""
|
|
if created_at is None:
|
|
created_at = datetime.now(timezone.utc)
|
|
return MemoryItem(
|
|
key=f"prompt_reflection:{task_hash}:{version}",
|
|
value={
|
|
"input_summary": "task input",
|
|
"output_summary": improved_prompt,
|
|
"reflection": reflection,
|
|
"quality_score": score,
|
|
},
|
|
metadata={
|
|
"agent_name": "test_agent",
|
|
"task_type": "prompt_reflection",
|
|
"task_hash": task_hash,
|
|
"version": version,
|
|
"score": score,
|
|
"created_at": created_at.isoformat(),
|
|
},
|
|
score=score,
|
|
created_at=created_at,
|
|
)
|
|
|
|
|
|
class TestComparePromptVersions:
|
|
"""Tests for ABTester.compare_prompt_versions()."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_episodic_memory_returns_no_data(self):
|
|
"""No episodic_memory wired → recommendation='no_data', empty versions."""
|
|
tester = ABTester(episodic_memory=None)
|
|
result = await tester.compare_prompt_versions("any_hash")
|
|
|
|
assert result["recommendation"] == "no_data"
|
|
assert result["total_versions"] == 0
|
|
assert result["versions"] == []
|
|
assert result["best_version"] is None
|
|
assert result["task_hash"] == "any_hash"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_multiple_versions_sorted_by_score_desc(self):
|
|
"""3 versions (0.8, 0.6, 0.4) → best_version score=0.8."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(
|
|
return_value=[
|
|
_make_item(version=1, score=0.4),
|
|
_make_item(version=2, score=0.8),
|
|
_make_item(version=3, score=0.6),
|
|
]
|
|
)
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("abc123")
|
|
|
|
assert result["total_versions"] == 3
|
|
assert result["recommendation"] == "keep_best"
|
|
versions = result["versions"]
|
|
# Sorted descending by score
|
|
assert versions[0]["score"] == 0.8
|
|
assert versions[1]["score"] == 0.6
|
|
assert versions[2]["score"] == 0.4
|
|
# best_version is the highest-scored
|
|
assert result["best_version"]["version"] == 2
|
|
assert result["best_version"]["score"] == 0.8
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_single_version_is_best(self):
|
|
"""Only 1 version → that version is best_version."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(
|
|
return_value=[_make_item(version=1, score=0.7)]
|
|
)
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("abc123")
|
|
|
|
assert result["total_versions"] == 1
|
|
assert result["recommendation"] == "keep_best"
|
|
assert result["best_version"]["version"] == 1
|
|
assert result["best_version"]["score"] == 0.7
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_matching_versions_returns_no_data(self):
|
|
"""EpisodicMemory returns empty list → 'no_data'."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(return_value=[])
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("unknown_hash")
|
|
|
|
assert result["recommendation"] == "no_data"
|
|
assert result["total_versions"] == 0
|
|
assert result["best_version"] is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retrieval_failure_returns_no_data(self):
|
|
"""list_prompt_reflections_by_hash raises → 'no_data' (non-blocking)."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(
|
|
side_effect=RuntimeError("db connection failed")
|
|
)
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("abc123")
|
|
|
|
assert result["recommendation"] == "no_data"
|
|
assert result["total_versions"] == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_versions_include_reflection_and_prompt_fields(self):
|
|
"""Each version dict carries reflection_summary + improved_prompt."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(
|
|
return_value=[
|
|
_make_item(
|
|
version=1,
|
|
score=0.9,
|
|
reflection="avoid retrying on schema error",
|
|
improved_prompt="use structured output schema",
|
|
)
|
|
]
|
|
)
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("abc123")
|
|
|
|
version = result["versions"][0]
|
|
assert "avoid retrying on schema error" in version["reflection_summary"]
|
|
assert "use structured output schema" in version["improved_prompt"]
|
|
assert version["version"] == 1
|
|
assert version["score"] == 0.9
|
|
# timestamp is a string (ISO format)
|
|
assert isinstance(version["timestamp"], str)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_low_score_versions_kept_in_list(self):
|
|
"""Low-score versions remain in versions list (no cleanup; recommendation only)."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(
|
|
return_value=[
|
|
_make_item(version=1, score=0.1), # low score
|
|
_make_item(version=2, score=0.9), # high score
|
|
]
|
|
)
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("abc123")
|
|
|
|
# Both versions present — compare does not filter, only sorts
|
|
assert result["total_versions"] == 2
|
|
assert result["best_version"]["score"] == 0.9
|
|
# Low-score version still in list (for inspection)
|
|
scores = [v["score"] for v in result["versions"]]
|
|
assert 0.1 in scores
|
|
assert 0.9 in scores
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_task_hash_echoed_in_result(self):
|
|
"""task_hash field in result matches input."""
|
|
episodic = MagicMock()
|
|
episodic.list_prompt_reflections_by_hash = AsyncMock(return_value=[])
|
|
tester = ABTester(episodic_memory=episodic)
|
|
|
|
result = await tester.compare_prompt_versions("specific_hash_456")
|
|
|
|
assert result["task_hash"] == "specific_hash_456"
|