"""U7: ABTester prompt-version offline comparison (R14). Covers: - compare_prompt_versions(): no episodic_memory → "no_data" - Multiple versions sorted by score descending - Single version → that version is best_version - No matching versions → "no_data" - Retrieval failure → "no_data" (non-blocking) - best_version is the highest-scored - Low-score versions included (no cleanup in compare; recommendation only) """ from __future__ import annotations from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock import pytest from agentkit.evolution.ab_tester import ABTester from agentkit.memory.base import MemoryItem def _make_item( version: int, score: float, task_hash: str = "abc123", reflection: str = "reflection text", improved_prompt: str = "improved prompt", created_at: datetime | None = None, ) -> MemoryItem: """Build a MemoryItem mimicking list_prompt_reflections_by_hash output.""" if created_at is None: created_at = datetime.now(timezone.utc) return MemoryItem( key=f"prompt_reflection:{task_hash}:{version}", value={ "input_summary": "task input", "output_summary": improved_prompt, "reflection": reflection, "quality_score": score, }, metadata={ "agent_name": "test_agent", "task_type": "prompt_reflection", "task_hash": task_hash, "version": version, "score": score, "created_at": created_at.isoformat(), }, score=score, created_at=created_at, ) class TestComparePromptVersions: """Tests for ABTester.compare_prompt_versions().""" @pytest.mark.asyncio async def test_no_episodic_memory_returns_no_data(self): """No episodic_memory wired → recommendation='no_data', empty versions.""" tester = ABTester(episodic_memory=None) result = await tester.compare_prompt_versions("any_hash") assert result["recommendation"] == "no_data" assert result["total_versions"] == 0 assert result["versions"] == [] assert result["best_version"] is None assert result["task_hash"] == "any_hash" @pytest.mark.asyncio async def test_multiple_versions_sorted_by_score_desc(self): """3 versions (0.8, 0.6, 0.4) → best_version score=0.8.""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock( return_value=[ _make_item(version=1, score=0.4), _make_item(version=2, score=0.8), _make_item(version=3, score=0.6), ] ) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("abc123") assert result["total_versions"] == 3 assert result["recommendation"] == "keep_best" versions = result["versions"] # Sorted descending by score assert versions[0]["score"] == 0.8 assert versions[1]["score"] == 0.6 assert versions[2]["score"] == 0.4 # best_version is the highest-scored assert result["best_version"]["version"] == 2 assert result["best_version"]["score"] == 0.8 @pytest.mark.asyncio async def test_single_version_is_best(self): """Only 1 version → that version is best_version.""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock( return_value=[_make_item(version=1, score=0.7)] ) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("abc123") assert result["total_versions"] == 1 assert result["recommendation"] == "keep_best" assert result["best_version"]["version"] == 1 assert result["best_version"]["score"] == 0.7 @pytest.mark.asyncio async def test_no_matching_versions_returns_no_data(self): """EpisodicMemory returns empty list → 'no_data'.""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock(return_value=[]) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("unknown_hash") assert result["recommendation"] == "no_data" assert result["total_versions"] == 0 assert result["best_version"] is None @pytest.mark.asyncio async def test_retrieval_failure_returns_no_data(self): """list_prompt_reflections_by_hash raises → 'no_data' (non-blocking).""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock( side_effect=RuntimeError("db connection failed") ) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("abc123") assert result["recommendation"] == "no_data" assert result["total_versions"] == 0 @pytest.mark.asyncio async def test_versions_include_reflection_and_prompt_fields(self): """Each version dict carries reflection_summary + improved_prompt.""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock( return_value=[ _make_item( version=1, score=0.9, reflection="avoid retrying on schema error", improved_prompt="use structured output schema", ) ] ) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("abc123") version = result["versions"][0] assert "avoid retrying on schema error" in version["reflection_summary"] assert "use structured output schema" in version["improved_prompt"] assert version["version"] == 1 assert version["score"] == 0.9 # timestamp is a string (ISO format) assert isinstance(version["timestamp"], str) @pytest.mark.asyncio async def test_low_score_versions_kept_in_list(self): """Low-score versions remain in versions list (no cleanup; recommendation only).""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock( return_value=[ _make_item(version=1, score=0.1), # low score _make_item(version=2, score=0.9), # high score ] ) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("abc123") # Both versions present — compare does not filter, only sorts assert result["total_versions"] == 2 assert result["best_version"]["score"] == 0.9 # Low-score version still in list (for inspection) scores = [v["score"] for v in result["versions"]] assert 0.1 in scores assert 0.9 in scores @pytest.mark.asyncio async def test_task_hash_echoed_in_result(self): """task_hash field in result matches input.""" episodic = MagicMock() episodic.list_prompt_reflections_by_hash = AsyncMock(return_value=[]) tester = ABTester(episodic_memory=episodic) result = await tester.compare_prompt_versions("specific_hash_456") assert result["task_hash"] == "specific_hash_456"