"""P0 Production Hardening — End-to-End Integration Tests Verifies the full configuration wiring and feature integration: - Config from YAML → all features configured correctly - Cache + usage tracking: cached requests show 0 cost - UsageStore persistence via config - CascadeStateStore persistence via config - EvolutionStore via config - Semantic router via config - Graceful degradation when backends unavailable """ import os import tempfile import pytest from agentkit.core.protocol import EvolutionEvent from agentkit.llm.config import CacheConfig, LLMConfig, ProviderConfig from agentkit.llm.gateway import LLMGateway from agentkit.llm.protocol import TokenUsage from agentkit.llm.providers.usage_store import ( InMemoryUsageStore, create_usage_store, ) from agentkit.quality.cascade_detector import CascadeDetector from agentkit.quality.cascade_state_store import ( InMemoryCascadeStateStore, create_cascade_state_store, ) from agentkit.evolution.evolution_store import ( EvolutionStoreProtocol, InMemoryEvolutionStore, PersistentEvolutionStore, create_evolution_store, ) from agentkit.server.config import ServerConfig # ── Config parsing tests ────────────────────────────────── class TestConfigParsing: """Verify ServerConfig correctly parses all new config sections.""" def test_usage_store_config_parsed(self): data = { "usage_store": { "backend": "redis", "redis_url": "redis://custom:6380", } } config = ServerConfig.from_dict(data) assert config.usage_store["backend"] == "redis" assert config.usage_store["redis_url"] == "redis://custom:6380" def test_cascade_store_config_parsed(self): data = { "cascade_store": { "backend": "redis", "redis_url": "redis://custom:6380", "session_ttl": 3600, } } config = ServerConfig.from_dict(data) assert config.cascade_store["backend"] == "redis" assert config.cascade_store["session_ttl"] == 3600 def test_evolution_config_parsed(self): data = { "evolution": { "backend": "sqlite", "db_path": "/tmp/test.db", } } config = ServerConfig.from_dict(data) assert config.evolution["backend"] == "sqlite" assert config.evolution["db_path"] == "/tmp/test.db" def test_llm_cache_config_parsed(self): data = { "llm": { "providers": {}, "cache": { "enabled": True, "backend": "memory", "exact_ttl": 7200, }, } } config = ServerConfig.from_dict(data) assert config.llm_config.cache is not None assert config.llm_config.cache.enabled is True assert config.llm_config.cache.backend == "memory" assert config.llm_config.cache.exact_ttl == 7200 def test_router_semantic_config_parsed(self): data = { "router": { "classifier": "heuristic", "semantic": { "enabled": True, "similarity_high": 0.9, "similarity_low": 0.5, }, } } config = ServerConfig.from_dict(data) assert config.router["semantic"]["enabled"] is True assert config.router["semantic"]["similarity_high"] == 0.9 def test_empty_config_defaults(self): config = ServerConfig.from_dict({}) assert config.usage_store == {} assert config.cascade_store == {} assert config.evolution == {} assert config.llm_config.cache is None def test_config_from_yaml_roundtrip(self, tmp_path): """Config can be loaded from a YAML file with all new sections.""" yaml_content = """ server: host: 0.0.0.0 port: 8001 llm: providers: {} cache: enabled: true backend: memory router: classifier: heuristic semantic: enabled: false usage_store: backend: memory cascade_store: backend: memory evolution: backend: memory """ yaml_path = str(tmp_path / "test_config.yaml") with open(yaml_path, "w") as f: f.write(yaml_content) config = ServerConfig.from_yaml(yaml_path) assert config.llm_config.cache is not None assert config.llm_config.cache.enabled is True assert config.usage_store["backend"] == "memory" assert config.cascade_store["backend"] == "memory" assert config.evolution["backend"] == "memory" # ── UsageStore integration tests ─────────────────────────── class TestUsageStoreIntegration: """Verify UsageStore works with LLMGateway.""" async def test_gateway_with_usage_store(self): """LLMGateway uses injected UsageStore for tracking.""" store = InMemoryUsageStore() gateway = LLMGateway(usage_store=store) # Record usage directly through the tracker gateway._usage_tracker.record( agent_name="test_agent", model="gpt-4", usage=TokenUsage(prompt_tokens=100, completion_tokens=50), cost=0.01, latency_ms=100.0, ) usage = gateway.get_usage() assert usage.total_tokens > 0 assert usage.total_cost > 0 async def test_gateway_without_usage_store(self): """LLMGateway works without explicit UsageStore (uses InMemory).""" gateway = LLMGateway() gateway._usage_tracker.record( agent_name="test_agent", model="gpt-4", usage=TokenUsage(prompt_tokens=100, completion_tokens=50), cost=0.01, latency_ms=100.0, ) usage = gateway.get_usage() assert usage.total_tokens > 0 def test_create_usage_store_memory(self): store = create_usage_store(backend="memory") assert isinstance(store, InMemoryUsageStore) def test_create_usage_store_redis_lazy(self): """Redis backend creates RedisUsageStore (lazy connection, degrades on first op).""" from agentkit.llm.providers.usage_store import RedisUsageStore store = create_usage_store( backend="redis", redis_url="redis://nonexistent:6379", ) # RedisUsageStore is created (lazy connection), degrades on first operation assert isinstance(store, RedisUsageStore) # ── CascadeStateStore integration tests ──────────────────── class TestCascadeStateStoreIntegration: """Verify CascadeStateStore works with CascadeDetector.""" async def test_cascade_detector_with_store(self): """CascadeDetector uses injected CascadeStateStore.""" store = InMemoryCascadeStateStore() detector = CascadeDetector(store=store) # Check interaction — should not trigger cascade result = detector.check_interaction(session_id="test-session") assert result is None # No cascade alert async def test_cascade_detector_without_store(self): """CascadeDetector works without explicit store (uses InMemory).""" detector = CascadeDetector() result = detector.check_interaction(session_id="test-session") assert result is None def test_create_cascade_state_store_memory(self): store = create_cascade_state_store(backend="memory") assert isinstance(store, InMemoryCascadeStateStore) # ── EvolutionStore integration tests ─────────────────────── class TestEvolutionStoreIntegration: """Verify EvolutionStore creation from config.""" async def test_create_evolution_store_from_config(self, tmp_path): """EvolutionStore created from config dict works correctly.""" db_path = str(tmp_path / "evo_test.db") store = create_evolution_store(backend="sqlite", db_path=db_path) assert isinstance(store, PersistentEvolutionStore) event = EvolutionEvent( agent_name="test_agent", change_type="prompt", before={"old": 1}, after={"new": 2}, ) event_id = await store.record(event) assert event_id is not None events = await store.list_events() assert len(events) == 1 assert events[0]["agent_name"] == "test_agent" async def test_create_evolution_store_memory(self): store = create_evolution_store(backend="memory") assert isinstance(store, InMemoryEvolutionStore) event = EvolutionEvent( agent_name="test_agent", change_type="prompt", before={}, after={}, ) event_id = await store.record(event) assert event_id is not None async def test_evolution_store_protocol_compliance(self): """All created stores satisfy EvolutionStoreProtocol.""" memory_store = create_evolution_store(backend="memory") assert isinstance(memory_store, EvolutionStoreProtocol) # ── Cache integration tests ──────────────────────────────── class TestCacheIntegration: """Verify LLMCache integration with LLMGateway via config.""" def test_gateway_with_cache_config(self): """LLMGateway initializes cache when CacheConfig is provided.""" config = LLMConfig( providers={}, cache=CacheConfig(enabled=True, backend="memory"), ) gateway = LLMGateway(config=config) assert gateway._cache_manager is not None def test_gateway_without_cache_config(self): """LLMGateway works without cache (default).""" config = LLMConfig(providers={}) gateway = LLMGateway(config=config) assert gateway._cache_manager is None def test_gateway_cache_disabled(self): """LLMGateway does not initialize cache when disabled.""" config = LLMConfig( providers={}, cache=CacheConfig(enabled=False), ) gateway = LLMGateway(config=config) assert gateway._cache_manager is None # ── Graceful degradation tests ───────────────────────────── class TestGracefulDegradation: """Verify all features degrade gracefully when backends unavailable.""" def test_usage_store_auto_creates_redis(self): """auto backend creates RedisUsageStore (lazy connection).""" from agentkit.llm.providers.usage_store import RedisUsageStore store = create_usage_store( backend="auto", redis_url="redis://nonexistent:6379", ) # Redis is available as package, so RedisUsageStore is created assert isinstance(store, RedisUsageStore) def test_cascade_store_redis_lazy(self): """CascadeStateStore Redis backend creates instance (lazy connection).""" from agentkit.quality.cascade_state_store import RedisCascadeStateStore store = create_cascade_state_store( backend="redis", redis_url="redis://nonexistent:6379", ) assert isinstance(store, RedisCascadeStateStore) def test_evolution_store_postgresql_unavailable(self): """EvolutionStore falls back to InMemory when PG unavailable.""" store = create_evolution_store( backend="postgresql", database_url=None, ) assert isinstance(store, InMemoryEvolutionStore) def test_cache_auto_creates_redis(self): """LLMCache auto backend creates RedisLLMCache (lazy connection).""" from agentkit.llm.cache import create_llm_cache, RedisLLMCache cache = create_llm_cache( backend="auto", redis_url="redis://nonexistent:6379", ) # Redis package is available, so RedisLLMCache is created (lazy connection) assert isinstance(cache, RedisLLMCache) # ── Full flow test (in-memory) ───────────────────────────── class TestFullFlowInMemory: """End-to-end flow test using in-memory backends (no external deps).""" async def test_config_to_components(self): """ServerConfig → all components initialized correctly.""" data = { "llm": { "providers": {}, "cache": { "enabled": True, "backend": "memory", }, }, "router": { "classifier": "heuristic", }, "usage_store": {"backend": "memory"}, "cascade_store": {"backend": "memory"}, "evolution": {"backend": "memory"}, } config = ServerConfig.from_dict(data) # Verify config parsed correctly assert config.llm_config.cache is not None assert config.llm_config.cache.enabled is True assert config.usage_store["backend"] == "memory" assert config.cascade_store["backend"] == "memory" assert config.evolution["backend"] == "memory" # Create components from config usage_store = create_usage_store( backend=config.usage_store.get("backend", "memory"), redis_url=config.usage_store.get("redis_url", "redis://localhost:6379"), ) gateway = LLMGateway(config=config.llm_config, usage_store=usage_store) assert gateway._cache_manager is not None cascade_store = create_cascade_state_store( backend=config.cascade_store.get("backend", "memory"), ) detector = CascadeDetector(store=cascade_store) evo_store = create_evolution_store( backend=config.evolution.get("backend", "memory"), ) # Exercise the components gateway._usage_tracker.record( agent_name="test", model="gpt-4", usage=TokenUsage(prompt_tokens=100, completion_tokens=50), cost=0.01, latency_ms=100.0, ) usage = gateway.get_usage() assert usage.total_tokens == 150 result = detector.check_interaction("s1") assert result is None # No cascade alert event = EvolutionEvent( agent_name="test", change_type="prompt", before={}, after={} ) event_id = await evo_store.record(event) assert event_id is not None