423 lines
14 KiB
Python
423 lines
14 KiB
Python
"""P0 Production Hardening — End-to-End Integration Tests
|
|
|
|
Verifies the full configuration wiring and feature integration:
|
|
- Config from YAML → all features configured correctly
|
|
- Cache + usage tracking: cached requests show 0 cost
|
|
- UsageStore persistence via config
|
|
- CascadeStateStore persistence via config
|
|
- EvolutionStore via config
|
|
- Semantic router via config
|
|
- Graceful degradation when backends unavailable
|
|
"""
|
|
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
|
|
from agentkit.core.protocol import EvolutionEvent
|
|
from agentkit.llm.config import CacheConfig, LLMConfig, ProviderConfig
|
|
from agentkit.llm.gateway import LLMGateway
|
|
from agentkit.llm.protocol import TokenUsage
|
|
from agentkit.llm.providers.usage_store import (
|
|
InMemoryUsageStore,
|
|
create_usage_store,
|
|
)
|
|
from agentkit.quality.cascade_detector import CascadeDetector
|
|
from agentkit.quality.cascade_state_store import (
|
|
InMemoryCascadeStateStore,
|
|
create_cascade_state_store,
|
|
)
|
|
from agentkit.evolution.evolution_store import (
|
|
EvolutionStoreProtocol,
|
|
InMemoryEvolutionStore,
|
|
PersistentEvolutionStore,
|
|
create_evolution_store,
|
|
)
|
|
from agentkit.server.config import ServerConfig
|
|
|
|
|
|
# ── Config parsing tests ──────────────────────────────────
|
|
|
|
|
|
class TestConfigParsing:
|
|
"""Verify ServerConfig correctly parses all new config sections."""
|
|
|
|
def test_usage_store_config_parsed(self):
|
|
data = {
|
|
"usage_store": {
|
|
"backend": "redis",
|
|
"redis_url": "redis://custom:6380",
|
|
}
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
assert config.usage_store["backend"] == "redis"
|
|
assert config.usage_store["redis_url"] == "redis://custom:6380"
|
|
|
|
def test_cascade_store_config_parsed(self):
|
|
data = {
|
|
"cascade_store": {
|
|
"backend": "redis",
|
|
"redis_url": "redis://custom:6380",
|
|
"session_ttl": 3600,
|
|
}
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
assert config.cascade_store["backend"] == "redis"
|
|
assert config.cascade_store["session_ttl"] == 3600
|
|
|
|
def test_evolution_config_parsed(self):
|
|
data = {
|
|
"evolution": {
|
|
"backend": "sqlite",
|
|
"db_path": "/tmp/test.db",
|
|
}
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
assert config.evolution["backend"] == "sqlite"
|
|
assert config.evolution["db_path"] == "/tmp/test.db"
|
|
|
|
def test_llm_cache_config_parsed(self):
|
|
data = {
|
|
"llm": {
|
|
"providers": {},
|
|
"cache": {
|
|
"enabled": True,
|
|
"backend": "memory",
|
|
"exact_ttl": 7200,
|
|
},
|
|
}
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
assert config.llm_config.cache is not None
|
|
assert config.llm_config.cache.enabled is True
|
|
assert config.llm_config.cache.backend == "memory"
|
|
assert config.llm_config.cache.exact_ttl == 7200
|
|
|
|
def test_router_semantic_config_parsed(self):
|
|
data = {
|
|
"router": {
|
|
"classifier": "heuristic",
|
|
"semantic": {
|
|
"enabled": True,
|
|
"similarity_high": 0.9,
|
|
"similarity_low": 0.5,
|
|
},
|
|
}
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
assert config.router["semantic"]["enabled"] is True
|
|
assert config.router["semantic"]["similarity_high"] == 0.9
|
|
|
|
def test_empty_config_defaults(self):
|
|
config = ServerConfig.from_dict({})
|
|
assert config.usage_store == {}
|
|
assert config.cascade_store == {}
|
|
assert config.evolution == {}
|
|
assert config.llm_config.cache is None
|
|
|
|
def test_config_from_yaml_roundtrip(self, tmp_path):
|
|
"""Config can be loaded from a YAML file with all new sections."""
|
|
yaml_content = """
|
|
server:
|
|
host: 0.0.0.0
|
|
port: 8001
|
|
llm:
|
|
providers: {}
|
|
cache:
|
|
enabled: true
|
|
backend: memory
|
|
router:
|
|
classifier: heuristic
|
|
semantic:
|
|
enabled: false
|
|
usage_store:
|
|
backend: memory
|
|
cascade_store:
|
|
backend: memory
|
|
evolution:
|
|
backend: memory
|
|
"""
|
|
yaml_path = str(tmp_path / "test_config.yaml")
|
|
with open(yaml_path, "w") as f:
|
|
f.write(yaml_content)
|
|
|
|
config = ServerConfig.from_yaml(yaml_path)
|
|
assert config.llm_config.cache is not None
|
|
assert config.llm_config.cache.enabled is True
|
|
assert config.usage_store["backend"] == "memory"
|
|
assert config.cascade_store["backend"] == "memory"
|
|
assert config.evolution["backend"] == "memory"
|
|
|
|
|
|
# ── UsageStore integration tests ───────────────────────────
|
|
|
|
|
|
class TestUsageStoreIntegration:
|
|
"""Verify UsageStore works with LLMGateway."""
|
|
|
|
async def test_gateway_with_usage_store(self):
|
|
"""LLMGateway uses injected UsageStore for tracking."""
|
|
store = InMemoryUsageStore()
|
|
gateway = LLMGateway(usage_store=store)
|
|
|
|
# Record usage directly through the tracker
|
|
gateway._usage_tracker.record(
|
|
agent_name="test_agent",
|
|
model="gpt-4",
|
|
usage=TokenUsage(prompt_tokens=100, completion_tokens=50),
|
|
cost=0.01,
|
|
latency_ms=100.0,
|
|
)
|
|
|
|
usage = gateway.get_usage()
|
|
assert usage.total_tokens > 0
|
|
assert usage.total_cost > 0
|
|
|
|
async def test_gateway_without_usage_store(self):
|
|
"""LLMGateway works without explicit UsageStore (uses InMemory)."""
|
|
gateway = LLMGateway()
|
|
gateway._usage_tracker.record(
|
|
agent_name="test_agent",
|
|
model="gpt-4",
|
|
usage=TokenUsage(prompt_tokens=100, completion_tokens=50),
|
|
cost=0.01,
|
|
latency_ms=100.0,
|
|
)
|
|
|
|
usage = gateway.get_usage()
|
|
assert usage.total_tokens > 0
|
|
|
|
def test_create_usage_store_memory(self):
|
|
store = create_usage_store(backend="memory")
|
|
assert isinstance(store, InMemoryUsageStore)
|
|
|
|
def test_create_usage_store_redis_lazy(self):
|
|
"""Redis backend creates RedisUsageStore (lazy connection, degrades on first op)."""
|
|
from agentkit.llm.providers.usage_store import RedisUsageStore
|
|
|
|
store = create_usage_store(
|
|
backend="redis",
|
|
redis_url="redis://nonexistent:6379",
|
|
)
|
|
# RedisUsageStore is created (lazy connection), degrades on first operation
|
|
assert isinstance(store, RedisUsageStore)
|
|
|
|
|
|
# ── CascadeStateStore integration tests ────────────────────
|
|
|
|
|
|
class TestCascadeStateStoreIntegration:
|
|
"""Verify CascadeStateStore works with CascadeDetector."""
|
|
|
|
async def test_cascade_detector_with_store(self):
|
|
"""CascadeDetector uses injected CascadeStateStore."""
|
|
store = InMemoryCascadeStateStore()
|
|
detector = CascadeDetector(store=store)
|
|
|
|
# Check interaction — should not trigger cascade
|
|
result = detector.check_interaction(session_id="test-session")
|
|
assert result is None # No cascade alert
|
|
|
|
async def test_cascade_detector_without_store(self):
|
|
"""CascadeDetector works without explicit store (uses InMemory)."""
|
|
detector = CascadeDetector()
|
|
result = detector.check_interaction(session_id="test-session")
|
|
assert result is None
|
|
|
|
def test_create_cascade_state_store_memory(self):
|
|
store = create_cascade_state_store(backend="memory")
|
|
assert isinstance(store, InMemoryCascadeStateStore)
|
|
|
|
|
|
# ── EvolutionStore integration tests ───────────────────────
|
|
|
|
|
|
class TestEvolutionStoreIntegration:
|
|
"""Verify EvolutionStore creation from config."""
|
|
|
|
async def test_create_evolution_store_from_config(self, tmp_path):
|
|
"""EvolutionStore created from config dict works correctly."""
|
|
db_path = str(tmp_path / "evo_test.db")
|
|
store = create_evolution_store(backend="sqlite", db_path=db_path)
|
|
assert isinstance(store, PersistentEvolutionStore)
|
|
|
|
event = EvolutionEvent(
|
|
agent_name="test_agent",
|
|
change_type="prompt",
|
|
before={"old": 1},
|
|
after={"new": 2},
|
|
)
|
|
event_id = await store.record(event)
|
|
assert event_id is not None
|
|
|
|
events = await store.list_events()
|
|
assert len(events) == 1
|
|
assert events[0]["agent_name"] == "test_agent"
|
|
|
|
async def test_create_evolution_store_memory(self):
|
|
store = create_evolution_store(backend="memory")
|
|
assert isinstance(store, InMemoryEvolutionStore)
|
|
|
|
event = EvolutionEvent(
|
|
agent_name="test_agent",
|
|
change_type="prompt",
|
|
before={},
|
|
after={},
|
|
)
|
|
event_id = await store.record(event)
|
|
assert event_id is not None
|
|
|
|
async def test_evolution_store_protocol_compliance(self):
|
|
"""All created stores satisfy EvolutionStoreProtocol."""
|
|
memory_store = create_evolution_store(backend="memory")
|
|
assert isinstance(memory_store, EvolutionStoreProtocol)
|
|
|
|
|
|
# ── Cache integration tests ────────────────────────────────
|
|
|
|
|
|
class TestCacheIntegration:
|
|
"""Verify LLMCache integration with LLMGateway via config."""
|
|
|
|
def test_gateway_with_cache_config(self):
|
|
"""LLMGateway initializes cache when CacheConfig is provided."""
|
|
config = LLMConfig(
|
|
providers={},
|
|
cache=CacheConfig(enabled=True, backend="memory"),
|
|
)
|
|
gateway = LLMGateway(config=config)
|
|
assert gateway._cache is not None
|
|
|
|
def test_gateway_without_cache_config(self):
|
|
"""LLMGateway works without cache (default)."""
|
|
config = LLMConfig(providers={})
|
|
gateway = LLMGateway(config=config)
|
|
assert gateway._cache is None
|
|
|
|
def test_gateway_cache_disabled(self):
|
|
"""LLMGateway does not initialize cache when disabled."""
|
|
config = LLMConfig(
|
|
providers={},
|
|
cache=CacheConfig(enabled=False),
|
|
)
|
|
gateway = LLMGateway(config=config)
|
|
assert gateway._cache is None
|
|
|
|
|
|
# ── Graceful degradation tests ─────────────────────────────
|
|
|
|
|
|
class TestGracefulDegradation:
|
|
"""Verify all features degrade gracefully when backends unavailable."""
|
|
|
|
def test_usage_store_auto_creates_redis(self):
|
|
"""auto backend creates RedisUsageStore (lazy connection)."""
|
|
from agentkit.llm.providers.usage_store import RedisUsageStore
|
|
|
|
store = create_usage_store(
|
|
backend="auto",
|
|
redis_url="redis://nonexistent:6379",
|
|
)
|
|
# Redis is available as package, so RedisUsageStore is created
|
|
assert isinstance(store, RedisUsageStore)
|
|
|
|
def test_cascade_store_redis_lazy(self):
|
|
"""CascadeStateStore Redis backend creates instance (lazy connection)."""
|
|
from agentkit.quality.cascade_state_store import RedisCascadeStateStore
|
|
|
|
store = create_cascade_state_store(
|
|
backend="redis",
|
|
redis_url="redis://nonexistent:6379",
|
|
)
|
|
assert isinstance(store, RedisCascadeStateStore)
|
|
|
|
def test_evolution_store_postgresql_unavailable(self):
|
|
"""EvolutionStore falls back to InMemory when PG unavailable."""
|
|
store = create_evolution_store(
|
|
backend="postgresql",
|
|
database_url=None,
|
|
)
|
|
assert isinstance(store, InMemoryEvolutionStore)
|
|
|
|
def test_cache_auto_creates_redis(self):
|
|
"""LLMCache auto backend creates RedisLLMCache (lazy connection)."""
|
|
from agentkit.llm.cache import create_llm_cache, RedisLLMCache
|
|
|
|
cache = create_llm_cache(
|
|
backend="auto",
|
|
redis_url="redis://nonexistent:6379",
|
|
)
|
|
# Redis package is available, so RedisLLMCache is created (lazy connection)
|
|
assert isinstance(cache, RedisLLMCache)
|
|
|
|
|
|
# ── Full flow test (in-memory) ─────────────────────────────
|
|
|
|
|
|
class TestFullFlowInMemory:
|
|
"""End-to-end flow test using in-memory backends (no external deps)."""
|
|
|
|
async def test_config_to_components(self):
|
|
"""ServerConfig → all components initialized correctly."""
|
|
data = {
|
|
"llm": {
|
|
"providers": {},
|
|
"cache": {
|
|
"enabled": True,
|
|
"backend": "memory",
|
|
},
|
|
},
|
|
"router": {
|
|
"classifier": "heuristic",
|
|
},
|
|
"usage_store": {"backend": "memory"},
|
|
"cascade_store": {"backend": "memory"},
|
|
"evolution": {"backend": "memory"},
|
|
}
|
|
config = ServerConfig.from_dict(data)
|
|
|
|
# Verify config parsed correctly
|
|
assert config.llm_config.cache is not None
|
|
assert config.llm_config.cache.enabled is True
|
|
assert config.usage_store["backend"] == "memory"
|
|
assert config.cascade_store["backend"] == "memory"
|
|
assert config.evolution["backend"] == "memory"
|
|
|
|
# Create components from config
|
|
usage_store = create_usage_store(
|
|
backend=config.usage_store.get("backend", "memory"),
|
|
redis_url=config.usage_store.get("redis_url", "redis://localhost:6379"),
|
|
)
|
|
gateway = LLMGateway(config=config.llm_config, usage_store=usage_store)
|
|
assert gateway._cache is not None
|
|
|
|
cascade_store = create_cascade_state_store(
|
|
backend=config.cascade_store.get("backend", "memory"),
|
|
)
|
|
detector = CascadeDetector(store=cascade_store)
|
|
|
|
evo_store = create_evolution_store(
|
|
backend=config.evolution.get("backend", "memory"),
|
|
)
|
|
|
|
# Exercise the components
|
|
gateway._usage_tracker.record(
|
|
agent_name="test",
|
|
model="gpt-4",
|
|
usage=TokenUsage(prompt_tokens=100, completion_tokens=50),
|
|
cost=0.01,
|
|
latency_ms=100.0,
|
|
)
|
|
usage = gateway.get_usage()
|
|
assert usage.total_tokens == 150
|
|
|
|
result = detector.check_interaction("s1")
|
|
assert result is None # No cascade alert
|
|
|
|
event = EvolutionEvent(
|
|
agent_name="test", change_type="prompt", before={}, after={}
|
|
)
|
|
event_id = await evo_store.record(event)
|
|
assert event_id is not None
|