389 lines
18 KiB
Python
389 lines
18 KiB
Python
"""E2E API Coverage Tests — endpoints not covered by test_basic_api.py.
|
||
|
||
Covers:
|
||
1. /api/v1/experts — 专家模板列表/详情
|
||
2. /api/v1/workflows — 工作流列表/详情
|
||
3. /api/v1/config — 配置同步 (version/all/skills/agents/workflows)
|
||
4. /api/v1/system/resources — 系统资源监控
|
||
5. /api/v1/evolution — 进化事件 (store 未配置时 503)
|
||
6. /api/v1/evolution-dashboard — 进化看板 (metrics/experiences/usage/pitfalls)
|
||
7. /api/v1/settings — 设置 (GET 各类配置；PUT 需 SYSTEM_CONFIG → 403)
|
||
8. /api/v1/auth — 认证 (login 无效凭据 401；whoami 无 bearer 401)
|
||
9. /api/v1/memory — 记忆 (retriever 未配置时 503)
|
||
|
||
每个端点至少 2 个测试：Happy path + Auth/Error path。
|
||
api_client fixture 已携带 X-API-Key（operator 角色），所以主要测试 happy path；
|
||
对需要更高权限的端点验证 403，对未配置后端的端点验证 503/空数据。
|
||
"""
|
||
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from typing import Generator
|
||
|
||
import httpx
|
||
import pytest
|
||
import yaml
|
||
|
||
pytestmark = pytest.mark.e2e_basic
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Local e2e_server fixture — overrides conftest's version for this module.
|
||
#
|
||
# The conftest has two bugs that prevent the server from starting:
|
||
# 1. Uses `python -m agentkit.cli.main` — but main.py has no
|
||
# `if __name__ == "__main__": app()` block, so the module imports
|
||
# silently and exits without invoking the Typer app.
|
||
# 2. Even with the correct entry point (`python -m agentkit`), the CLI's
|
||
# `serve` command calls `Confirm.ask` for onboarding when
|
||
# `has_llm_provider()` is False (mock provider has no api_key),
|
||
# blocking forever in a subprocess with no TTY.
|
||
#
|
||
# This fixture bypasses the CLI entirely: it writes a minimal startup script
|
||
# that imports `create_app` and runs uvicorn directly. The config path is
|
||
# passed via the `AGENTKIT_CONFIG_PATH` env var that `create_app` reads.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture(scope="session")
|
||
def e2e_server(tmp_path_factory: pytest.TempPathFactory) -> Generator[str, None, None]:
|
||
"""Start a real AgentKit server for this module's tests."""
|
||
from tests.e2e.conftest import (
|
||
E2E_API_KEY,
|
||
E2E_HOST,
|
||
E2E_PORT,
|
||
_build_mock_env,
|
||
)
|
||
|
||
tmp_path = tmp_path_factory.mktemp("e2e_cov_server")
|
||
config_dir = tmp_path / "config"
|
||
config_dir.mkdir()
|
||
config_file = config_dir / "agentkit.yaml"
|
||
config_file.write_text(
|
||
yaml.dump(
|
||
{
|
||
"server": {
|
||
"host": E2E_HOST,
|
||
"port": E2E_PORT,
|
||
# server.api_key is what AuthMiddleware reads (NOT auth.api_keys).
|
||
# Without it, the middleware runs in dev mode and all requests pass.
|
||
"api_key": E2E_API_KEY,
|
||
},
|
||
"llm": {"default_provider": "mock", "providers": {"mock": {"type": "mock"}}},
|
||
}
|
||
)
|
||
)
|
||
|
||
# Minimal startup script: set config path, import create_app, run uvicorn.
|
||
# This avoids the CLI's interactive onboarding prompt entirely.
|
||
startup_script = tmp_path / "start_server.py"
|
||
startup_script.write_text(
|
||
"import os, uvicorn\n"
|
||
"from agentkit.server.app import create_app\n"
|
||
f'uvicorn.run(create_app, factory=True, host="{E2E_HOST}", '
|
||
f"port={E2E_PORT}, log_level='warning')\n"
|
||
)
|
||
|
||
env = _build_mock_env(tmp_path)
|
||
env["AGENTKIT_CONFIG_PATH"] = str(config_file)
|
||
# Ensure PYTHONPATH includes the project root so `agentkit` is importable
|
||
env["PYTHONPATH"] = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
||
|
||
proc = subprocess.Popen(
|
||
[sys.executable, "-u", str(startup_script)],
|
||
env=env,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
cwd=str(tmp_path),
|
||
)
|
||
|
||
base_url = f"http://{E2E_HOST}:{E2E_PORT}"
|
||
deadline = time.monotonic() + 45
|
||
ready = False
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
resp = httpx.get(f"{base_url}/api/v1/health", timeout=2)
|
||
if resp.status_code == 200:
|
||
ready = True
|
||
break
|
||
except httpx.RequestError:
|
||
# ConnectError (port not open) OR ReadTimeout (lifespan not ready)
|
||
pass
|
||
time.sleep(0.5)
|
||
|
||
if not ready:
|
||
proc.terminate()
|
||
try:
|
||
stdout, stderr = proc.communicate(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
stdout, stderr = proc.communicate()
|
||
pytest.fail(
|
||
f"E2E server failed to start within 45s.\n"
|
||
f"stdout: {stdout.decode()[:2000]}\n"
|
||
f"stderr: {stderr.decode()[:2000]}"
|
||
)
|
||
|
||
yield base_url
|
||
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 1. Experts API — /api/v1/experts
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestExpertsAPI:
|
||
def test_list_experts(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/experts")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "experts" in data
|
||
assert "total" in data
|
||
assert isinstance(data["experts"], list)
|
||
assert data["total"] == len(data["experts"])
|
||
|
||
def test_get_expert_not_found(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/experts/nonexistent_expert_xyz")
|
||
assert resp.status_code == 404
|
||
|
||
def test_list_experts_no_api_key(self, e2e_server: str):
|
||
"""无 API key 时应被中间件拒绝（401）。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.get("/api/v1/experts")
|
||
assert resp.status_code == 401
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 2. Workflows API — /api/v1/workflows
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestWorkflowsAPI:
|
||
def test_list_workflows(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/workflows")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "workflows" in data
|
||
assert "total" in data
|
||
assert isinstance(data["workflows"], list)
|
||
|
||
def test_get_workflow_not_found(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/workflows/nonexistent_wf_id")
|
||
assert resp.status_code == 404
|
||
|
||
def test_create_workflow_invalid_body(self, api_client: httpx.Client):
|
||
"""缺少必填字段应返回 422。"""
|
||
resp = api_client.post("/api/v1/workflows", json={})
|
||
assert resp.status_code == 422
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 3. Config Sync API — /api/v1/config
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestConfigSyncAPI:
|
||
def test_get_config_version(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/config/version")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "version" in data
|
||
assert "skill_count" in data
|
||
assert "workflow_count" in data
|
||
|
||
def test_get_all_configs(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/config/all")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "version" in data
|
||
assert "skills" in data
|
||
assert "workflows" in data
|
||
assert "synced_at" in data
|
||
|
||
def test_get_skill_configs(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/config/skills")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "skills" in data
|
||
assert "count" in data
|
||
|
||
def test_get_config_no_api_key(self, e2e_server: str):
|
||
"""config 端点需 CHAT 权限；无 API key 时中间件返回 401。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.get("/api/v1/config/version")
|
||
assert resp.status_code == 401
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 4. System Resources API — /api/v1/system/resources
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestSystemAPI:
|
||
def test_get_system_resources(self, api_client: httpx.Client):
|
||
"""端点明确不挂 SYSTEM_CONFIG（见路由注释），operator 可读。"""
|
||
resp = api_client.get("/api/v1/system/resources")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "cpu" in data
|
||
assert "memory" in data
|
||
assert "disk" in data
|
||
assert "timestamp" in data
|
||
|
||
def test_get_system_resources_no_api_key(self, e2e_server: str):
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.get("/api/v1/system/resources")
|
||
assert resp.status_code == 401
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 5. Evolution API — /api/v1/evolution
|
||
# E2E 配置无 evolution 段，store 为 None → 503
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestEvolutionAPI:
|
||
def test_list_evolution_events_store_unconfigured(self, api_client: httpx.Client):
|
||
"""E2E server 未配置 evolution store，应返回 503。"""
|
||
resp = api_client.get("/api/v1/evolution/events")
|
||
assert resp.status_code == 503
|
||
|
||
def test_list_ab_tests_store_unconfigured(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/evolution/ab-tests")
|
||
assert resp.status_code == 503
|
||
|
||
def test_trigger_evolution_no_agent(self, api_client: httpx.Client):
|
||
"""trigger 在 store 未配置时也应返回 503（_get_evolution_store 抛 503）。"""
|
||
resp = api_client.post(
|
||
"/api/v1/evolution/trigger",
|
||
json={"agent_name": "no_such_agent"},
|
||
)
|
||
assert resp.status_code == 503
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 6. Evolution Dashboard API — /api/v1/evolution-dashboard
|
||
# 使用自带 _verify_api_key；store 未配置时回退到内存空数据 → 200
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestEvolutionDashboardAPI:
|
||
def test_get_metrics(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/evolution-dashboard/metrics")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
# 默认空指标结构
|
||
assert "total_tasks" in data or "metrics" in data
|
||
|
||
def test_list_experiences(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/evolution-dashboard/experiences")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "experiences" in data
|
||
assert "total" in data
|
||
|
||
def test_get_metrics_no_api_key(self, e2e_server: str):
|
||
"""dashboard 端点自带 _verify_api_key，无 key 时返回 401。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.get("/api/v1/evolution-dashboard/metrics")
|
||
assert resp.status_code == 401
|
||
|
||
def test_pitfalls_missing_required_param(self, api_client: httpx.Client):
|
||
"""pitfalls 端点 task_type 为必填，缺失时 422。"""
|
||
resp = api_client.get("/api/v1/evolution-dashboard/pitfalls")
|
||
assert resp.status_code == 422
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 7. Settings API — /api/v1/settings
|
||
# GET 不需特殊权限；PUT 需 SYSTEM_CONFIG（operator 无 → 403）
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestSettingsAPI:
|
||
def test_get_llm_settings(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/settings/llm")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "providers" in data
|
||
|
||
def test_get_skills_settings(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/settings/skills")
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert "paths" in data
|
||
|
||
def test_get_general_settings(self, api_client: httpx.Client):
|
||
resp = api_client.get("/api/v1/settings/general")
|
||
assert resp.status_code == 200
|
||
|
||
def test_update_llm_settings_forbidden(self, api_client: httpx.Client):
|
||
"""PUT /settings/llm 需 SYSTEM_CONFIG；api_client 为 operator 角色 → 403。"""
|
||
resp = api_client.put(
|
||
"/api/v1/settings/llm",
|
||
json={"providers": []},
|
||
)
|
||
# operator 无 SYSTEM_CONFIG；非 dev mode 高风险权限 → 403
|
||
assert resp.status_code in (401, 403)
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 8. Auth API — /api/v1/auth
|
||
# login/whoami 在中间件白名单内（不需要 X-API-Key）
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestAuthAPI:
|
||
def test_login_invalid_credentials(self, e2e_server: str):
|
||
"""login 端点白名单，无需 API key；无效凭据 → 401。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.post(
|
||
"/api/v1/auth/login",
|
||
json={"username": "no_such_user", "password": "wrong_password"},
|
||
)
|
||
assert resp.status_code == 401
|
||
|
||
def test_whoami_without_bearer(self, e2e_server: str):
|
||
"""whoami 自行校验 bearer token；缺失 → 401。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.get("/api/v1/auth/whoami")
|
||
assert resp.status_code == 401
|
||
|
||
def test_login_missing_fields(self, e2e_server: str):
|
||
"""login 请求体缺少必填字段 → 422。"""
|
||
client = httpx.Client(base_url=e2e_server, timeout=10)
|
||
resp = client.post("/api/v1/auth/login", json={})
|
||
assert resp.status_code == 422
|
||
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
# 9. Memory API — /api/v1/memory
|
||
# E2E 配置无 memory 段，retriever 未设置 → 503
|
||
# ═══════════════════════════════════════════════════════════════════════════
|
||
|
||
|
||
class TestMemoryAPI:
|
||
def test_search_episodic_retriever_unconfigured(self, api_client: httpx.Client):
|
||
"""retriever 未配置 → 503。"""
|
||
resp = api_client.get("/api/v1/memory/episodic", params={"query": "test"})
|
||
assert resp.status_code == 503
|
||
|
||
def test_search_semantic_retriever_unconfigured(self, api_client: httpx.Client):
|
||
resp = api_client.get(
|
||
"/api/v1/memory/semantic/search", params={"query": "test"}
|
||
)
|
||
assert resp.status_code == 503
|
||
|
||
def test_search_episodic_missing_query(self, api_client: httpx.Client):
|
||
"""query 为必填 query 参数，缺失 → 422。"""
|
||
resp = api_client.get("/api/v1/memory/episodic")
|
||
assert resp.status_code == 422
|