561 lines
20 KiB
Python
561 lines
20 KiB
Python
"""End-to-end admin workflow integration tests (U10).
|
|
|
|
Verifies complete admin workflows end-to-end via the FastAPI TestClient:
|
|
|
|
- Department lifecycle: create → update → bind skill → bind KB →
|
|
disable → enable → unbind skill → delete.
|
|
- User lifecycle: create → assign department → reset password →
|
|
remove department → soft delete.
|
|
- LLM config lifecycle: add provider → set API key → set fallback →
|
|
set quota → list → delete provider.
|
|
- Skill management flow: disable skill → verify excluded from
|
|
``GET /skills`` → enable → verify included.
|
|
- Usage dashboard flow: query usage endpoints with empty data →
|
|
verify empty results + CSV header.
|
|
|
|
The tests mount a minimal FastAPI app with the ``admin_router`` (and
|
|
the public ``skills`` router for the skill-management flow). The
|
|
``_require_admin`` dependency is overridden so the tests don't need
|
|
real JWTs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import os
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
import yaml
|
|
from fastapi import FastAPI, HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
from agentkit.server.admin.context import DepartmentContext
|
|
from agentkit.server.admin.kb_service import set_kb_service
|
|
from agentkit.server.admin.llm_config_service import (
|
|
LlmConfigService,
|
|
set_llm_config_service,
|
|
)
|
|
from agentkit.server.admin.skill_service import set_skill_service
|
|
from agentkit.server.admin.usage_service import set_usage_service
|
|
from agentkit.server.auth.models import init_auth_db
|
|
from agentkit.server.auth.session_service import SessionService, set_session_service
|
|
from agentkit.server.routes import admin as admin_routes_module
|
|
from agentkit.server.routes import skills as skills_routes_module
|
|
from agentkit.skills.registry import SkillRegistry
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test data
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
_VALID_SKILL_YAML = """\
|
|
name: e2e_test_skill
|
|
agent_type: simple_generation
|
|
version: "1.0.0"
|
|
description: "E2E test skill"
|
|
task_mode: llm_generate
|
|
execution_mode: direct
|
|
max_steps: 1
|
|
prompt:
|
|
identity: "E2E"
|
|
instructions: "Handle test"
|
|
tools: []
|
|
"""
|
|
|
|
|
|
def _sample_agentkit_config() -> dict[str, Any]:
|
|
"""A minimal agentkit.yaml-style config for testing."""
|
|
return {
|
|
"server": {"host": "0.0.0.0", "port": 8001},
|
|
"llm": {
|
|
"providers": {
|
|
"openai": {
|
|
"type": "openai",
|
|
"api_key": "sk-test-12345678",
|
|
"base_url": "https://api.openai.com/v1",
|
|
"models": {"gpt-4o": {}},
|
|
"max_tokens": 4096,
|
|
"timeout": 120.0,
|
|
},
|
|
},
|
|
"model_aliases": {"gpt4": "openai/gpt-4o"},
|
|
"fallbacks": {},
|
|
},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
async def tmp_auth_db(tmp_path: Path) -> Path:
|
|
db_path = tmp_path / "e2e_admin.db"
|
|
await init_auth_db(db_path)
|
|
return db_path
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_agentkit_yaml(tmp_path: Path) -> Path:
|
|
"""Create a temporary agentkit.yaml config file."""
|
|
path = tmp_path / "agentkit.yaml"
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
yaml.dump(_sample_agentkit_config(), f, default_flow_style=False, allow_unicode=True)
|
|
return path
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_skills_dir(tmp_path: Path) -> str:
|
|
"""A temp skills directory for YAML files."""
|
|
d = tmp_path / "skills"
|
|
d.mkdir()
|
|
return str(d)
|
|
|
|
|
|
@pytest.fixture
|
|
def skill_registry() -> SkillRegistry:
|
|
return SkillRegistry()
|
|
|
|
|
|
@pytest.fixture
|
|
def session_service(tmp_auth_db: Path):
|
|
"""Install a SessionService singleton backed by the temp DB.
|
|
|
|
Required so that ``UserService.reset_password`` can find the
|
|
SessionService via ``get_session_service()`` and revoke sessions.
|
|
"""
|
|
svc = SessionService(db_path=tmp_auth_db)
|
|
set_session_service(svc)
|
|
yield svc
|
|
set_session_service(None)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_singletons():
|
|
"""Reset singletons before/after each test to avoid state leakage."""
|
|
set_llm_config_service(None)
|
|
set_skill_service(None)
|
|
set_kb_service(None)
|
|
set_usage_service(None)
|
|
yield
|
|
set_llm_config_service(None)
|
|
set_skill_service(None)
|
|
set_kb_service(None)
|
|
set_usage_service(None)
|
|
|
|
|
|
def _make_admin_user() -> dict[str, Any]:
|
|
return {"user_id": "admin-1", "username": "admin", "role": "admin"}
|
|
|
|
|
|
def _raise_forbidden() -> dict[str, Any]:
|
|
raise HTTPException(status_code=403, detail="Admin permission required")
|
|
|
|
|
|
@pytest.fixture
|
|
def admin_app(
|
|
tmp_auth_db: Path,
|
|
tmp_agentkit_yaml: Path,
|
|
tmp_skills_dir: str,
|
|
skill_registry: SkillRegistry,
|
|
) -> FastAPI:
|
|
"""A FastAPI app with admin + skills routers mounted.
|
|
|
|
The ``_require_admin`` dependency is overridden to return a fake
|
|
admin user. The :class:`LlmConfigService` singleton is pre-populated
|
|
so the routes use the temp YAML file.
|
|
"""
|
|
set_llm_config_service(LlmConfigService(tmp_agentkit_yaml))
|
|
|
|
app = FastAPI()
|
|
app.state.auth_db_path = str(tmp_auth_db)
|
|
app.state.skill_registry = skill_registry
|
|
|
|
class _FakeServerConfig:
|
|
skill_paths = [tmp_skills_dir]
|
|
|
|
app.state.server_config = _FakeServerConfig()
|
|
|
|
app.include_router(admin_routes_module.admin_router, prefix="/api/v1")
|
|
app.include_router(skills_routes_module.router, prefix="/api/v1")
|
|
|
|
# Default: allow admin access.
|
|
app.dependency_overrides[admin_routes_module._require_admin] = lambda: _make_admin_user()
|
|
# Admin context for skills route (bypass department filtering).
|
|
app.dependency_overrides[skills_routes_module.get_department_context] = lambda: (
|
|
DepartmentContext(user_id="admin-1", department_ids=[], is_admin=True)
|
|
)
|
|
return app
|
|
|
|
|
|
@pytest.fixture
|
|
def admin_client(
|
|
admin_app: FastAPI,
|
|
session_service: SessionService,
|
|
) -> TestClient:
|
|
"""TestClient with admin access and SessionService installed."""
|
|
return TestClient(admin_app)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helper functions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _create_department(client: TestClient, name: str, description: str = "") -> dict:
|
|
resp = client.post(
|
|
"/api/v1/admin/departments",
|
|
json={"name": name, "description": description},
|
|
)
|
|
assert resp.status_code == 201, resp.text
|
|
return resp.json()
|
|
|
|
|
|
def _create_user(
|
|
client: TestClient,
|
|
*,
|
|
username: str,
|
|
email: str,
|
|
password: str = "Secret123!",
|
|
role: str = "member",
|
|
department_ids: list[str] | None = None,
|
|
) -> dict:
|
|
payload: dict[str, Any] = {
|
|
"username": username,
|
|
"email": email,
|
|
"password": password,
|
|
"role": role,
|
|
}
|
|
if department_ids is not None:
|
|
payload["department_ids"] = department_ids
|
|
resp = client.post("/api/v1/admin/users", json=payload)
|
|
assert resp.status_code == 201, resp.text
|
|
return resp.json()
|
|
|
|
|
|
def _write_skill_yaml(skills_dir: str, name: str, content: str) -> str:
|
|
path = os.path.join(skills_dir, f"{name}.yaml")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
return path
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# E2E admin flow tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestE2EAdminFlow:
|
|
"""End-to-end admin workflow test."""
|
|
|
|
def test_full_department_lifecycle(
|
|
self,
|
|
admin_client: TestClient,
|
|
tmp_skills_dir: str,
|
|
skill_registry: SkillRegistry,
|
|
):
|
|
"""Create department → update → bind skill → bind KB → disable →
|
|
enable → unbind skill → delete."""
|
|
# 1. Create department "HR".
|
|
hr = _create_department(admin_client, "HR", "Human Resources")
|
|
dept_id = hr["id"]
|
|
assert hr["name"] == "HR"
|
|
assert hr["is_active"] is True
|
|
|
|
# 2. GET /admin/departments → HR in list.
|
|
resp = admin_client.get("/api/v1/admin/departments")
|
|
assert resp.status_code == 200
|
|
names = {d["name"] for d in resp.json()}
|
|
assert "HR" in names
|
|
|
|
# 3. PATCH /admin/departments/{id} → update name to "Human Resources".
|
|
resp = admin_client.patch(
|
|
f"/api/v1/admin/departments/{dept_id}",
|
|
json={"name": "Human Resources", "description": "HR department"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["name"] == "Human Resources"
|
|
|
|
# 4. POST /admin/departments/{id}/skills/code_reviewer → bind skill.
|
|
resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/skills/code_reviewer")
|
|
assert resp.status_code == 201
|
|
assert resp.json()["skill_name"] == "code_reviewer"
|
|
|
|
# 5. GET /admin/departments/{id}/skills → ["code_reviewer"].
|
|
resp = admin_client.get(f"/api/v1/admin/departments/{dept_id}/skills")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == ["code_reviewer"]
|
|
|
|
# 6. POST /admin/departments/{id}/kb/source-1 → bind KB.
|
|
resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/kb/source-1")
|
|
assert resp.status_code == 201
|
|
assert resp.json()["kb_source_id"] == "source-1"
|
|
|
|
# 7. GET /admin/departments/{id}/kb → ["source-1"].
|
|
resp = admin_client.get(f"/api/v1/admin/departments/{dept_id}/kb")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == ["source-1"]
|
|
|
|
# 8. POST /admin/departments/{id}/disable → disabled.
|
|
resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/disable")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["is_active"] is False
|
|
|
|
# 9. POST /admin/departments/{id}/enable → enabled.
|
|
resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/enable")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["is_active"] is True
|
|
|
|
# 10. DELETE /admin/departments/{id}/skills/code_reviewer → unbound.
|
|
resp = admin_client.delete(f"/api/v1/admin/departments/{dept_id}/skills/code_reviewer")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"unbound": True}
|
|
assert admin_client.get(f"/api/v1/admin/departments/{dept_id}/skills").json() == []
|
|
|
|
# 11. DELETE /admin/departments/{id} → deleted.
|
|
resp = admin_client.delete(f"/api/v1/admin/departments/{dept_id}")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": True}
|
|
assert admin_client.get(f"/api/v1/admin/departments/{dept_id}").status_code == 404
|
|
|
|
def test_full_user_lifecycle(self, admin_client: TestClient):
|
|
"""Create user → assign department → reset password → remove
|
|
department → delete."""
|
|
# 1. Create department "Engineering".
|
|
eng = _create_department(admin_client, "Engineering")
|
|
eng_id = eng["id"]
|
|
|
|
# 2. POST /admin/users → create user "alice".
|
|
alice = _create_user(
|
|
admin_client,
|
|
username="alice",
|
|
email="alice@example.com",
|
|
)
|
|
alice_id = alice["id"]
|
|
assert alice["username"] == "alice"
|
|
assert alice["departments"] == []
|
|
|
|
# 3. POST /admin/users/{id}/departments/{dept_id} → assign.
|
|
resp = admin_client.post(f"/api/v1/admin/users/{alice_id}/departments/{eng_id}")
|
|
assert resp.status_code == 201
|
|
assert resp.json() == {"assigned": True}
|
|
|
|
# 4. GET /admin/users/{id} → has department.
|
|
resp = admin_client.get(f"/api/v1/admin/users/{alice_id}")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert len(body["departments"]) == 1
|
|
assert body["departments"][0]["name"] == "Engineering"
|
|
|
|
# 5. POST /admin/users/{id}/reset-password → reset.
|
|
resp = admin_client.post(
|
|
f"/api/v1/admin/users/{alice_id}/reset-password",
|
|
json={"new_password": "NewSecret456!"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"reset": True}
|
|
|
|
# 6. DELETE /admin/users/{id}/departments/{dept_id} → remove.
|
|
resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}/departments/{eng_id}")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"removed": True}
|
|
|
|
# Verify removal.
|
|
resp = admin_client.get(f"/api/v1/admin/users/{alice_id}")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["departments"] == []
|
|
|
|
# 7. DELETE /admin/users/{id} → soft delete.
|
|
resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": True}
|
|
|
|
# Second delete on the now-inactive user returns 404.
|
|
resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}")
|
|
assert resp.status_code == 404
|
|
|
|
def test_llm_config_lifecycle(
|
|
self,
|
|
admin_client: TestClient,
|
|
tmp_agentkit_yaml: Path,
|
|
):
|
|
"""Add provider → set API key → set fallback → set quota →
|
|
delete provider."""
|
|
# 1. POST /admin/llm/providers → create "test-provider".
|
|
resp = admin_client.post(
|
|
"/api/v1/admin/llm/providers",
|
|
json={
|
|
"name": "test-provider",
|
|
"type": "openai",
|
|
"api_key": "sk-test-abcdef1234",
|
|
"base_url": "https://api.test.com",
|
|
"models": {"test-model": {}},
|
|
"max_tokens": 2048,
|
|
"timeout": 60.0,
|
|
},
|
|
)
|
|
assert resp.status_code == 201, resp.text
|
|
assert resp.json()["name"] == "test-provider"
|
|
assert resp.json()["api_key"].startswith("****")
|
|
|
|
# 2. POST /admin/llm/providers/test-provider/api-key → set key.
|
|
resp = admin_client.post(
|
|
"/api/v1/admin/llm/providers/test-provider/api-key",
|
|
json={"api_key": "sk-brand-new-key-9876"},
|
|
)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["api_key"].startswith("****")
|
|
|
|
# 3. PUT /admin/llm/fallbacks/gpt-4 → set fallback chain.
|
|
resp = admin_client.put(
|
|
"/api/v1/admin/llm/fallbacks/gpt-4",
|
|
json={"chain": ["openai", "test-provider"]},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["model"] == "gpt-4"
|
|
assert body["chain"] == ["openai", "test-provider"]
|
|
|
|
# 4. PUT /admin/departments/{id}/quotas → set token_limit.
|
|
dept_id = str(uuid.uuid4())
|
|
resp = admin_client.put(
|
|
f"/api/v1/admin/departments/{dept_id}/quotas",
|
|
json={
|
|
"quota_type": "token_limit",
|
|
"limit_value": 1000,
|
|
"period": "daily",
|
|
},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["quota_type"] == "token_limit"
|
|
assert body["limit_value"] == 1000
|
|
|
|
# 5. GET /admin/llm/providers → provider in list.
|
|
resp = admin_client.get("/api/v1/admin/llm/providers")
|
|
assert resp.status_code == 200
|
|
names = {p["name"] for p in resp.json()}
|
|
assert "test-provider" in names
|
|
assert "openai" in names
|
|
|
|
# 6. DELETE /admin/llm/providers/test-provider → fails (used in fallback).
|
|
resp = admin_client.delete("/api/v1/admin/llm/providers/test-provider")
|
|
assert resp.status_code == 400
|
|
assert "fallback" in resp.json()["detail"].lower()
|
|
|
|
# Remove the fallback chain first, then delete.
|
|
admin_client.delete("/api/v1/admin/llm/fallbacks/gpt-4")
|
|
resp = admin_client.delete("/api/v1/admin/llm/providers/test-provider")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"deleted": True}
|
|
|
|
# Confirm it's gone.
|
|
resp = admin_client.get("/api/v1/admin/llm/providers")
|
|
names = {p["name"] for p in resp.json()}
|
|
assert "test-provider" not in names
|
|
|
|
def test_skill_management_flow(
|
|
self,
|
|
admin_client: TestClient,
|
|
tmp_skills_dir: str,
|
|
skill_registry: SkillRegistry,
|
|
):
|
|
"""Disable skill → verify excluded → enable → verify included."""
|
|
# 1. Create a test skill YAML in tmp_skills_dir and register it.
|
|
_write_skill_yaml(tmp_skills_dir, "e2e_test_skill", _VALID_SKILL_YAML)
|
|
from agentkit.skills.loader import SkillLoader
|
|
|
|
SkillLoader(skill_registry=skill_registry).load_from_file(
|
|
os.path.join(tmp_skills_dir, "e2e_test_skill.yaml")
|
|
)
|
|
|
|
# Verify the skill is initially listed.
|
|
resp = admin_client.get("/api/v1/skills")
|
|
assert resp.status_code == 200
|
|
names = [s["name"] for s in resp.json()]
|
|
assert "e2e_test_skill" in names
|
|
|
|
# 2. POST /admin/skills/{name}/disable → disabled.
|
|
resp = admin_client.post("/api/v1/admin/skills/e2e_test_skill/disable")
|
|
assert resp.status_code == 200, resp.text
|
|
body = resp.json()
|
|
assert body["skill_name"] == "e2e_test_skill"
|
|
assert body["is_disabled"] is True
|
|
|
|
# 3. Verify via GET /skills that the skill is excluded.
|
|
resp = admin_client.get("/api/v1/skills")
|
|
assert resp.status_code == 200
|
|
names = [s["name"] for s in resp.json()]
|
|
assert "e2e_test_skill" not in names
|
|
|
|
# 4. POST /admin/skills/{name}/enable → enabled.
|
|
resp = admin_client.post("/api/v1/admin/skills/e2e_test_skill/enable")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["enabled"] is True
|
|
|
|
# Verify via GET /skills that the skill is included again.
|
|
resp = admin_client.get("/api/v1/skills")
|
|
assert resp.status_code == 200
|
|
names = [s["name"] for s in resp.json()]
|
|
assert "e2e_test_skill" in names
|
|
|
|
def test_usage_dashboard_flow(self, admin_client: TestClient):
|
|
"""Query usage endpoints with empty data → verify empty results."""
|
|
# The admin_app fixture doesn't install an llm_gateway on app.state,
|
|
# so the usage routes will return 500. We need to install a stub
|
|
# gateway with an empty InMemoryUsageStore.
|
|
from agentkit.llm.providers.usage_store import InMemoryUsageStore
|
|
|
|
class _StubTracker:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
|
|
class _StubGateway:
|
|
def __init__(self, store):
|
|
self._usage_tracker = _StubTracker(store)
|
|
|
|
store = InMemoryUsageStore()
|
|
# Install the stub gateway on the app state.
|
|
admin_client.app.state.llm_gateway = _StubGateway(store)
|
|
|
|
# 1. GET /admin/usage/summary → 200 with zeros.
|
|
resp = admin_client.get("/api/v1/admin/usage/summary")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["total_tokens"] == 0
|
|
assert body["total_cost"] == 0.0
|
|
assert body["total_requests"] == 0
|
|
|
|
# 2. GET /admin/usage/timeseries → 200 with empty list.
|
|
resp = admin_client.get("/api/v1/admin/usage/timeseries")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
# 3. GET /admin/usage/by-model → 200 with empty list.
|
|
resp = admin_client.get("/api/v1/admin/usage/by-model")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
# 4. GET /admin/usage/top-users → 200 with empty list.
|
|
resp = admin_client.get("/api/v1/admin/usage/top-users")
|
|
assert resp.status_code == 200
|
|
assert resp.json() == []
|
|
|
|
# 5. GET /admin/usage/export?format=csv → 200 with CSV header.
|
|
resp = admin_client.get("/api/v1/admin/usage/export", params={"format": "csv"})
|
|
assert resp.status_code == 200
|
|
assert resp.headers["content-type"].startswith("text/csv")
|
|
reader = csv.DictReader(io.StringIO(resp.text))
|
|
rows = list(reader)
|
|
assert rows == []
|
|
# Header should still be present.
|
|
assert "timestamp" in resp.text
|
|
assert "user_id" in resp.text
|
|
assert "department_id" in resp.text
|