"""End-to-end admin workflow integration tests (U10). Verifies complete admin workflows end-to-end via the FastAPI TestClient: - Department lifecycle: create → update → bind skill → bind KB → disable → enable → unbind skill → delete. - User lifecycle: create → assign department → reset password → remove department → soft delete. - LLM config lifecycle: add provider → set API key → set fallback → set quota → list → delete provider. - Skill management flow: disable skill → verify excluded from ``GET /skills`` → enable → verify included. - Usage dashboard flow: query usage endpoints with empty data → verify empty results + CSV header. The tests mount a minimal FastAPI app with the ``admin_router`` (and the public ``skills`` router for the skill-management flow). The ``_require_admin`` dependency is overridden so the tests don't need real JWTs. """ from __future__ import annotations import csv import io import os import uuid from pathlib import Path from typing import Any import pytest import yaml from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from agentkit.server.admin.context import DepartmentContext from agentkit.server.admin.kb_service import set_kb_service from agentkit.server.admin.llm_config_service import ( LlmConfigService, set_llm_config_service, ) from agentkit.server.admin.skill_service import set_skill_service from agentkit.server.admin.usage_service import set_usage_service from agentkit.server.auth.models import init_auth_db from agentkit.server.auth.session_service import SessionService, set_session_service from agentkit.server.routes import admin as admin_routes_module from agentkit.server.routes import skills as skills_routes_module from agentkit.skills.registry import SkillRegistry # --------------------------------------------------------------------------- # Test data # --------------------------------------------------------------------------- _VALID_SKILL_YAML = """\ name: e2e_test_skill agent_type: simple_generation version: "1.0.0" description: "E2E test skill" task_mode: llm_generate execution_mode: direct max_steps: 1 prompt: identity: "E2E" instructions: "Handle test" tools: [] """ def _sample_agentkit_config() -> dict[str, Any]: """A minimal agentkit.yaml-style config for testing.""" return { "server": {"host": "0.0.0.0", "port": 8001}, "llm": { "providers": { "openai": { "type": "openai", "api_key": "sk-test-12345678", "base_url": "https://api.openai.com/v1", "models": {"gpt-4o": {}}, "max_tokens": 4096, "timeout": 120.0, }, }, "model_aliases": {"gpt4": "openai/gpt-4o"}, "fallbacks": {}, }, } # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest.fixture async def tmp_auth_db(tmp_path: Path) -> Path: db_path = tmp_path / "e2e_admin.db" await init_auth_db(db_path) return db_path @pytest.fixture def tmp_agentkit_yaml(tmp_path: Path) -> Path: """Create a temporary agentkit.yaml config file.""" path = tmp_path / "agentkit.yaml" with open(path, "w", encoding="utf-8") as f: yaml.dump(_sample_agentkit_config(), f, default_flow_style=False, allow_unicode=True) return path @pytest.fixture def tmp_skills_dir(tmp_path: Path) -> str: """A temp skills directory for YAML files.""" d = tmp_path / "skills" d.mkdir() return str(d) @pytest.fixture def skill_registry() -> SkillRegistry: return SkillRegistry() @pytest.fixture def session_service(tmp_auth_db: Path): """Install a SessionService singleton backed by the temp DB. Required so that ``UserService.reset_password`` can find the SessionService via ``get_session_service()`` and revoke sessions. """ svc = SessionService(db_path=tmp_auth_db) set_session_service(svc) yield svc set_session_service(None) @pytest.fixture(autouse=True) def _reset_singletons(): """Reset singletons before/after each test to avoid state leakage.""" set_llm_config_service(None) set_skill_service(None) set_kb_service(None) set_usage_service(None) yield set_llm_config_service(None) set_skill_service(None) set_kb_service(None) set_usage_service(None) def _make_admin_user() -> dict[str, Any]: return {"user_id": "admin-1", "username": "admin", "role": "admin"} def _raise_forbidden() -> dict[str, Any]: raise HTTPException(status_code=403, detail="Admin permission required") @pytest.fixture def admin_app( tmp_auth_db: Path, tmp_agentkit_yaml: Path, tmp_skills_dir: str, skill_registry: SkillRegistry, ) -> FastAPI: """A FastAPI app with admin + skills routers mounted. The ``_require_admin`` dependency is overridden to return a fake admin user. The :class:`LlmConfigService` singleton is pre-populated so the routes use the temp YAML file. """ set_llm_config_service(LlmConfigService(tmp_agentkit_yaml)) app = FastAPI() app.state.auth_db_path = str(tmp_auth_db) app.state.skill_registry = skill_registry class _FakeServerConfig: skill_paths = [tmp_skills_dir] app.state.server_config = _FakeServerConfig() app.include_router(admin_routes_module.admin_router, prefix="/api/v1") app.include_router(skills_routes_module.router, prefix="/api/v1") # Default: allow admin access. app.dependency_overrides[admin_routes_module._require_admin] = lambda: _make_admin_user() # Admin context for skills route (bypass department filtering). app.dependency_overrides[skills_routes_module.get_department_context] = lambda: ( DepartmentContext(user_id="admin-1", department_ids=[], is_admin=True) ) return app @pytest.fixture def admin_client( admin_app: FastAPI, session_service: SessionService, ) -> TestClient: """TestClient with admin access and SessionService installed.""" return TestClient(admin_app) # --------------------------------------------------------------------------- # Helper functions # --------------------------------------------------------------------------- def _create_department(client: TestClient, name: str, description: str = "") -> dict: resp = client.post( "/api/v1/admin/departments", json={"name": name, "description": description}, ) assert resp.status_code == 201, resp.text return resp.json() def _create_user( client: TestClient, *, username: str, email: str, password: str = "Secret123!", role: str = "member", department_ids: list[str] | None = None, ) -> dict: payload: dict[str, Any] = { "username": username, "email": email, "password": password, "role": role, } if department_ids is not None: payload["department_ids"] = department_ids resp = client.post("/api/v1/admin/users", json=payload) assert resp.status_code == 201, resp.text return resp.json() def _write_skill_yaml(skills_dir: str, name: str, content: str) -> str: path = os.path.join(skills_dir, f"{name}.yaml") with open(path, "w", encoding="utf-8") as f: f.write(content) return path # --------------------------------------------------------------------------- # E2E admin flow tests # --------------------------------------------------------------------------- class TestE2EAdminFlow: """End-to-end admin workflow test.""" def test_full_department_lifecycle( self, admin_client: TestClient, tmp_skills_dir: str, skill_registry: SkillRegistry, ): """Create department → update → bind skill → bind KB → disable → enable → unbind skill → delete.""" # 1. Create department "HR". hr = _create_department(admin_client, "HR", "Human Resources") dept_id = hr["id"] assert hr["name"] == "HR" assert hr["is_active"] is True # 2. GET /admin/departments → HR in list. resp = admin_client.get("/api/v1/admin/departments") assert resp.status_code == 200 names = {d["name"] for d in resp.json()} assert "HR" in names # 3. PATCH /admin/departments/{id} → update name to "Human Resources". resp = admin_client.patch( f"/api/v1/admin/departments/{dept_id}", json={"name": "Human Resources", "description": "HR department"}, ) assert resp.status_code == 200 assert resp.json()["name"] == "Human Resources" # 4. POST /admin/departments/{id}/skills/code_reviewer → bind skill. resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/skills/code_reviewer") assert resp.status_code == 201 assert resp.json()["skill_name"] == "code_reviewer" # 5. GET /admin/departments/{id}/skills → ["code_reviewer"]. resp = admin_client.get(f"/api/v1/admin/departments/{dept_id}/skills") assert resp.status_code == 200 assert resp.json() == ["code_reviewer"] # 6. POST /admin/departments/{id}/kb/source-1 → bind KB. resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/kb/source-1") assert resp.status_code == 201 assert resp.json()["kb_source_id"] == "source-1" # 7. GET /admin/departments/{id}/kb → ["source-1"]. resp = admin_client.get(f"/api/v1/admin/departments/{dept_id}/kb") assert resp.status_code == 200 assert resp.json() == ["source-1"] # 8. POST /admin/departments/{id}/disable → disabled. resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/disable") assert resp.status_code == 200 assert resp.json()["is_active"] is False # 9. POST /admin/departments/{id}/enable → enabled. resp = admin_client.post(f"/api/v1/admin/departments/{dept_id}/enable") assert resp.status_code == 200 assert resp.json()["is_active"] is True # 10. DELETE /admin/departments/{id}/skills/code_reviewer → unbound. resp = admin_client.delete(f"/api/v1/admin/departments/{dept_id}/skills/code_reviewer") assert resp.status_code == 200 assert resp.json() == {"unbound": True} assert admin_client.get(f"/api/v1/admin/departments/{dept_id}/skills").json() == [] # 11. DELETE /admin/departments/{id} → deleted. resp = admin_client.delete(f"/api/v1/admin/departments/{dept_id}") assert resp.status_code == 200 assert resp.json() == {"deleted": True} assert admin_client.get(f"/api/v1/admin/departments/{dept_id}").status_code == 404 def test_full_user_lifecycle(self, admin_client: TestClient): """Create user → assign department → reset password → remove department → delete.""" # 1. Create department "Engineering". eng = _create_department(admin_client, "Engineering") eng_id = eng["id"] # 2. POST /admin/users → create user "alice". alice = _create_user( admin_client, username="alice", email="alice@example.com", ) alice_id = alice["id"] assert alice["username"] == "alice" assert alice["departments"] == [] # 3. POST /admin/users/{id}/departments/{dept_id} → assign. resp = admin_client.post(f"/api/v1/admin/users/{alice_id}/departments/{eng_id}") assert resp.status_code == 201 assert resp.json() == {"assigned": True} # 4. GET /admin/users/{id} → has department. resp = admin_client.get(f"/api/v1/admin/users/{alice_id}") assert resp.status_code == 200 body = resp.json() assert len(body["departments"]) == 1 assert body["departments"][0]["name"] == "Engineering" # 5. POST /admin/users/{id}/reset-password → reset. resp = admin_client.post( f"/api/v1/admin/users/{alice_id}/reset-password", json={"new_password": "NewSecret456!"}, ) assert resp.status_code == 200 assert resp.json() == {"reset": True} # 6. DELETE /admin/users/{id}/departments/{dept_id} → remove. resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}/departments/{eng_id}") assert resp.status_code == 200 assert resp.json() == {"removed": True} # Verify removal. resp = admin_client.get(f"/api/v1/admin/users/{alice_id}") assert resp.status_code == 200 assert resp.json()["departments"] == [] # 7. DELETE /admin/users/{id} → soft delete. resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}") assert resp.status_code == 200 assert resp.json() == {"deleted": True} # Second delete on the now-inactive user returns 404. resp = admin_client.delete(f"/api/v1/admin/users/{alice_id}") assert resp.status_code == 404 def test_llm_config_lifecycle( self, admin_client: TestClient, tmp_agentkit_yaml: Path, ): """Add provider → set API key → set fallback → set quota → delete provider.""" # 1. POST /admin/llm/providers → create "test-provider". resp = admin_client.post( "/api/v1/admin/llm/providers", json={ "name": "test-provider", "type": "openai", "api_key": "sk-test-abcdef1234", "base_url": "https://api.test.com", "models": {"test-model": {}}, "max_tokens": 2048, "timeout": 60.0, }, ) assert resp.status_code == 201, resp.text assert resp.json()["name"] == "test-provider" assert resp.json()["api_key"].startswith("****") # 2. POST /admin/llm/providers/test-provider/api-key → set key. resp = admin_client.post( "/api/v1/admin/llm/providers/test-provider/api-key", json={"api_key": "sk-brand-new-key-9876"}, ) assert resp.status_code == 200 assert resp.json()["api_key"].startswith("****") # 3. PUT /admin/llm/fallbacks/gpt-4 → set fallback chain. resp = admin_client.put( "/api/v1/admin/llm/fallbacks/gpt-4", json={"chain": ["openai", "test-provider"]}, ) assert resp.status_code == 200 body = resp.json() assert body["model"] == "gpt-4" assert body["chain"] == ["openai", "test-provider"] # 4. PUT /admin/departments/{id}/quotas → set token_limit. dept_id = str(uuid.uuid4()) resp = admin_client.put( f"/api/v1/admin/departments/{dept_id}/quotas", json={ "quota_type": "token_limit", "limit_value": 1000, "period": "daily", }, ) assert resp.status_code == 200 body = resp.json() assert body["quota_type"] == "token_limit" assert body["limit_value"] == 1000 # 5. GET /admin/llm/providers → provider in list. resp = admin_client.get("/api/v1/admin/llm/providers") assert resp.status_code == 200 names = {p["name"] for p in resp.json()} assert "test-provider" in names assert "openai" in names # 6. DELETE /admin/llm/providers/test-provider → fails (used in fallback). resp = admin_client.delete("/api/v1/admin/llm/providers/test-provider") assert resp.status_code == 400 assert "fallback" in resp.json()["detail"].lower() # Remove the fallback chain first, then delete. admin_client.delete("/api/v1/admin/llm/fallbacks/gpt-4") resp = admin_client.delete("/api/v1/admin/llm/providers/test-provider") assert resp.status_code == 200 assert resp.json() == {"deleted": True} # Confirm it's gone. resp = admin_client.get("/api/v1/admin/llm/providers") names = {p["name"] for p in resp.json()} assert "test-provider" not in names def test_skill_management_flow( self, admin_client: TestClient, tmp_skills_dir: str, skill_registry: SkillRegistry, ): """Disable skill → verify excluded → enable → verify included.""" # 1. Create a test skill YAML in tmp_skills_dir and register it. _write_skill_yaml(tmp_skills_dir, "e2e_test_skill", _VALID_SKILL_YAML) from agentkit.skills.loader import SkillLoader SkillLoader(skill_registry=skill_registry).load_from_file( os.path.join(tmp_skills_dir, "e2e_test_skill.yaml") ) # Verify the skill is initially listed. resp = admin_client.get("/api/v1/skills") assert resp.status_code == 200 names = [s["name"] for s in resp.json()] assert "e2e_test_skill" in names # 2. POST /admin/skills/{name}/disable → disabled. resp = admin_client.post("/api/v1/admin/skills/e2e_test_skill/disable") assert resp.status_code == 200, resp.text body = resp.json() assert body["skill_name"] == "e2e_test_skill" assert body["is_disabled"] is True # 3. Verify via GET /skills that the skill is excluded. resp = admin_client.get("/api/v1/skills") assert resp.status_code == 200 names = [s["name"] for s in resp.json()] assert "e2e_test_skill" not in names # 4. POST /admin/skills/{name}/enable → enabled. resp = admin_client.post("/api/v1/admin/skills/e2e_test_skill/enable") assert resp.status_code == 200 assert resp.json()["enabled"] is True # Verify via GET /skills that the skill is included again. resp = admin_client.get("/api/v1/skills") assert resp.status_code == 200 names = [s["name"] for s in resp.json()] assert "e2e_test_skill" in names def test_usage_dashboard_flow(self, admin_client: TestClient): """Query usage endpoints with empty data → verify empty results.""" # The admin_app fixture doesn't install an llm_gateway on app.state, # so the usage routes will return 500. We need to install a stub # gateway with an empty InMemoryUsageStore. from agentkit.llm.providers.usage_store import InMemoryUsageStore class _StubTracker: def __init__(self, store): self.store = store class _StubGateway: def __init__(self, store): self._usage_tracker = _StubTracker(store) store = InMemoryUsageStore() # Install the stub gateway on the app state. admin_client.app.state.llm_gateway = _StubGateway(store) # 1. GET /admin/usage/summary → 200 with zeros. resp = admin_client.get("/api/v1/admin/usage/summary") assert resp.status_code == 200 body = resp.json() assert body["total_tokens"] == 0 assert body["total_cost"] == 0.0 assert body["total_requests"] == 0 # 2. GET /admin/usage/timeseries → 200 with empty list. resp = admin_client.get("/api/v1/admin/usage/timeseries") assert resp.status_code == 200 assert resp.json() == [] # 3. GET /admin/usage/by-model → 200 with empty list. resp = admin_client.get("/api/v1/admin/usage/by-model") assert resp.status_code == 200 assert resp.json() == [] # 4. GET /admin/usage/top-users → 200 with empty list. resp = admin_client.get("/api/v1/admin/usage/top-users") assert resp.status_code == 200 assert resp.json() == [] # 5. GET /admin/usage/export?format=csv → 200 with CSV header. resp = admin_client.get("/api/v1/admin/usage/export", params={"format": "csv"}) assert resp.status_code == 200 assert resp.headers["content-type"].startswith("text/csv") reader = csv.DictReader(io.StringIO(resp.text)) rows = list(reader) assert rows == [] # Header should still be present. assert "timestamp" in resp.text assert "user_id" in resp.text assert "department_id" in resp.text