476 lines
14 KiB
Python
476 lines
14 KiB
Python
"""Integration tests for Expert Team Mode.
|
|
|
|
Covers Key Flows F1-F6 and Acceptance Examples AE1-AE5 from the requirements document.
|
|
Uses mocked AgentPool and LLM calls to test orchestration logic.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
|
|
from agentkit.experts.config import ExpertConfig, ExpertTemplate
|
|
from agentkit.experts.plan import (
|
|
CollaborationPlan,
|
|
MergeStrategy,
|
|
ParallelType,
|
|
PhaseStatus,
|
|
PlanPhase,
|
|
PlanStatus,
|
|
)
|
|
from agentkit.experts.team import TeamStatus
|
|
from agentkit.experts.router import ExpertTeamRouter
|
|
from agentkit.experts.registry import ExpertTemplateRegistry
|
|
from agentkit.core.handoff_transport import InProcessHandoffTransport
|
|
from agentkit.core.shared_workspace import SharedWorkspace
|
|
|
|
|
|
# --- Helpers ---
|
|
|
|
|
|
def make_expert_config(
|
|
name: str,
|
|
persona: str = "",
|
|
is_lead: bool = False,
|
|
bound_skills: list[str] | None = None,
|
|
) -> ExpertConfig:
|
|
"""Helper to create ExpertConfig for testing."""
|
|
return ExpertConfig(
|
|
name=name,
|
|
persona=persona,
|
|
is_lead=is_lead,
|
|
bound_skills=bound_skills or [],
|
|
agent_type="expert",
|
|
task_mode="llm_generate",
|
|
prompt={"identity": f"You are {name}, {persona}"} if persona else {"identity": f"You are {name}"},
|
|
)
|
|
|
|
|
|
# --- Fixtures ---
|
|
|
|
|
|
@pytest.fixture
|
|
def workspace():
|
|
return SharedWorkspace()
|
|
|
|
|
|
@pytest.fixture
|
|
def registry():
|
|
reg = ExpertTemplateRegistry()
|
|
reg.register(
|
|
ExpertTemplate(
|
|
name="analyst",
|
|
description="Data Analyst",
|
|
config=make_expert_config(
|
|
"analyst", "数据分析专家", bound_skills=["data_analysis"]
|
|
),
|
|
)
|
|
)
|
|
reg.register(
|
|
ExpertTemplate(
|
|
name="strategist",
|
|
description="Strategy Consultant",
|
|
config=make_expert_config(
|
|
"strategist", "战略顾问", bound_skills=["strategy_planning"]
|
|
),
|
|
)
|
|
)
|
|
reg.register(
|
|
ExpertTemplate(
|
|
name="architect",
|
|
description="Software Architect",
|
|
config=make_expert_config(
|
|
"architect", "软件架构师", bound_skills=["system_design"]
|
|
),
|
|
)
|
|
)
|
|
return reg
|
|
|
|
|
|
# --- F1: Manual Team Formation ---
|
|
|
|
|
|
class TestManualTeamFormation:
|
|
"""Covers F1: User specifies expert team members."""
|
|
|
|
async def test_manual_team_with_templates(self, registry):
|
|
"""AE1: User specifies expert team by template names."""
|
|
router = ExpertTeamRouter(registry)
|
|
result = router.resolve("@team:analyst,strategist 分析这份市场报告")
|
|
|
|
assert result.team_mode is True
|
|
assert result.specified_experts == ["analyst", "strategist"]
|
|
assert result.auto_compose is False
|
|
|
|
# Resolve to configs
|
|
configs = router.resolve_expert_configs(result.specified_experts)
|
|
assert len(configs) == 2
|
|
assert configs[0].name == "analyst"
|
|
assert configs[1].name == "strategist"
|
|
|
|
async def test_manual_team_subtask_parallel(self):
|
|
"""AE1: Subtask-level parallel execution after manual team formation."""
|
|
plan = CollaborationPlan(
|
|
id="plan-1",
|
|
task="分析市场报告",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="数据分析",
|
|
assigned_expert="analyst",
|
|
task_description="执行数据分析",
|
|
parallel_type=ParallelType.SUBTASK_PARALLEL,
|
|
),
|
|
PlanPhase(
|
|
id="p2",
|
|
name="战略建议",
|
|
assigned_expert="strategist",
|
|
task_description="提供战略建议",
|
|
parallel_type=ParallelType.SUBTASK_PARALLEL,
|
|
),
|
|
],
|
|
)
|
|
|
|
errors = plan.validate()
|
|
assert errors == []
|
|
|
|
|
|
# --- F2: Auto Team Formation ---
|
|
|
|
|
|
class TestAutoTeamFormation:
|
|
"""Covers F2: System auto-composes expert team."""
|
|
|
|
async def test_auto_team_high_complexity(self, registry):
|
|
"""AE2: High complexity triggers team mode suggestion."""
|
|
router = ExpertTeamRouter(registry)
|
|
result = router.resolve("评审这个复杂的技术方案", complexity=0.85)
|
|
|
|
assert result.team_mode is True
|
|
assert result.auto_compose is True
|
|
assert result.match_method == "complexity_suggestion"
|
|
|
|
async def test_auto_team_competitive_parallel(self):
|
|
"""AE2: Competitive parallel with BEST strategy."""
|
|
plan = CollaborationPlan(
|
|
id="plan-2",
|
|
task="技术方案评审",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="架构方案A",
|
|
assigned_expert="architect_a",
|
|
task_description="设计架构方案A",
|
|
parallel_type=ParallelType.COMPETITIVE_PARALLEL,
|
|
merge_strategy=MergeStrategy.BEST,
|
|
),
|
|
PlanPhase(
|
|
id="p2",
|
|
name="架构方案B",
|
|
assigned_expert="architect_b",
|
|
task_description="设计架构方案B",
|
|
parallel_type=ParallelType.COMPETITIVE_PARALLEL,
|
|
merge_strategy=MergeStrategy.BEST,
|
|
),
|
|
],
|
|
)
|
|
|
|
errors = plan.validate()
|
|
assert errors == []
|
|
|
|
|
|
# --- F3: Decentralized Collaboration ---
|
|
|
|
|
|
class TestDecentralizedCollaboration:
|
|
"""Covers F3: Experts collaborate directly without Lead mediation."""
|
|
|
|
async def test_expert_direct_handoff(self):
|
|
"""AE4: Expert A requests assistance from Expert B directly."""
|
|
transport = InProcessHandoffTransport()
|
|
channel = "expert:analyst:handoff"
|
|
|
|
# Start listening first (consumer must be registered before send)
|
|
messages = []
|
|
|
|
async def listener():
|
|
async for msg in transport.listen(channel):
|
|
messages.append(msg)
|
|
break
|
|
|
|
task = asyncio.create_task(listener())
|
|
await asyncio.sleep(0.05)
|
|
|
|
# Expert A sends assist request
|
|
await transport.send(
|
|
channel,
|
|
{
|
|
"source_expert": "analyst",
|
|
"target_expert": "researcher",
|
|
"task": "需要行业数据",
|
|
"type": "assist_request",
|
|
},
|
|
)
|
|
|
|
await asyncio.wait_for(task, timeout=2.0)
|
|
|
|
assert len(messages) == 1
|
|
assert messages[0]["source_expert"] == "analyst"
|
|
assert messages[0]["type"] == "assist_request"
|
|
|
|
transport.close()
|
|
|
|
async def test_team_channel_broadcast(self):
|
|
"""All experts receive team channel messages."""
|
|
transport = InProcessHandoffTransport()
|
|
channel = "team:test-team"
|
|
|
|
# Two consumers listening
|
|
consumer1_msgs = []
|
|
consumer2_msgs = []
|
|
|
|
async def consumer1():
|
|
async for msg in transport.listen(channel):
|
|
consumer1_msgs.append(msg)
|
|
if len(consumer1_msgs) >= 1:
|
|
break
|
|
|
|
async def consumer2():
|
|
async for msg in transport.listen(channel):
|
|
consumer2_msgs.append(msg)
|
|
if len(consumer2_msgs) >= 1:
|
|
break
|
|
|
|
# Start consumers
|
|
t1 = asyncio.create_task(consumer1())
|
|
t2 = asyncio.create_task(consumer2())
|
|
|
|
await asyncio.sleep(0.05)
|
|
|
|
# Send message
|
|
await transport.send(channel, {"type": "chat", "content": "hello"})
|
|
|
|
await asyncio.wait_for(asyncio.gather(t1, t2), timeout=2.0)
|
|
|
|
assert len(consumer1_msgs) == 1
|
|
assert len(consumer2_msgs) == 1
|
|
|
|
transport.close()
|
|
|
|
|
|
# --- F4: User Intervention ---
|
|
|
|
|
|
class TestUserIntervention:
|
|
"""Covers F4: User intervenes during collaboration."""
|
|
|
|
async def test_user_intervention_broadcast(self):
|
|
"""AE3: User intervention message reaches all experts."""
|
|
transport = InProcessHandoffTransport()
|
|
channel = "team:intervention-test"
|
|
|
|
received = []
|
|
|
|
async def listener():
|
|
async for msg in transport.listen(channel):
|
|
received.append(msg)
|
|
if len(received) >= 1:
|
|
break
|
|
|
|
task = asyncio.create_task(listener())
|
|
await asyncio.sleep(0.05)
|
|
|
|
await transport.send(
|
|
channel,
|
|
{"type": "user_intervention", "content": "重点看成本优化"},
|
|
)
|
|
|
|
await asyncio.wait_for(task, timeout=2.0)
|
|
|
|
assert len(received) == 1
|
|
assert received[0]["type"] == "user_intervention"
|
|
|
|
transport.close()
|
|
|
|
async def test_plan_modification_by_user(self):
|
|
"""User can modify the collaboration plan."""
|
|
plan = CollaborationPlan(
|
|
id="plan-3",
|
|
task="分析报告",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="数据分析",
|
|
assigned_expert="analyst",
|
|
task_description="执行数据分析",
|
|
),
|
|
],
|
|
)
|
|
|
|
# User modifies plan — add a new phase
|
|
plan.phases.append(
|
|
PlanPhase(
|
|
id="p2",
|
|
name="成本优化",
|
|
assigned_expert="cost_analyst",
|
|
task_description="优化成本",
|
|
depends_on=["p1"],
|
|
)
|
|
)
|
|
|
|
errors = plan.validate()
|
|
assert errors == []
|
|
assert len(plan.phases) == 2
|
|
|
|
|
|
# --- F5: Competitive Parallel ---
|
|
|
|
|
|
class TestCompetitiveParallel:
|
|
"""Covers F5: Competitive parallel execution with merge strategies."""
|
|
|
|
async def test_vote_strategy_with_tie_break(self):
|
|
"""R23: Vote strategy with Lead Expert tie-breaking."""
|
|
plan = CollaborationPlan(
|
|
id="plan-4",
|
|
task="方案评审",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="方案竞争",
|
|
assigned_expert="architect",
|
|
task_description="竞争性方案设计",
|
|
parallel_type=ParallelType.COMPETITIVE_PARALLEL,
|
|
merge_strategy=MergeStrategy.VOTE,
|
|
),
|
|
],
|
|
)
|
|
|
|
errors = plan.validate()
|
|
assert errors == []
|
|
assert plan.phases[0].merge_strategy == MergeStrategy.VOTE
|
|
|
|
async def test_fusion_strategy(self):
|
|
"""R24: Fusion strategy merges multiple results."""
|
|
plan = CollaborationPlan(
|
|
id="plan-5",
|
|
task="方案融合",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="方案融合",
|
|
assigned_expert="lead",
|
|
task_description="融合多个方案",
|
|
parallel_type=ParallelType.COMPETITIVE_PARALLEL,
|
|
merge_strategy=MergeStrategy.FUSION,
|
|
),
|
|
],
|
|
)
|
|
|
|
errors = plan.validate()
|
|
assert errors == []
|
|
|
|
|
|
# --- F6: Team Dissolution ---
|
|
|
|
|
|
class TestTeamDissolution:
|
|
"""Covers F6: Team dissolution and output preservation."""
|
|
|
|
async def test_dissolution_preserves_outputs(self, workspace):
|
|
"""R36: Temporary Expert outputs preserved in SharedWorkspace after dissolution."""
|
|
# Write some output to workspace
|
|
await workspace.write(
|
|
"team:test:analyst:result", {"report": "analysis result"}, "analyst"
|
|
)
|
|
|
|
# Verify output exists
|
|
data = await workspace.read("team:test:analyst:result")
|
|
assert data is not None
|
|
assert data["value"]["report"] == "analysis result"
|
|
|
|
async def test_dissolution_sets_status(self):
|
|
"""Team status becomes DISSOLVED after dissolution."""
|
|
assert TeamStatus.DISSOLVED == "dissolved"
|
|
|
|
|
|
# --- Retry and Fallback ---
|
|
|
|
|
|
class TestRetryAndFallback:
|
|
"""Tests retry + fallback degradation strategy."""
|
|
|
|
async def test_plan_failure_triggers_retry(self):
|
|
"""Failed phase triggers retry before fallback."""
|
|
plan = CollaborationPlan(
|
|
id="plan-6",
|
|
task="测试任务",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="阶段1",
|
|
assigned_expert="expert1",
|
|
task_description="执行阶段1",
|
|
),
|
|
],
|
|
)
|
|
|
|
# Simulate failure
|
|
plan.update_phase_status("p1", PhaseStatus.FAILED)
|
|
assert plan.phases[0].status == PhaseStatus.FAILED
|
|
|
|
# Reset for retry
|
|
plan.update_phase_status("p1", PhaseStatus.PENDING)
|
|
assert plan.phases[0].status == PhaseStatus.PENDING
|
|
|
|
async def test_fallback_after_retry_failure(self):
|
|
"""After retry still fails, fallback to single agent."""
|
|
plan = CollaborationPlan(
|
|
id="plan-7",
|
|
task="测试任务",
|
|
lead_expert="lead",
|
|
phases=[
|
|
PlanPhase(
|
|
id="p1",
|
|
name="阶段1",
|
|
assigned_expert="expert1",
|
|
task_description="执行阶段1",
|
|
)
|
|
],
|
|
)
|
|
|
|
# Mark as fallback
|
|
plan.status = PlanStatus.FALLBACK
|
|
assert plan.status == PlanStatus.FALLBACK
|
|
|
|
|
|
# --- Dynamic Expert Addition/Removal ---
|
|
|
|
|
|
class TestDynamicExpertManagement:
|
|
"""AE5: Dynamic addition and removal of experts."""
|
|
|
|
async def test_add_expert_by_template(self, registry):
|
|
"""Add expert by template name."""
|
|
router = ExpertTeamRouter(registry)
|
|
result = router.resolve("@team:analyst 分析报告")
|
|
|
|
configs = router.resolve_expert_configs(result.specified_experts)
|
|
assert len(configs) == 1
|
|
assert configs[0].name == "analyst"
|
|
assert configs[0].bound_skills == ["data_analysis"]
|
|
|
|
async def test_add_expert_dynamic(self, registry):
|
|
"""Add expert with non-existent template creates dynamic config."""
|
|
router = ExpertTeamRouter(registry)
|
|
configs = router.resolve_expert_configs(["legal_advisor"])
|
|
|
|
assert len(configs) == 1
|
|
assert configs[0].name == "legal_advisor"
|
|
assert "legal_advisor" in configs[0].persona
|