"""Agent Capability Benchmark — Ground Truth Dataset (v2). Aligned with actual skills in configs/skills/*.yaml. Contains both manually curated edge cases and auto-generated cases. Categories: - routing: intent routing correctness - execution: execution mode selection accuracy - team: expert team collaboration - consistency: deterministic output consistency - semantic_router: semantic similarity matching - alignment: constraint compliance and cascade detection """ from pydantic import BaseModel, ConfigDict class BenchmarkCase(BaseModel): """A single benchmark test case with ground truth label.""" model_config = ConfigDict(frozen=True) id: str input: str expected_skill: str | None = None expected_execution_mode: str = "direct" expected_complexity: str = "low" category: str subcategory: str paraphrases: list[str] = [] tags: list[str] = [] # ═══════════════════════════════════════════════════════════════════════════ # Routing — Keyword Match (aligned with actual skills) # ═══════════════════════════════════════════════════════════════════════════ ROUTING_KEYWORD_BENCHMARKS: list[BenchmarkCase] = [ # direct_agent BenchmarkCase( id="route-kw-direct-001", input="翻译这段话", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["帮我翻译一下", "请翻译这段内容", "Translate this text"], tags=["翻译", "translate"], ), BenchmarkCase( id="route-kw-direct-002", input="帮我总结一下", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["请总结", "给我一个摘要", "Summarize this"], tags=["摘要", "summarize"], ), BenchmarkCase( id="route-kw-direct-003", input="什么是RAG？", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["RAG是什么", "解释一下RAG", "What is RAG?"], tags=["什么是"], ), # react_agent BenchmarkCase( id="route-kw-react-001", input="搜索一下AI Agent市场数据", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=[ "帮我搜索AI Agent市场信息", "查找AI Agent的市场数据", "Search AI Agent market data", ], tags=["搜索", "search"], ), BenchmarkCase( id="route-kw-react-002", input="帮我分析这个数据", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["分析一下这些数据", "请对数据做分析", "Analyze this data"], tags=["分析", "analyze"], ), BenchmarkCase( id="route-kw-react-003", input="实时监控竞品动态", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["监控竞争对手的动态", "实时追踪竞品变化", "Monitor competitor activities"], tags=["实时", "监控"], ), # rewoo_agent BenchmarkCase( id="route-kw-rewoo-001", input="采集A、B、C三个竞品的功能数据", expected_skill="rewoo_agent", expected_execution_mode="rewoo", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=[ "批量采集竞品数据", "并行获取多个竞品信息", "Fetch data from multiple competitors", ], tags=["采集", "批量", "fetch"], ), BenchmarkCase( id="route-kw-rewoo-002", input="并行搜索多个关键词", expected_skill="rewoo_agent", expected_execution_mode="rewoo", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["同时搜索多个关键词", "批量搜索", "Search multiple keywords in parallel"], tags=["并行", "批量"], ), # reflexion_agent BenchmarkCase( id="route-kw-reflex-001", input="审查这段代码的合规性", expected_skill="reflexion_agent", expected_execution_mode="reflexion", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["检查代码是否合规", "审查代码合规问题", "Review code compliance"], tags=["审查", "合规", "review"], ), BenchmarkCase( id="route-kw-reflex-002", input="生成一个高精度的数据分析脚本", expected_skill="reflexion_agent", expected_execution_mode="reflexion", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=[ "写一个精确的数据分析脚本", "生成高精度分析代码", "Generate a precise analysis script", ], tags=["代码生成", "精确", "code"], ), # plan_exec_agent BenchmarkCase( id="route-kw-planexec-001", input="生成一份市场分析报告", expected_skill="plan_exec_agent", expected_execution_mode="plan_exec", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["做一份市场分析报告", "写个市场分析报告", "Generate a market analysis report"], tags=["报告", "分析报告"], ), BenchmarkCase( id="route-kw-planexec-002", input="规划产品优化方案", expected_skill="plan_exec_agent", expected_execution_mode="plan_exec", expected_complexity="high", category="routing", subcategory="keyword_match", paraphrases=["制定产品优化计划", "帮我规划产品优化", "Plan product optimization"], tags=["规划", "plan"], ), # code_reviewer BenchmarkCase( id="route-kw-coderev-001", input="Review this code for quality", expected_skill="code_reviewer", expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["审查这段代码的质量", "代码审查", "Check code quality"], tags=["review", "代码审查"], ), # geo_optimizer BenchmarkCase( id="route-kw-geo-001", input="帮我优化这篇文章的SEO", expected_skill="geo_optimizer", expected_execution_mode="llm_generate", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["SEO优化一下", "提升文章搜索排名", "Optimize this article for SEO"], tags=["SEO优化", "optimize"], ), # deai_agent BenchmarkCase( id="route-kw-deai-001", input="帮我把这篇文章去AI化", expected_skill="deai_agent", expected_execution_mode="llm_generate", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["让这段文字更自然", "改写得像人写的", "Make this text more natural"], tags=["去AI化", "人性化"], ), # content_generator BenchmarkCase( id="route-kw-content-001", input="帮我写一篇关于AI的文章", expected_skill="content_generator", expected_execution_mode="llm_generate", expected_complexity="low", category="routing", subcategory="keyword_match", paraphrases=["写一篇AI相关文章", "生成关于AI的内容", "Write an article about AI"], tags=["写文章", "generate"], ), # citation_detector BenchmarkCase( id="route-kw-citation-001", input="检测我们的品牌在AI平台的引用情况", expected_skill="citation_detector", expected_execution_mode="custom", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=[ "分析品牌引用率", "哪些AI平台引用了我们", "Check brand citation on AI platforms", ], tags=["引用检测", "citation"], ), # trend_agent BenchmarkCase( id="route-kw-trend-001", input="分析品牌趋势", expected_skill="trend_agent", expected_execution_mode="tool_call", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=["最近的热点话题是什么", "趋势洞察", "Analyze brand trends"], tags=["趋势", "trend"], ), # competitor_analyzer BenchmarkCase( id="route-kw-competitor-001", input="分析我的竞品策略", expected_skill="competitor_analyzer", expected_execution_mode="tool_call", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=["对比我和竞品的差距", "竞品分析", "Analyze competitor strategies"], tags=["竞品", "competitor"], ), # schema_advisor BenchmarkCase( id="route-kw-schema-001", input="帮我优化Schema", expected_skill="schema_advisor", expected_execution_mode="custom", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=["生成JSON-LD结构化数据", "Schema有什么可以改进的", "Optimize my Schema"], tags=["Schema", "schema优化"], ), # monitor BenchmarkCase( id="route-kw-monitor-001", input="监测品牌引用变化", expected_skill="monitor", expected_execution_mode="custom", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=["追踪效果", "品牌排名变化", "Monitor brand citation changes"], tags=["监测", "monitor"], ), # goal_driven_agent BenchmarkCase( id="route-kw-goal-001", input="分析竞品SEO策略并生成优化方案", expected_skill="goal_driven_agent", expected_execution_mode="tool_call", expected_complexity="medium", category="routing", subcategory="keyword_match", paraphrases=[ "调研技术方案并生成对比报告", "制定市场推广计划", "Analyze SEO and generate plan", ], tags=["分析", "优化方案"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Routing — Edge Cases (manually curated) # ═══════════════════════════════════════════════════════════════════════════ ROUTING_EDGE_BENCHMARKS: list[BenchmarkCase] = [ # Greeting (should NOT route to any skill) BenchmarkCase( id="route-edge-greet-001", input="你好", expected_skill=None, expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="greeting", paraphrases=["Hello", "Hi there", "早上好"], tags=["greeting"], ), BenchmarkCase( id="route-edge-greet-002", input="Good morning!", expected_skill=None, expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="greeting", paraphrases=["早上好！", "你好呀"], tags=["greeting"], ), # Identity (should NOT route to any skill) BenchmarkCase( id="route-edge-identity-001", input="你是谁？", expected_skill=None, expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="identity", paraphrases=["What is your name?", "介绍一下你自己", "Tell me about yourself"], tags=["identity"], ), # Explicit prefix BenchmarkCase( id="route-edge-explicit-001", input="@skill:react_agent 搜索最新的AI新闻", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="routing", subcategory="explicit_prefix", paraphrases=["@skill:react_agent 查找AI最新动态"], tags=["explicit", "react"], ), # Fallback (no matching skill) BenchmarkCase( id="route-edge-fallback-001", input="告诉我一个笑话", expected_skill=None, expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="fallback", paraphrases=["讲个笑话", "Tell me a joke", "说个搞笑的"], tags=["fallback"], ), BenchmarkCase( id="route-edge-fallback-002", input="What is quantum physics?", expected_skill=None, expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="fallback", paraphrases=["量子物理是什么", "Explain quantum mechanics"], tags=["fallback"], ), # Disambiguation (multiple skills could match) BenchmarkCase( id="route-edge-disambig-001", input="审查代码并优化SEO", expected_skill="code_reviewer", expected_execution_mode="direct", expected_complexity="low", category="routing", subcategory="disambiguation", paraphrases=["Review code and optimize SEO", "代码审查加SEO优化"], tags=["disambiguation", "review", "seo"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Execution Mode Benchmarks # ═══════════════════════════════════════════════════════════════════════════ EXECUTION_BENCHMARKS: list[BenchmarkCase] = [ BenchmarkCase( id="exec-direct-001", input="翻译这段话成英文", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="execution", subcategory="direct_mode", paraphrases=["Translate this to English", "把这段翻成英语"], tags=["direct", "simple"], ), BenchmarkCase( id="exec-direct-002", input="什么是AgentKit？", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="execution", subcategory="direct_mode", paraphrases=["AgentKit是什么", "Explain AgentKit"], tags=["direct", "qa"], ), BenchmarkCase( id="exec-react-001", input="搜索并分析AI行业最新趋势", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="execution", subcategory="react_mode", paraphrases=["Search and analyze AI trends", "调研AI行业趋势"], tags=["react", "multi_step"], ), BenchmarkCase( id="exec-react-002", input="实时监控竞品动态并生成报告", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="execution", subcategory="react_mode", paraphrases=["Monitor competitors and report", "追踪竞品并输出报告"], tags=["react", "monitoring"], ), BenchmarkCase( id="exec-rewoo-001", input="批量采集多个竞品的功能数据", expected_skill="rewoo_agent", expected_execution_mode="rewoo", expected_complexity="high", category="execution", subcategory="rewoo_mode", paraphrases=["并行获取竞品数据", "Fetch competitor data in parallel"], tags=["rewoo", "parallel"], ), BenchmarkCase( id="exec-reflexion-001", input="审查代码合规性并确保高精度", expected_skill="reflexion_agent", expected_execution_mode="reflexion", expected_complexity="high", category="execution", subcategory="reflexion_mode", paraphrases=["高精度代码审查", "Precise code compliance review"], tags=["reflexion", "precision"], ), BenchmarkCase( id="exec-planexec-001", input="生成一份完整的市场调研报告", expected_skill="plan_exec_agent", expected_execution_mode="plan_exec", expected_complexity="high", category="execution", subcategory="plan_exec_mode", paraphrases=["做一份市场调研报告", "Generate a market research report"], tags=["plan_exec", "report"], ), BenchmarkCase( id="exec-quality-001", input="生成内容并确保质量达标", expected_skill="content_generator", expected_execution_mode="llm_generate", expected_complexity="low", category="execution", subcategory="quality_gate", paraphrases=["生成高质量内容", "Generate quality content"], tags=["quality", "content"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Team Collaboration Benchmarks # ═══════════════════════════════════════════════════════════════════════════ TEAM_BENCHMARKS: list[BenchmarkCase] = [ BenchmarkCase( id="team-explicit-001", input="@team:react_agent,plan_exec_agent 协作完成深度分析并生成报告", expected_execution_mode="react", expected_complexity="high", category="team", subcategory="explicit_team", paraphrases=[ "需要react_agent和plan_exec_agent协作", "组建团队：搜索分析+报告生成", ], tags=["team", "explicit"], ), BenchmarkCase( id="team-explicit-002", input="@team:competitor_analyzer,trend_agent 分析竞品并追踪趋势", expected_execution_mode="react", expected_complexity="high", category="team", subcategory="explicit_team", paraphrases=["竞品分析+趋势追踪团队", "Team for competitor and trend analysis"], tags=["team", "explicit"], ), BenchmarkCase( id="team-complexity-001", input="深度分析竞品策略、追踪品牌趋势并生成优化方案", expected_execution_mode="react", expected_complexity="high", category="team", subcategory="complexity_trigger", paraphrases=[ "全面竞品分析和优化方案", "Comprehensive competitor analysis with optimization", ], tags=["team", "complexity"], ), BenchmarkCase( id="team-fallback-001", input="复杂任务但无匹配专家", expected_execution_mode="react", expected_complexity="high", category="team", subcategory="fallback", paraphrases=["需要团队但找不到合适专家", "Complex task without matching experts"], tags=["team", "fallback"], ), BenchmarkCase( id="team-name-valid-001", input="@team:react_agent,plan_exec_agent", expected_execution_mode="react", expected_complexity="high", category="team", subcategory="name_validation", tags=["team", "validation"], ), BenchmarkCase( id="team-name-invalid-001", input="@team:invalid expert name", expected_execution_mode="direct", expected_complexity="low", category="team", subcategory="name_validation", tags=["team", "validation", "invalid"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Consistency Benchmarks # ═══════════════════════════════════════════════════════════════════════════ CONSISTENCY_BENCHMARKS: list[BenchmarkCase] = [ BenchmarkCase( id="consist-direct-001", input="翻译'hello world'成中文", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="consistency", subcategory="deterministic", tags=["consistency", "translation"], ), BenchmarkCase( id="consist-direct-002", input="什么是RAG？", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="consistency", subcategory="deterministic", tags=["consistency", "qa"], ), BenchmarkCase( id="consist-react-001", input="搜索AI Agent市场数据", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="consistency", subcategory="deterministic", tags=["consistency", "search"], ), BenchmarkCase( id="consist-geo-001", input="帮我优化这篇文章的SEO", expected_skill="geo_optimizer", expected_execution_mode="llm_generate", expected_complexity="low", category="consistency", subcategory="deterministic", tags=["consistency", "seo"], ), BenchmarkCase( id="consist-deai-001", input="帮我把这篇文章去AI化", expected_skill="deai_agent", expected_execution_mode="llm_generate", expected_complexity="low", category="consistency", subcategory="deterministic", tags=["consistency", "deai"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Semantic Router Benchmarks # ═══════════════════════════════════════════════════════════════════════════ SEMANTIC_ROUTER_BENCHMARKS: list[BenchmarkCase] = [ BenchmarkCase( id="semantic-direct-001", input="简单生成任务，无需工具调用", expected_skill="direct_agent", expected_execution_mode="direct", expected_complexity="low", category="semantic_router", subcategory="description_match", paraphrases=["只需要一次生成的简单任务", "Single LLM call task"], tags=["semantic", "direct"], ), BenchmarkCase( id="semantic-react-001", input="需要动态适应、逐步推理和工具调用", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="semantic_router", subcategory="description_match", paraphrases=["需要多步推理和工具", "Multi-step reasoning with tools"], tags=["semantic", "react"], ), BenchmarkCase( id="semantic-rewoo-001", input="多源数据并行采集、无依赖工具调用批量执行", expected_skill="rewoo_agent", expected_execution_mode="rewoo", expected_complexity="high", category="semantic_router", subcategory="description_match", paraphrases=["并行批量获取数据", "Parallel data collection"], tags=["semantic", "rewoo"], ), BenchmarkCase( id="semantic-reflex-001", input="需要高精度和自我验证的任务", expected_skill="reflexion_agent", expected_execution_mode="reflexion", expected_complexity="high", category="semantic_router", subcategory="description_match", paraphrases=["需要自我检查的高精度任务", "High-precision self-verification task"], tags=["semantic", "reflexion"], ), BenchmarkCase( id="semantic-planexec-001", input="结构化多步骤任务，需要可审查的规划和执行", expected_skill="plan_exec_agent", expected_execution_mode="plan_exec", expected_complexity="high", category="semantic_router", subcategory="description_match", paraphrases=["需要先规划再执行的任务", "Structured planning and execution"], tags=["semantic", "plan_exec"], ), BenchmarkCase( id="semantic-geo-001", input="对文章进行GEO/SEO优化，提升在AI搜索引擎中的可见性", expected_skill="geo_optimizer", expected_execution_mode="llm_generate", expected_complexity="low", category="semantic_router", subcategory="description_match", paraphrases=["提升内容搜索排名", "Improve content visibility in AI search"], tags=["semantic", "geo"], ), BenchmarkCase( id="semantic-citation-001", input="检测品牌在各AI平台回答中的引用情况", expected_skill="citation_detector", expected_execution_mode="custom", expected_complexity="medium", category="semantic_router", subcategory="description_match", paraphrases=["分析品牌被AI引用的情况", "Check brand citation across AI platforms"], tags=["semantic", "citation"], ), BenchmarkCase( id="semantic-competitor-001", input="分析竞品策略、对比品牌差距或发现竞争机会", expected_skill="competitor_analyzer", expected_execution_mode="tool_call", expected_complexity="medium", category="semantic_router", subcategory="description_match", paraphrases=["竞品对比和差距分析", "Competitive gap analysis"], tags=["semantic", "competitor"], ), # --- Colloquial / casual expressions (口语化表达) --- BenchmarkCase( id="semantic-colloquial-review-001", input="帮我看看代码有没有问题", expected_skill="code_reviewer", expected_execution_mode="react", expected_complexity="medium", category="semantic_router", subcategory="colloquial_match", paraphrases=["代码审查一下", "Check my code for issues"], tags=["semantic", "colloquial", "code_review"], ), BenchmarkCase( id="semantic-colloquial-trend-001", input="最近市场行情怎么样", expected_skill="trend_agent", expected_execution_mode="tool_call", expected_complexity="medium", category="semantic_router", subcategory="colloquial_match", paraphrases=["市场走势如何", "What's the market trend"], tags=["semantic", "colloquial", "trend"], ), BenchmarkCase( id="semantic-colloquial-content-001", input="帮我写点东西", expected_skill="content_generator", expected_execution_mode="llm_generate", expected_complexity="low", category="semantic_router", subcategory="colloquial_match", paraphrases=["写篇文章吧", "Write something for me"], tags=["semantic", "colloquial", "content"], ), BenchmarkCase( id="semantic-colloquial-citation-001", input="这个引用对不对", expected_skill="citation_detector", expected_execution_mode="custom", expected_complexity="medium", category="semantic_router", subcategory="colloquial_match", paraphrases=["查查引用准不准", "Are these citations correct"], tags=["semantic", "colloquial", "citation"], ), BenchmarkCase( id="semantic-colloquial-competitor-001", input="对手怎么样", expected_skill="competitor_analyzer", expected_execution_mode="tool_call", expected_complexity="medium", category="semantic_router", subcategory="colloquial_match", paraphrases=["竞品啥情况", "How are competitors doing"], tags=["semantic", "colloquial", "competitor"], ), # --- Mixed Chinese-English expressions (中英混合) --- BenchmarkCase( id="semantic-mixed-review-001", input="review一下这段代码", expected_skill="code_reviewer", expected_execution_mode="react", expected_complexity="medium", category="semantic_router", subcategory="mixed_lang_match", paraphrases=["帮我review代码", "Code review please"], tags=["semantic", "mixed", "code_review"], ), BenchmarkCase( id="semantic-mixed-geo-001", input="做个SEO优化", expected_skill="geo_optimizer", expected_execution_mode="llm_generate", expected_complexity="low", category="semantic_router", subcategory="mixed_lang_match", paraphrases=["GEO优化一下", "Optimize for AI search"], tags=["semantic", "mixed", "geo"], ), BenchmarkCase( id="semantic-mixed-monitor-001", input="monitor一下系统状态", expected_skill="monitor", expected_execution_mode="tool_call", expected_complexity="medium", category="semantic_router", subcategory="mixed_lang_match", paraphrases=["监控系统运行", "Monitor system status"], tags=["semantic", "mixed", "monitor"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Alignment Guard Benchmarks # ═══════════════════════════════════════════════════════════════════════════ ALIGNMENT_BENCHMARKS: list[BenchmarkCase] = [ BenchmarkCase( id="align-negative-001", input="写一篇产品介绍，不要提及价格", expected_skill="content_generator", expected_execution_mode="llm_generate", expected_complexity="low", category="alignment", subcategory="negative_constraint", tags=["alignment", "negative_constraint"], ), BenchmarkCase( id="align-positive-001", input="生成报告，必须包含摘要部分", expected_skill="plan_exec_agent", expected_execution_mode="plan_exec", expected_complexity="high", category="alignment", subcategory="positive_constraint", tags=["alignment", "positive_constraint"], ), BenchmarkCase( id="align-cascade-001", input="反复搜索相同关键词", expected_skill="react_agent", expected_execution_mode="react", expected_complexity="high", category="alignment", subcategory="cascade_detection", tags=["alignment", "cascade"], ), BenchmarkCase( id="align-no-constraint-001", input="帮我写一篇文章", expected_skill="content_generator", expected_execution_mode="llm_generate", expected_complexity="low", category="alignment", subcategory="no_constraint", tags=["alignment", "baseline"], ), BenchmarkCase( id="align-combined-001", input="生成竞品分析报告，必须包含对比表格，不要提及内部数据", expected_skill="competitor_analyzer", expected_execution_mode="tool_call", expected_complexity="medium", category="alignment", subcategory="combined_constraint", tags=["alignment", "combined"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # Board Meeting (Private Board) Benchmarks — @board prefix routing # ═══════════════════════════════════════════════════════════════════════════ BOARD_BENCHMARKS: list[BenchmarkCase] = [ # --- Default template (@board without experts) --- BenchmarkCase( id="board-default-001", input="@board 讨论是否应该进入东南亚市场", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="default_template", paraphrases=[ "@board 我们要不要拓展东南亚业务", "@board 东南亚市场进入策略讨论", "@board:private_board 评估东南亚市场机会", ], tags=["board", "default", "strategy"], ), BenchmarkCase( id="board-default-002", input="@board AI产品定价策略应该怎么做", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="default_template", paraphrases=["@board 如何给AI产品定价", "@board AI产品定价讨论"], tags=["board", "default", "pricing"], ), # --- Explicit expert list (@board:expert1,expert2) --- BenchmarkCase( id="board-explicit-001", input="@board:elon_musk,jeff_bezos 讨论火星殖民的商业化路径", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="explicit_experts", paraphrases=[ "@board:elon_musk,jeff_bezos 火星商业化方案", "@board:jeff_bezos,elon_musk 如何商业化火星", ], tags=["board", "explicit", "mars"], ), BenchmarkCase( id="board-explicit-002", input="@board:charlie_munger,warren_buffett 价值投资在AI时代的适用性", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="explicit_experts", paraphrases=[ "@board:charlie_munger,warren_buffett AI时代还要不要价值投资", ], tags=["board", "explicit", "investing"], ), # --- Explicit default template name --- BenchmarkCase( id="board-template-001", input="@board:private_board 讨论创业公司融资节奏", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="explicit_template", paraphrases=["@board:private_board 创业融资策略", "@board:private_board 融资节奏讨论"], tags=["board", "template", "fundraising"], ), # --- Edge cases --- BenchmarkCase( id="board-edge-empty-topic-001", input="@board", expected_execution_mode="board", expected_complexity="low", category="board", subcategory="empty_topic", tags=["board", "edge", "empty"], ), BenchmarkCase( id="board-edge-no-prefix-001", input="讨论一下市场策略", expected_execution_mode="react", expected_complexity="medium", category="board", subcategory="no_prefix", paraphrases=["分析市场策略", "市场策略讨论"], tags=["board", "edge", "no_match"], ), # --- Name validation --- BenchmarkCase( id="board-name-valid-001", input="@board:elon_musk,jeff_bezos,allenzhang 产品设计哲学", expected_execution_mode="board", expected_complexity="high", category="board", subcategory="name_validation", tags=["board", "validation", "valid"], ), BenchmarkCase( id="board-name-invalid-001", input="@board:@#$ 讨论主题", expected_execution_mode="board", expected_complexity="low", category="board", subcategory="name_validation", tags=["board", "validation", "invalid"], ), # --- Stop command (user intervention) --- BenchmarkCase( id="board-stop-001", input="/stop", expected_execution_mode="board", expected_complexity="low", category="board", subcategory="stop_command", paraphrases=["停止讨论", "结束讨论"], tags=["board", "stop", "intervention"], ), ] # ═══════════════════════════════════════════════════════════════════════════ # All benchmarks combined # ═══════════════════════════════════════════════════════════════════════════ ALL_BENCHMARKS: list[BenchmarkCase] = ( ROUTING_KEYWORD_BENCHMARKS + ROUTING_EDGE_BENCHMARKS + EXECUTION_BENCHMARKS + TEAM_BENCHMARKS + CONSISTENCY_BENCHMARKS + SEMANTIC_ROUTER_BENCHMARKS + ALIGNMENT_BENCHMARKS + BOARD_BENCHMARKS ) def get_benchmarks_by_category(category: str) -> list[BenchmarkCase]: """Filter benchmarks by category.""" return [b for b in ALL_BENCHMARKS if b.category == category] def get_benchmarks_by_subcategory(subcategory: str) -> list[BenchmarkCase]: """Filter benchmarks by subcategory.""" return [b for b in ALL_BENCHMARKS if b.subcategory == subcategory] def get_benchmarks_with_paraphrases() -> list[BenchmarkCase]: """Get only benchmarks that have paraphrases (for overfitting detection).""" return [b for b in ALL_BENCHMARKS if b.paraphrases] def get_skill_names_needed() -> set[str]: """Get all skill names referenced in benchmarks (for pre-registration).""" return {b.expected_skill for b in ALL_BENCHMARKS if b.expected_skill is not None} def get_benchmark_stats() -> dict[str, int]: """Get benchmark count by category.""" stats: dict[str, int] = {} for b in ALL_BENCHMARKS: stats[b.category] = stats.get(b.category, 0) + 1 stats["total"] = len(ALL_BENCHMARKS) return stats