"""Agent Capability Metrics — Collection, Analysis, and Reporting. Core components: 1. CapabilityMetrics: data model for a single test observation 2. MetricsCollector: session-scoped collector that gathers all observations 3. MetricsAnalyzer: computes recall/precision/F1, overfitting scores, weakness analysis 4. MetricsReporter: generates human-readable and machine-readable reports Design: - Collector is a pytest fixture (session-scoped), injected into capability tests - Each test records what actually happened vs what was expected - After all tests, analyzer computes aggregate metrics - Reporter outputs JSON + plain-text summary """ import json import os import time from collections import defaultdict from datetime import datetime, timezone from typing import Any from pydantic import BaseModel, ConfigDict from tests.e2e.benchmark_dataset import BenchmarkCase # ═══════════════════════════════════════════════════════════════════════════ # 1. Data Models # ═══════════════════════════════════════════════════════════════════════════ class CapabilityObservation(BaseModel): """A single test observation: what was expected vs what actually happened.""" model_config = ConfigDict() # Identity benchmark_id: str test_name: str timestamp: str # Input input_query: str is_paraphrase: bool = False # True if this is a paraphrase test (overfitting detection) # Expected (ground truth) expected_skill: str | None = None expected_execution_mode: str = "direct" expected_complexity: str = "low" # Actual (observed) actual_skill: str | None = None actual_execution_mode: str | None = None actual_status_code: int = 0 actual_response_keys: list[str] = [] actual_complexity_score: float | None = None actual_match_method: str | None = None actual_match_confidence: float | None = None # Judgments skill_correct: bool | None = None # None = couldn't determine execution_mode_correct: bool | None = None complexity_correct: bool | None = None task_succeeded: bool = False # HTTP 200 + valid response # Metadata category: str = "" subcategory: str = "" response_time_ms: float = 0.0 error_message: str | None = None # Alignment & Cascade fields (U5) alignment_violations: int = 0 # Number of constraint violations detected cascade_alert: bool = False # Whether a cascade alert was triggered # L3 Output Quality fields output_quality_score: float | None = None # 1-5 LLM-as-Judge score output_quality_reasoning: str | None = None # Judge's reasoning class OutputQualityObservation(BaseModel): """L3 output quality evaluation result.""" model_config = ConfigDict() benchmark_id: str input_query: str expected_skill: str | None = None actual_skill: str | None = None quality_score: float = 0.0 # 1-5 reasoning: str = "" evaluated: bool = False class CategoryMetrics(BaseModel): """Aggregate metrics for a specific category/subcategory.""" model_config = ConfigDict() category: str subcategory: str total: int = 0 skill_correct: int = 0 skill_recall: float = 0.0 skill_precision: float = 0.0 skill_f1: float = 0.0 execution_mode_correct: int = 0 execution_mode_accuracy: float = 0.0 complexity_correct: int = 0 complexity_accuracy: float = 0.0 task_success_rate: float = 0.0 avg_response_time_ms: float = 0.0 class OverfittingResult(BaseModel): """Overfitting detection result for a single benchmark case.""" model_config = ConfigDict() benchmark_id: str original_correct: bool paraphrase_results: list[bool] # True = correct for each paraphrase consistency_rate: float = 0.0 # % of paraphrases that match original result is_overfitted: bool = False # True if original correct but paraphrases mostly wrong class WeaknessItem(BaseModel): """A single identified weakness.""" model_config = ConfigDict() dimension: str # routing / execution / quality / team / consistency subcategory: str severity: str # critical / high / medium / low description: str evidence: str suggestion: str class RootCause(BaseModel): """Root cause analysis for a weakness.""" model_config = ConfigDict() cause_type: str # keyword_gap / complexity_misjudge / intent_ambiguous / fallback_missing / overfit_pattern / tool_missing / config_error / quality_threshold cause_description: str confidence: float = 0.0 # 0.0~1.0, how confident we are about this root cause affected_cases: list[str] = [] # benchmark IDs affected by this cause detail: str = "" # additional technical detail class ImprovementAction(BaseModel): """A single actionable improvement step.""" model_config = ConfigDict() action_id: str title: str description: str target_module: str # which code module to modify priority: str # P0 / P1 / P2 / P3 expected_impact: str # what improvement to expect effort: str # small / medium / large related_causes: list[str] = [] # cause_types this action addresses verification: str = "" # how to verify the fix works class ImprovementPlan(BaseModel): """Improvement plan for a specific weakness.""" model_config = ConfigDict() weakness_description: str root_causes: list[RootCause] actions: list[ImprovementAction] overall_strategy: str class CapabilityReport(BaseModel): """Full capability analysis report.""" model_config = ConfigDict() generated_at: str total_observations: int overall_skill_recall: float overall_skill_precision: float overall_skill_f1: float overall_execution_mode_accuracy: float overall_task_success_rate: float category_metrics: list[CategoryMetrics] overfitting_results: list[OverfittingResult] overfitting_score: float # 0.0 = no overfitting, 1.0 = fully overfitted weaknesses: list[WeaknessItem] root_causes: list[RootCause] improvement_plans: list[ImprovementPlan] raw_observations: list[CapabilityObservation] output_quality_evaluations: list[OutputQualityObservation] = [] # ═══════════════════════════════════════════════════════════════════════════ # 2. Metrics Collector # ═══════════════════════════════════════════════════════════════════════════ class MetricsCollector: """Collects capability observations during E2E test execution. Usage in tests: collector.record(observation) collector.record_benchmark_result(benchmark, actual_skill, ...) """ def __init__(self) -> None: self._observations: list[CapabilityObservation] = [] self._start_times: dict[str, float] = {} def start_timer(self, benchmark_id: str) -> None: self._start_times[benchmark_id] = time.monotonic() def stop_timer(self, benchmark_id: str) -> float: start = self._start_times.pop(benchmark_id, None) if start is None: return 0.0 return (time.monotonic() - start) * 1000 # ms def record(self, observation: CapabilityObservation) -> None: self._observations.append(observation) def record_benchmark_result( self, benchmark: BenchmarkCase, *, test_name: str, actual_skill: str | None = None, actual_execution_mode: str | None = None, actual_status_code: int = 0, actual_response_keys: list[str] | None = None, task_succeeded: bool = False, is_paraphrase: bool = False, error_message: str | None = None, ) -> CapabilityObservation: """Record a benchmark test result with automatic correctness judgment.""" response_time = self.stop_timer(benchmark.id) # Judge skill correctness skill_correct: bool | None = None if benchmark.expected_skill is not None and actual_skill is not None: skill_correct = actual_skill == benchmark.expected_skill elif benchmark.expected_skill is None: # Expected no specific skill, so any non-error is acceptable skill_correct = actual_skill is None or task_succeeded # Judge execution mode correctness execution_mode_correct: bool | None = None if actual_execution_mode is not None: # Normalize both sides for comparison: # actual: "skill_react" / "rewoo" / "direct_chat" etc. # expected: "react" / "rewoo" / "direct" etc. _MODE_EQUIVALENCE: dict[str, str] = { "skill_react": "react", "direct_chat": "direct", "team_collab": "team_collab", } actual_norm = _MODE_EQUIVALENCE.get(actual_execution_mode, actual_execution_mode) execution_mode_correct = actual_norm == benchmark.expected_execution_mode # Judge complexity correctness (approximate: based on execution mode match) complexity_correct: bool | None = None if execution_mode_correct is not None: complexity_correct = execution_mode_correct obs = CapabilityObservation( benchmark_id=benchmark.id, test_name=test_name, timestamp=datetime.now(timezone.utc).isoformat(), input_query=benchmark.input, is_paraphrase=is_paraphrase, expected_skill=benchmark.expected_skill, expected_execution_mode=benchmark.expected_execution_mode, expected_complexity=benchmark.expected_complexity, actual_skill=actual_skill, actual_execution_mode=actual_execution_mode, actual_status_code=actual_status_code, actual_response_keys=actual_response_keys or [], skill_correct=skill_correct, execution_mode_correct=execution_mode_correct, complexity_correct=complexity_correct, task_succeeded=task_succeeded, category=benchmark.category, subcategory=benchmark.subcategory, response_time_ms=response_time, error_message=error_message, ) self._observations.append(obs) return obs @property def observations(self) -> list[CapabilityObservation]: return self._observations def get_observations_by_category(self, category: str) -> list[CapabilityObservation]: return [o for o in self._observations if o.category == category] def get_observations_by_subcategory(self, subcategory: str) -> list[CapabilityObservation]: return [o for o in self._observations if o.subcategory == subcategory] def get_original_observations(self) -> list[CapabilityObservation]: """Get non-paraphrase observations.""" return [o for o in self._observations if not o.is_paraphrase] def get_paraphrase_observations(self) -> list[CapabilityObservation]: """Get paraphrase observations only.""" return [o for o in self._observations if o.is_paraphrase] def evaluate_output_quality( self, llm_gateway: Any ) -> list[OutputQualityObservation]: """L3 Output Quality Evaluation using LLM-as-Judge. Evaluates only keyword_match and semantic_match categories. Returns list of OutputQualityObservation with quality scores. """ results: list[OutputQualityObservation] = [] eval_categories = {"routing", "semantic_router"} for obs in self._observations: if obs.category not in eval_categories: continue if obs.actual_skill is None: continue if not obs.task_succeeded: continue prompt = ( f"评估以下Agent路由-执行结果的质量（1-5分）。\n\n" f"用户输入: {obs.input_query}\n" f"期望技能: {obs.expected_skill}\n" f"实际路由技能: {obs.actual_skill}\n" f"执行模式: {obs.actual_execution_mode}\n\n" f"评分标准:\n" f"1分: 完全错误的路由，输出与用户意图无关\n" f"2分: 路由有偏差，输出部分相关但缺少关键内容\n" f"3分: 路由基本正确，输出相关但不完整\n" f"4分: 路由正确，输出完整且相关\n" f"5分: 路由精准，输出完全匹配用户意图且质量优秀\n\n" f"请只输出JSON: {{\"score\": <1-5>, \"reasoning\": \"<一句话理由>\"}}" ) try: import asyncio response = asyncio.run( llm_gateway.chat( messages=[{"role": "user", "content": prompt}], model="default", temperature=0.0, max_tokens=200, ) ) content = response.get("content", "") if isinstance(response, dict) else str(response) # Parse JSON from response import re json_match = re.search(r'\{[^}]+\}', content) if json_match: import json as _json parsed = _json.loads(json_match.group()) score = float(parsed.get("score", 0)) reasoning = parsed.get("reasoning", "") else: score = 0.0 reasoning = f"Parse failed: {content[:100]}" results.append( OutputQualityObservation( benchmark_id=obs.benchmark_id, input_query=obs.input_query, expected_skill=obs.expected_skill, actual_skill=obs.actual_skill, quality_score=max(1.0, min(5.0, score)), reasoning=reasoning, evaluated=True, ) ) except Exception as e: results.append( OutputQualityObservation( benchmark_id=obs.benchmark_id, input_query=obs.input_query, expected_skill=obs.expected_skill, actual_skill=obs.actual_skill, quality_score=0.0, reasoning=f"Evaluation error: {e}", evaluated=False, ) ) return results # ═══════════════════════════════════════════════════════════════════════════ # 3. Metrics Analyzer # ═══════════════════════════════════════════════════════════════════════════ class MetricsAnalyzer: """Analyzes collected metrics to compute recall/precision/F1, overfitting, weaknesses.""" @staticmethod def _safe_div(numerator: float, denominator: float) -> float: return numerator / denominator if denominator > 0 else 0.0 @staticmethod def compute_prf(tp: int, fp: int, fn: int) -> tuple[float, float, float]: """Compute precision, recall, F1 from counts.""" precision = MetricsAnalyzer._safe_div(tp, tp + fp) recall = MetricsAnalyzer._safe_div(tp, tp + fn) f1 = MetricsAnalyzer._safe_div(2 * precision * recall, precision + recall) return precision, recall, f1 def analyze_category( self, observations: list[CapabilityObservation], category: str, subcategory: str ) -> CategoryMetrics: """Compute aggregate metrics for a category/subcategory.""" filtered = [ o for o in observations if o.category == category and (not subcategory or o.subcategory == subcategory) ] if not filtered: return CategoryMetrics(category=category, subcategory=subcategory) total = len(filtered) skill_correct_count = sum(1 for o in filtered if o.skill_correct is True) exec_correct_count = sum(1 for o in filtered if o.execution_mode_correct is True) complexity_correct_count = sum(1 for o in filtered if o.complexity_correct is True) task_success_count = sum(1 for o in filtered if o.task_succeeded) avg_response_time = sum(o.response_time_ms for o in filtered) / total # For skill routing: compute per-skill PRF # TP = correctly routed to expected skill # FP = routed to wrong skill # FN = expected skill but not routed to it tp = skill_correct_count fp = sum(1 for o in filtered if o.skill_correct is False and o.actual_skill is not None) fn = sum(1 for o in filtered if o.skill_correct is False and o.expected_skill is not None) precision, recall, f1 = self.compute_prf(tp, fp, fn) return CategoryMetrics( category=category, subcategory=subcategory, total=total, skill_correct=skill_correct_count, skill_recall=round(recall, 4), skill_precision=round(precision, 4), skill_f1=round(f1, 4), execution_mode_correct=exec_correct_count, execution_mode_accuracy=round(self._safe_div(exec_correct_count, total), 4), complexity_correct=complexity_correct_count, complexity_accuracy=round(self._safe_div(complexity_correct_count, total), 4), task_success_rate=round(self._safe_div(task_success_count, total), 4), avg_response_time_ms=round(avg_response_time, 2), ) def detect_overfitting( self, observations: list[CapabilityObservation] ) -> tuple[list[OverfittingResult], float]: """Detect overfitting by comparing original vs paraphrase results. Returns (overfitting_results, overall_overfitting_score). overfitting_score = 0.0 means no overfitting (paraphrases work as well as originals). overfitting_score = 1.0 means complete overfitting (originals correct, paraphrases all wrong). """ originals = {o.benchmark_id: o for o in observations if not o.is_paraphrase} paraphrases: dict[str, list[CapabilityObservation]] = defaultdict(list) for o in observations: if o.is_paraphrase: paraphrases[o.benchmark_id].append(o) results: list[OverfittingResult] = [] total_inconsistency = 0.0 total_comparisons = 0 for bid, orig in originals.items(): paras = paraphrases.get(bid, []) if not paras: continue orig_correct = orig.skill_correct is True para_corrects = [p.skill_correct is True for p in paras] # Consistency: how many paraphrases match the original result matches = sum(1 for pc in para_corrects if pc == orig_correct) consistency_rate = self._safe_div(matches, len(para_corrects)) # Overfitted: original correct but paraphrases mostly wrong is_overfitted = orig_correct and consistency_rate < 0.5 results.append( OverfittingResult( benchmark_id=bid, original_correct=orig_correct, paraphrase_results=para_corrects, consistency_rate=round(consistency_rate, 4), is_overfitted=is_overfitted, ) ) if orig_correct: # Only count inconsistency when original was correct total_inconsistency += 1.0 - consistency_rate total_comparisons += 1 overfitting_score = self._safe_div(total_inconsistency, total_comparisons) return results, round(overfitting_score, 4) def identify_weaknesses( self, category_metrics: list[CategoryMetrics], overfitting_results: list[OverfittingResult], ) -> list[WeaknessItem]: """Identify intelligence weaknesses based on metrics analysis.""" weaknesses: list[WeaknessItem] = [] for cm in category_metrics: # Low skill F1 if cm.skill_f1 < 0.5 and cm.total >= 2: weaknesses.append( WeaknessItem( dimension=cm.category, subcategory=cm.subcategory, severity="critical" if cm.skill_f1 < 0.3 else "high", description=f"技能路由F1过低 ({cm.skill_f1:.2f})，子类别: {cm.subcategory}", evidence=f"召回率={cm.skill_recall:.2%}, 精确率={cm.skill_precision:.2%}, 样本数={cm.total}", suggestion="改进该子类别的关键词匹配或意图分类逻辑", ) ) elif cm.skill_f1 < 0.8 and cm.total >= 2: weaknesses.append( WeaknessItem( dimension=cm.category, subcategory=cm.subcategory, severity="medium", description=f"技能路由F1偏低 ({cm.skill_f1:.2f})，子类别: {cm.subcategory}", evidence=f"召回率={cm.skill_recall:.2%}, 精确率={cm.skill_precision:.2%}, 样本数={cm.total}", suggestion="微调路由阈值或增加更多意图示例", ) ) # Low execution mode accuracy if cm.execution_mode_accuracy < 0.6 and cm.total >= 2: weaknesses.append( WeaknessItem( dimension=cm.category, subcategory=cm.subcategory, severity="high" if cm.execution_mode_accuracy < 0.4 else "medium", description=f"执行模式准确率过低 ({cm.execution_mode_accuracy:.2%})，子类别: {cm.subcategory}", evidence=f"正确数={cm.execution_mode_correct}/{cm.total}", suggestion="检查复杂度估算和模式选择逻辑", ) ) # Low task success rate if cm.task_success_rate < 0.8 and cm.total >= 2: weaknesses.append( WeaknessItem( dimension=cm.category, subcategory=cm.subcategory, severity="critical" if cm.task_success_rate < 0.5 else "high", description=f"任务成功率过低 ({cm.task_success_rate:.2%})，子类别: {cm.subcategory}", evidence=f"成功数={int(cm.task_success_rate * cm.total)}/{cm.total}", suggestion="排查该子类别的任务执行失败原因", ) ) # Overfitting weaknesses overfitted_cases = [r for r in overfitting_results if r.is_overfitted] if overfitted_cases: weaknesses.append( WeaknessItem( dimension="routing", subcategory="overfitting", severity="high", description=f"检测到 {len(overfitted_cases)} 个用例存在过拟合", evidence=f"过拟合用例: {', '.join(r.benchmark_id for r in overfitted_cases)}", suggestion="增加更多样化的训练样本和同义改写，提升泛化能力", ) ) # Sort by severity severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3} weaknesses.sort(key=lambda w: severity_order.get(w.severity, 99)) return weaknesses # ═════════════════════════════════════════════════════════════════════ # Root Cause Analysis Engine # ═════════════════════════════════════════════════════════════════════ def analyze_root_causes( self, observations: list[CapabilityObservation], category_metrics: list[CategoryMetrics], overfitting_results: list[OverfittingResult], weaknesses: list[WeaknessItem], ) -> list[RootCause]: """Perform root cause analysis based on observation data. Strategy: 1. For each weakness, examine the raw observations to find patterns 2. Cross-reference paraphrase vs original results for overfitting clues 3. Analyze error messages for common failure modes 4. Check recall vs precision imbalance to distinguish cause types """ root_causes: list[RootCause] = [] originals = [o for o in observations if not o.is_paraphrase] paraphrases = [o for o in observations if o.is_paraphrase] # --- Cause 1: Keyword gap (low recall = keywords not matching) --- low_recall_cases = [ o for o in originals if o.skill_correct is False and o.expected_skill is not None and o.actual_skill is None ] if low_recall_cases: affected = [o.benchmark_id for o in low_recall_cases] # Check if paraphrases also fail → confirms keyword gap para_also_fail = sum( 1 for p in paraphrases if p.benchmark_id in affected and p.skill_correct is False ) confidence = min(1.0, 0.5 + 0.1 * para_also_fail) if paraphrases else 0.6 root_causes.append( RootCause( cause_type="keyword_gap", cause_description="关键词覆盖不足：用户输入无法匹配到目标技能的关键词", confidence=round(confidence, 2), affected_cases=affected[:10], detail=( f"共 {len(low_recall_cases)} 个原始输入未能路由到期望技能。" f"改写输入中也有 {para_also_fail} 个失败，" f"说明关键词库对同义表达的覆盖不足。" f"受影响子类别: {', '.join(set(o.subcategory for o in low_recall_cases))}" ), ) ) # --- Cause 2: Precision gap (wrong skill routed = intent ambiguous) --- wrong_route_cases = [ o for o in originals if o.skill_correct is False and o.actual_skill is not None and o.expected_skill is not None ] if wrong_route_cases: affected = [o.benchmark_id for o in wrong_route_cases] # Check which skills are being confused confusion_pairs: dict[tuple[str, str], int] = defaultdict(int) for o in wrong_route_cases: confusion_pairs[(o.expected_skill, o.actual_skill)] += 1 top_confusions = sorted(confusion_pairs.items(), key=lambda x: -x[1])[:5] confusion_detail = "; ".join( f"{exp}→{act}({cnt}次)" for (exp, act), cnt in top_confusions ) root_causes.append( RootCause( cause_type="intent_ambiguous", cause_description="意图歧义：不同技能的关键词/意图描述重叠，导致路由混淆", confidence=0.7, affected_cases=affected[:10], detail=f"技能混淆对: {confusion_detail}", ) ) # --- Cause 3: Complexity misjudge (execution mode wrong) --- exec_wrong_cases = [o for o in originals if o.execution_mode_correct is False] if exec_wrong_cases: affected = [o.benchmark_id for o in exec_wrong_cases] # Analyze direction of misjudgment over_simplified = sum( 1 for o in exec_wrong_cases if o.expected_complexity in ("high", "medium") and o.actual_execution_mode == "direct" ) over_complicated = sum( 1 for o in exec_wrong_cases if o.expected_complexity == "low" and o.actual_execution_mode in ("react", "rewoo", "reflexion") ) direction = "" if over_simplified > over_complicated: direction = "倾向低估复杂度（将复杂任务误判为简单直接调用）" elif over_complicated > over_simplified: direction = "倾向高估复杂度（将简单任务误判为需要多步推理）" else: direction = "复杂度误判方向不明确，双向均有偏差" root_causes.append( RootCause( cause_type="complexity_misjudge", cause_description=f"复杂度估算偏差：{direction}", confidence=0.75, affected_cases=affected[:10], detail=( f"共 {len(exec_wrong_cases)} 个执行模式判断错误。" f"低估复杂度 {over_simplified} 次，高估复杂度 {over_complicated} 次。" f"受影响子类别: {', '.join(set(o.subcategory for o in exec_wrong_cases))}" ), ) ) # --- Cause 4: Fallback missing (no skill matched, task failed) --- fallback_fail_cases = [ o for o in originals if o.expected_skill is None and not o.task_succeeded ] if fallback_fail_cases: affected = [o.benchmark_id for o in fallback_fail_cases] root_causes.append( RootCause( cause_type="fallback_missing", cause_description="回退机制不足：无匹配技能时，直接聊天模式未能正常处理", confidence=0.65, affected_cases=affected[:10], detail=( f"共 {len(fallback_fail_cases)} 个无技能匹配的任务执行失败。" f"错误信息: {'; '.join(set(o.error_message or 'N/A' for o in fallback_fail_cases[:5]))}" ), ) ) # --- Cause 5: Overfit pattern (paraphrases fail while original succeeds) --- overfitted = [r for r in overfitting_results if r.is_overfitted] if overfitted: affected = [r.benchmark_id for r in overfitted] # Analyze what kind of paraphrases fail para_fail_details: list[str] = [] for r in overfitted: fail_count = sum(1 for ok in r.paraphrase_results if not ok) para_fail_details.append( f"{r.benchmark_id}({fail_count}/{len(r.paraphrase_results)}改写失败)" ) root_causes.append( RootCause( cause_type="overfit_pattern", cause_description="路由过拟合：对特定表述形式过度敏感，同义改写后路由失败", confidence=0.85, affected_cases=affected, detail=( f"共 {len(overfitted)} 个用例存在过拟合。" f"详情: {'; '.join(para_fail_details)}。" f"说明路由逻辑对输入的具体措辞过于敏感，缺乏语义层面的泛化能力。" ), ) ) # --- Cause 6: Quality threshold (task succeeded but output poor) --- success_but_wrong = [o for o in originals if o.task_succeeded and o.skill_correct is False] if len(success_but_wrong) >= 2: affected = [o.benchmark_id for o in success_but_wrong] root_causes.append( RootCause( cause_type="quality_threshold", cause_description="质量门控阈值过低：任务虽成功完成但输出了错误结果", confidence=0.6, affected_cases=affected[:10], detail=( f"共 {len(success_but_wrong)} 个任务虽然HTTP成功但路由到了错误技能。" f"质量门控未能拦截这些错误路由的结果。" ), ) ) # --- Cause 7: Config error (HTTP errors) --- error_cases = [o for o in originals if o.error_message and not o.task_succeeded] if error_cases: # Group by error pattern error_patterns: dict[str, int] = defaultdict(int) for o in error_cases: # Simplify error message to pattern msg = (o.error_message or "")[:80] error_patterns[msg] += 1 top_errors = sorted(error_patterns.items(), key=lambda x: -x[1])[:3] error_detail = "; ".join(f"{msg}({cnt}次)" for msg, cnt in top_errors) root_causes.append( RootCause( cause_type="config_error", cause_description="配置或服务端错误：请求处理过程中出现异常", confidence=0.5, affected_cases=[o.benchmark_id for o in error_cases[:10]], detail=f"常见错误: {error_detail}", ) ) # Sort by confidence root_causes.sort(key=lambda rc: -rc.confidence) return root_causes # ═════════════════════════════════════════════════════════════════════ # Improvement Strategy Planner # ═════════════════════════════════════════════════════════════════════ def plan_improvements( self, weaknesses: list[WeaknessItem], root_causes: list[RootCause], ) -> list[ImprovementPlan]: """Generate improvement plans based on weaknesses and root causes.""" plans: list[ImprovementPlan] = [] action_counter = 0 # Map root causes by type for quick lookup causes_by_type: dict[str, list[RootCause]] = defaultdict(list) for rc in root_causes: causes_by_type[rc.cause_type].append(rc) # --- Plan for keyword_gap --- if "keyword_gap" in causes_by_type: cause = causes_by_type["keyword_gap"][0] actions: list[ImprovementAction] = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="扩展技能关键词同义词库", description=( "为每个技能的 intent.keywords 添加更多同义词、近义词和用户常见表述。" "重点补充中文变体、口语化表达和行业术语。" ), target_module="configs/skills/*.yaml → intent.keywords", priority="P0", expected_impact=f"预计提升召回率 15~30%，影响 {len(cause.affected_cases)} 个用例", effort="small", related_causes=["keyword_gap"], verification="重新运行E2E回测，验证受影响用例的召回率提升", ) ) action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="引入语义相似度匹配（Layer 1.5）", description=( "在 CostAwareRouter 的 Layer 1.5 SemanticRouter 中，" "使用向量嵌入计算用户输入与技能描述的语义相似度，" "弥补关键词精确匹配的不足。" ), target_module="src/agentkit/chat/skill_routing.py", priority="P1", expected_impact="预计提升召回率 20~40%，显著改善同义改写场景", effort="large", related_causes=["keyword_gap", "overfit_pattern"], verification="运行过拟合检测回测，验证改写一致性提升至 >80%", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["keyword_gap"], actions=actions, overall_strategy=( "短期：扩充关键词库（低成本高收益）；" "中期：引入语义匹配层（高成本高收益）；" "长期：基于用户真实查询日志持续优化关键词库" ), ) ) # --- Plan for intent_ambiguous --- if "intent_ambiguous" in causes_by_type: cause = causes_by_type["intent_ambiguous"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="为易混淆技能添加互斥关键词", description=( "在技能配置中为容易混淆的技能对添加互斥关键词（disambiguation_keywords），" "当用户输入同时匹配多个技能时，优先选择包含互斥关键词的技能。" ), target_module="configs/skills/*.yaml → intent.disambiguation_keywords", priority="P1", expected_impact="预计提升精确率 10~25%，减少技能混淆", effort="small", related_causes=["intent_ambiguous"], verification="运行歧义消解回测，验证路由精确率提升", ) ) action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="实现LLM二次分类消歧", description=( "当 Layer 0/1 路由到多个候选技能时，" "调用 LLM quick_classify 进行二次意图判断，" "选择最匹配的技能。" ), target_module="src/agentkit/chat/skill_routing.py → Layer 1", priority="P2", expected_impact="预计提升精确率 15~30%，但增加 ~500ms 延迟和 ~100 tokens", effort="medium", related_causes=["intent_ambiguous"], verification="运行歧义消解回测，对比延迟和精确率变化", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["intent_ambiguous"], actions=actions, overall_strategy=( "短期：添加互斥关键词消歧；" "中期：启用LLM二次分类；" "长期：训练专用意图分类模型替代规则匹配" ), ) ) # --- Plan for complexity_misjudge --- if "complexity_misjudge" in causes_by_type: cause = causes_by_type["complexity_misjudge"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="优化复杂度估算启发式规则", description=( "调整 HeuristicClassifier 的复杂度评分权重：" "增加任务动词（分析/研究/设计）的权重，" "降低简单问答动词（是什么/多少）的权重。" ), target_module="src/agentkit/chat/skill_routing.py → HeuristicClassifier", priority="P1", expected_impact="预计提升执行模式准确率 10~20%", effort="small", related_causes=["complexity_misjudge"], verification="运行执行模式回测，验证准确率提升", ) ) action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="引入任务复杂度校准数据集", description=( "收集标注了复杂度等级的真实用户查询，" "构建校准数据集，定期评估和调整复杂度阈值。" ), target_module="tests/e2e/benchmark_dataset.py", priority="P2", expected_impact="持续提升复杂度判断准确性", effort="medium", related_causes=["complexity_misjudge"], verification="每次调整后运行回测，对比前后F1变化", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["complexity_misjudge"], actions=actions, overall_strategy=( "短期：调整启发式规则权重；" "中期：构建复杂度校准数据集；" "长期：训练复杂度评估模型替代规则" ), ) ) # --- Plan for fallback_missing --- if "fallback_missing" in causes_by_type: cause = causes_by_type["fallback_missing"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="增强DIRECT_CHAT回退路径", description=( "当无技能匹配时，确保DIRECT_CHAT模式能正常处理请求：" "1) 检查默认Agent是否正确初始化；" "2) 确保无技能时不会触发空指针异常；" "3) 添加友好的降级提示。" ), target_module="src/agentkit/chat/skill_routing.py → _fallback_direct_chat", priority="P0", expected_impact="确保100%的请求都有回退处理，消除任务失败", effort="small", related_causes=["fallback_missing"], verification="运行回退场景回测，验证所有无匹配请求均成功", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["fallback_missing"], actions=actions, overall_strategy=( "短期：修复回退路径确保基本可用；" "中期：优化回退模式的回答质量；" "长期：基于用户反馈自动发现新技能需求" ), ) ) # --- Plan for overfit_pattern --- if "overfit_pattern" in causes_by_type: cause = causes_by_type["overfit_pattern"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="添加意图描述和示例（intent.description + examples）", description=( "为每个技能添加 intent.description（自然语言描述）和 intent.examples（示例查询），" "使路由器能理解语义层面的意图，而不仅依赖关键词精确匹配。" ), target_module="configs/skills/*.yaml → intent.description / intent.examples", priority="P0", expected_impact="预计提升改写一致性 20~40%", effort="small", related_causes=["overfit_pattern", "keyword_gap"], verification="运行过拟合检测回测，验证改写一致性提升", ) ) action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="实现意图泛化测试CI", description=( "在CI中集成意图泛化回测：每次修改路由逻辑或技能配置后，" "自动运行包含改写的回测用例，确保不引入新的过拟合。" ), target_module=".github/workflows/ + tests/e2e/", priority="P2", expected_impact="防止过拟合回归，持续监控泛化能力", effort="medium", related_causes=["overfit_pattern"], verification="CI流水线中自动运行回测并检查过拟合分数", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["overfit_pattern"], actions=actions, overall_strategy=( "短期：补充意图描述和示例；" "中期：引入语义匹配（同keyword_gap方案）；" "长期：建立意图泛化CI防线" ), ) ) # --- Plan for quality_threshold --- if "quality_threshold" in causes_by_type: cause = causes_by_type["quality_threshold"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="增强质量门控的技能匹配验证", description=( "在QualityGate中增加技能匹配验证：" "检查输出是否与路由到的技能的能力范围一致，" "如果不一致则触发重试或降级。" ), target_module="src/agentkit/quality/gate.py", priority="P1", expected_impact="减少错误路由导致的低质量输出", effort="medium", related_causes=["quality_threshold"], verification="运行质量门控回测，验证错误路由拦截率", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["quality_threshold"], actions=actions, overall_strategy=( "短期：增加技能匹配验证；" "中期：引入输出质量评分模型；" "长期：实现自动质量回归检测" ), ) ) # --- Plan for config_error --- if "config_error" in causes_by_type: cause = causes_by_type["config_error"][0] actions = [] action_counter += 1 actions.append( ImprovementAction( action_id=f"ACT-{action_counter:03d}", title="修复服务端配置和异常处理", description=( "根据错误信息排查服务端配置问题：" "1) 检查API路由注册是否完整；" "2) 增加输入校验和错误提示；" "3) 确保所有异常都有友好的错误响应。" ), target_module="src/agentkit/server/routes/", priority="P0", expected_impact="消除服务端错误，提升任务成功率", effort="small", related_causes=["config_error"], verification="重新运行E2E回测，验证HTTP错误率降低", ) ) plans.append( ImprovementPlan( weakness_description=cause.cause_description, root_causes=causes_by_type["config_error"], actions=actions, overall_strategy=( "短期：修复已知配置错误；" "中期：增加输入校验和错误处理；" "长期：建立配置变更的自动化验证" ), ) ) return plans def analyze_alignment(self, observations: list[CapabilityObservation]) -> dict[str, Any]: """Analyze alignment guard and cascade detector metrics. Returns a dict with: - total_alignment_tests: number of observations in alignment category - violation_count: total constraint violations - violation_rate: ratio of tests with at least one violation - cascade_alert_count: number of cascade alerts triggered - cascade_alert_rate: ratio of tests that triggered cascade - neg_constraint_pass_rate: pass rate for negative constraints - pos_constraint_pass_rate: pass rate for positive constraints """ alignment_obs = [o for o in observations if o.category == "alignment"] if not alignment_obs: return { "total_alignment_tests": 0, "violation_count": 0, "violation_rate": 0.0, "cascade_alert_count": 0, "cascade_alert_rate": 0.0, "neg_constraint_pass_rate": 0.0, "pos_constraint_pass_rate": 0.0, } total = len(alignment_obs) with_violations = sum(1 for o in alignment_obs if o.alignment_violations > 0) total_violations = sum(o.alignment_violations for o in alignment_obs) with_cascade = sum(1 for o in alignment_obs if o.cascade_alert) # Separate by subcategory for neg/pos constraint pass rates neg_obs = [o for o in alignment_obs if o.subcategory == "negative_constraint"] pos_obs = [o for o in alignment_obs if o.subcategory == "positive_constraint"] neg_pass_rate = self._safe_div( sum(1 for o in neg_obs if o.alignment_violations == 0), len(neg_obs), ) pos_pass_rate = self._safe_div( sum(1 for o in pos_obs if o.alignment_violations == 0), len(pos_obs), ) return { "total_alignment_tests": total, "violation_count": total_violations, "violation_rate": round(self._safe_div(with_violations, total), 4), "cascade_alert_count": with_cascade, "cascade_alert_rate": round(self._safe_div(with_cascade, total), 4), "neg_constraint_pass_rate": round(neg_pass_rate, 4), "pos_constraint_pass_rate": round(pos_pass_rate, 4), } def generate_report(self, collector: MetricsCollector) -> CapabilityReport: """Generate a full capability analysis report from collected observations.""" observations = collector.observations originals = collector.get_original_observations() # Compute overall metrics total = len(originals) if total > 0: tp = sum(1 for o in originals if o.skill_correct is True) fp = sum( 1 for o in originals if o.skill_correct is False and o.actual_skill is not None ) fn = sum( 1 for o in originals if o.skill_correct is False and o.expected_skill is not None ) overall_precision, overall_recall, overall_f1 = self.compute_prf(tp, fp, fn) exec_correct = sum(1 for o in originals if o.execution_mode_correct is True) overall_exec_accuracy = self._safe_div(exec_correct, total) task_success = sum(1 for o in originals if o.task_succeeded) overall_success_rate = self._safe_div(task_success, total) else: overall_precision = overall_recall = overall_f1 = 0.0 overall_exec_accuracy = overall_success_rate = 0.0 # Compute per-category metrics categories: set[tuple[str, str]] = {(o.category, o.subcategory) for o in originals} category_metrics = [ self.analyze_category(observations, cat, subcat) for cat, subcat in sorted(categories) ] # Detect overfitting overfitting_results, overfitting_score = self.detect_overfitting(observations) # Identify weaknesses weaknesses = self.identify_weaknesses(category_metrics, overfitting_results) # Root cause analysis root_causes = self.analyze_root_causes( observations, category_metrics, overfitting_results, weaknesses ) # Improvement strategy planning improvement_plans = self.plan_improvements(weaknesses, root_causes) return CapabilityReport( generated_at=datetime.now(timezone.utc).isoformat(), total_observations=len(observations), overall_skill_recall=round(overall_recall, 4), overall_skill_precision=round(overall_precision, 4), overall_skill_f1=round(overall_f1, 4), overall_execution_mode_accuracy=round(overall_exec_accuracy, 4), overall_task_success_rate=round(overall_success_rate, 4), category_metrics=category_metrics, overfitting_results=overfitting_results, overfitting_score=overfitting_score, weaknesses=weaknesses, root_causes=root_causes, improvement_plans=improvement_plans, raw_observations=observations, ) # ═══════════════════════════════════════════════════════════════════════════ # 4. Metrics Reporter # ═══════════════════════════════════════════════════════════════════════════ class MetricsReporter: """Generate human-readable and machine-readable reports.""" @staticmethod def to_json(report: CapabilityReport, path: str) -> None: """Save report as JSON.""" with open(path, "w", encoding="utf-8") as f: json.dump(report.model_dump(), f, ensure_ascii=False, indent=2) @staticmethod def to_text(report: CapabilityReport) -> str: """Generate plain-text summary report in Chinese.""" lines: list[str] = [] lines.append("=" * 72) lines.append(" AgentKit 智能化能力分析报告") lines.append(f" 生成时间: {report.generated_at}") lines.append("=" * 72) lines.append("") # Overall metrics lines.append("── 总体指标 ──────────────────────────────────────────────") lines.append(f" 观测总数: {report.total_observations}") lines.append(f" 技能路由召回率: {report.overall_skill_recall:.2%}") lines.append(f" 技能路由精确率: {report.overall_skill_precision:.2%}") lines.append(f" 技能路由F1: {report.overall_skill_f1:.2%}") lines.append(f" 执行模式准确率: {report.overall_execution_mode_accuracy:.2%}") lines.append(f" 任务成功率: {report.overall_task_success_rate:.2%}") lines.append(f" 过拟合分数: {report.overfitting_score:.2%}") lines.append("") # Per-category breakdown lines.append("── 分类明细 ──────────────────────────────────────────────") for cm in report.category_metrics: cat_label = { "routing": "路由", "execution": "执行", "quality": "质量", "team": "团队", "consistency": "一致性", }.get(cm.category, cm.category) subcat_label = { "keyword_match": "关键词匹配", "explicit_prefix": "显式前缀", "greeting": "问候语", "identity": "身份识别", "disambiguation": "歧义消解", "fallback": "回退处理", "complexity_low": "低复杂度", "complexity_high": "高复杂度", "intent_variant": "意图变体", "direct_mode": "直接模式", "react_mode": "ReAct模式", "quality_gate": "质量门控", "output_std": "输出标准化", "explicit_team": "显式团队", "deterministic": "确定性", "overfitting": "过拟合", }.get(cm.subcategory, cm.subcategory) lines.append(f" [{cat_label}/{subcat_label}]") lines.append( f" 样本数={cm.total} 召回率={cm.skill_recall:.2%} " f"精确率={cm.skill_precision:.2%} F1={cm.skill_f1:.2%}" ) lines.append( f" 执行模式准确率={cm.execution_mode_accuracy:.2%} " f"成功率={cm.task_success_rate:.2%} " f"平均耗时={cm.avg_response_time_ms:.0f}ms" ) lines.append("") # Overfitting analysis if report.overfitting_results: lines.append("── 过拟合分析 ────────────────────────────────────────────") for r in report.overfitting_results: status = "⚠ 过拟合" if r.is_overfitted else "✓ 正常" orig_label = "✓" if r.original_correct else "✗" lines.append( f" [{status}] {r.benchmark_id}: " f"原始输入={orig_label}, " f"改写一致性={r.consistency_rate:.0%}" ) lines.append("") # Semantic router analysis semantic_cats = [cm for cm in report.category_metrics if cm.category == "semantic_router"] if semantic_cats: lines.append("── 语义路由分析 ──────────────────────────────────────────") for cm in semantic_cats: lines.append( f" [{cm.subcategory}] 样本数={cm.total} " f"精确率={cm.skill_precision:.2%} F1={cm.skill_f1:.2%}" ) lines.append("") # Team routing analysis team_cats = [cm for cm in report.category_metrics if cm.category == "team"] if team_cats: lines.append("── 团队路由分析 ──────────────────────────────────────────") for cm in team_cats: lines.append( f" [{cm.subcategory}] 样本数={cm.total} " f"成功率={cm.task_success_rate:.2%} " f"执行模式准确率={cm.execution_mode_accuracy:.2%}" ) lines.append("") # Alignment guard analysis alignment_obs = [o for o in report.raw_observations if o.category == "alignment"] if alignment_obs: analyzer = MetricsAnalyzer() alignment_metrics = analyzer.analyze_alignment(report.raw_observations) lines.append("── 对齐守卫分析 ──────────────────────────────────────────") lines.append(f" 测试总数: {alignment_metrics['total_alignment_tests']}") lines.append(f" 约束违规总数: {alignment_metrics['violation_count']}") lines.append(f" 违规率: {alignment_metrics['violation_rate']:.2%}") lines.append( f" 否定约束通过率: {alignment_metrics['neg_constraint_pass_rate']:.2%}" ) lines.append( f" 肯定约束通过率: {alignment_metrics['pos_constraint_pass_rate']:.2%}" ) lines.append(f" 级联告警次数: {alignment_metrics['cascade_alert_count']}") lines.append(f" 级联告警率: {alignment_metrics['cascade_alert_rate']:.2%}") lines.append("") # Weakness analysis if report.weaknesses: lines.append("── 智能化短板识别 ────────────────────────────────────────") for w in report.weaknesses: icon = {"critical": "🔴", "high": "🟠", "medium": "🟡", "low": "🟢"}.get( w.severity, "⚪" ) severity_label = { "critical": "严重", "high": "高", "medium": "中", "low": "低", }.get(w.severity, w.severity) lines.append(f" {icon} [{severity_label}] {w.description}") lines.append(f" 证据: {w.evidence}") lines.append(f" 建议: {w.suggestion}") lines.append("") else: lines.append("── 未检测到显著短板 ────────────────────────────────────") lines.append("") # Root cause analysis if report.root_causes: lines.append("── 根因分析 ──────────────────────────────────────────────") cause_type_labels = { "keyword_gap": "关键词覆盖不足", "intent_ambiguous": "意图歧义", "complexity_misjudge": "复杂度估算偏差", "fallback_missing": "回退机制不足", "overfit_pattern": "路由过拟合", "quality_threshold": "质量门控阈值过低", "config_error": "配置/服务端错误", "tool_missing": "工具缺失", } for rc in report.root_causes: type_label = cause_type_labels.get(rc.cause_type, rc.cause_type) conf_bar = "█" * int(rc.confidence * 10) + "░" * (10 - int(rc.confidence * 10)) lines.append(f" ▸ [{type_label}] 置信度: {conf_bar} {rc.confidence:.0%}") lines.append(f" 原因: {rc.cause_description}") if rc.detail: lines.append(f" 详情: {rc.detail}") if rc.affected_cases: lines.append( f" 受影响用例: {', '.join(rc.affected_cases[:5])}" f"{'...' if len(rc.affected_cases) > 5 else ''}" ) lines.append("") # Improvement strategy if report.improvement_plans: lines.append("── 改进策略规划 ──────────────────────────────────────────") for i, plan in enumerate(report.improvement_plans, 1): lines.append(f" ┌─ 策略 {i}: {plan.weakness_description}") lines.append(f" │ 总体策略: {plan.overall_strategy}") lines.append(" │") for action in plan.actions: priority_icon = {"P0": "🔴", "P1": "🟠", "P2": "🟡", "P3": "🟢"}.get( action.priority, "⚪" ) effort_label = {"small": "小", "medium": "中", "large": "大"}.get( action.effort, action.effort ) lines.append(f" │ {priority_icon} [{action.priority}] {action.title}") lines.append(f" │ 目标模块: {action.target_module}") lines.append(f" │ 具体操作: {action.description}") lines.append(f" │ 预期影响: {action.expected_impact}") lines.append(f" │ 工作量: {effort_label}") lines.append(f" │ 验证方式: {action.verification}") lines.append(" │") lines.append(f" └{'─' * 60}") lines.append("") # L3 Output Quality Evaluation if report.output_quality_evaluations: lines.append("── L3 输出质量评估 ──────────────────────────────────────────") evaluated = [e for e in report.output_quality_evaluations if e.evaluated] if evaluated: avg_score = sum(e.quality_score for e in evaluated) / len(evaluated) lines.append(f" 评估样本数: {len(evaluated)}") lines.append(f" 平均质量评分: {avg_score:.2f}/5.0") score_dist = {1: 0, 2: 0, 3: 0, 4: 0, 5: 0} for e in evaluated: bucket = max(1, min(5, int(e.quality_score))) score_dist[bucket] += 1 lines.append(f" 评分分布: 1分:{score_dist[1]} 2分:{score_dist[2]} 3分:{score_dist[3]} 4分:{score_dist[4]} 5分:{score_dist[5]}") # Show some examples lines.append("") lines.append(" 样例:") for e in evaluated[:5]: lines.append(f" [{e.benchmark_id}] 评分={e.quality_score:.0f} 期望={e.expected_skill} 实际={e.actual_skill}") if e.reasoning: lines.append(f" 理由: {e.reasoning}") else: lines.append(" 无有效评估结果") lines.append("") # L5 Adaptive Capability (reuse overfitting consistency data) if report.overfitting_results: lines.append("── L5 自适应能力 ──────────────────────────────────────────") consistency_rates = [r.consistency_rate for r in report.overfitting_results] if consistency_rates: avg_consistency = sum(consistency_rates) / len(consistency_rates) lines.append(f" 测试组数: {len(consistency_rates)}") lines.append(f" 平均自适应率: {avg_consistency:.2%}") high_adapt = sum(1 for r in consistency_rates if r >= 0.8) lines.append(f" 高自适应(>=80%): {high_adapt}/{len(consistency_rates)}") lines.append("") lines.append("=" * 72) return "\n".join(lines) @staticmethod def save_report(report: CapabilityReport, output_dir: str) -> dict[str, str]: """Save both JSON and text reports. Returns paths to saved files.""" os.makedirs(output_dir, exist_ok=True) json_path = os.path.join(output_dir, "capability_report.json") text_path = os.path.join(output_dir, "capability_report.txt") MetricsReporter.to_json(report, json_path) with open(text_path, "w", encoding="utf-8") as f: f.write(MetricsReporter.to_text(report)) return {"json": json_path, "text": text_path}