2017 lines
55 KiB
JSON
2017 lines
55 KiB
JSON
{
|
||
"timestamp": "2026-06-17T04:52:53.863927+00:00",
|
||
"version": "0.1.0",
|
||
"mode": "all",
|
||
"runs": 1,
|
||
"fast": false,
|
||
"overall_accuracy": 0.9524,
|
||
"overall_accuracy_mean": 0.9524,
|
||
"overall_accuracy_std": 0.0,
|
||
"summary": "60/63 tests passed (3 failed) across 9 dimensions.",
|
||
"dimensions": {
|
||
"preprocessing": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0128,
|
||
"latency_p95_ms": 0.057,
|
||
"latency_p99_ms": 0.1086,
|
||
"consistency": 1.0,
|
||
"total": 15,
|
||
"passed": 15,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.7961,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"greeting": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0133,
|
||
"latency_p95_ms": 0.026,
|
||
"latency_p99_ms": 0.0275,
|
||
"consistency": 1.0,
|
||
"total": 4,
|
||
"passed": 4,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5101,
|
||
"ci_upper": 1.0
|
||
},
|
||
"tool_query": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0115,
|
||
"latency_p95_ms": 0.0166,
|
||
"latency_p99_ms": 0.0172,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"skill_prefix": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0294,
|
||
"latency_p95_ms": 0.1123,
|
||
"latency_p99_ms": 0.1197,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
},
|
||
"complex": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0101,
|
||
"latency_p95_ms": 0.0125,
|
||
"latency_p99_ms": 0.0127,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0115,
|
||
"latency_p95_ms": 0.0253,
|
||
"latency_p99_ms": 0.0274,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0136,
|
||
"latency_p95_ms": 0.0263,
|
||
"latency_p99_ms": 0.0288,
|
||
"consistency": 1.0,
|
||
"total": 7,
|
||
"passed": 7,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6457,
|
||
"ci_upper": 1.0
|
||
},
|
||
"hard": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0128,
|
||
"latency_p95_ms": 0.1106,
|
||
"latency_p99_ms": 0.1193,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "prep-001",
|
||
"dimension": "preprocessing",
|
||
"category": "greeting",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0279,
|
||
"root_cause": "none",
|
||
"detail": "input='你好' method=regex_direct",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-002",
|
||
"dimension": "preprocessing",
|
||
"category": "greeting",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0151,
|
||
"root_cause": "none",
|
||
"detail": "input='hello' method=regex_direct",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-003",
|
||
"dimension": "preprocessing",
|
||
"category": "greeting",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0111,
|
||
"root_cause": "none",
|
||
"detail": "input='谢谢' method=regex_direct",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-004",
|
||
"dimension": "preprocessing",
|
||
"category": "greeting",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0115,
|
||
"root_cause": "none",
|
||
"detail": "input='你是谁' method=regex_direct",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-005",
|
||
"dimension": "preprocessing",
|
||
"category": "tool_query",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0136,
|
||
"root_cause": "none",
|
||
"detail": "input='搜索golang教程' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-006",
|
||
"dimension": "preprocessing",
|
||
"category": "tool_query",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0115,
|
||
"root_cause": "none",
|
||
"detail": "input='执行ls命令' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-007",
|
||
"dimension": "preprocessing",
|
||
"category": "tool_query",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0174,
|
||
"root_cause": "none",
|
||
"detail": "input='翻译hello为中文' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-008",
|
||
"dimension": "preprocessing",
|
||
"category": "tool_query",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0113,
|
||
"root_cause": "none",
|
||
"detail": "input='什么是机器学习' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-009",
|
||
"dimension": "preprocessing",
|
||
"category": "tool_query",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0109,
|
||
"root_cause": "none",
|
||
"detail": "input='帮我分析数据' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-010",
|
||
"dimension": "preprocessing",
|
||
"category": "skill_prefix",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "skill_react",
|
||
"actual": "skill_react",
|
||
"duration_ms": 0.0294,
|
||
"root_cause": "none",
|
||
"detail": "input='@skill:react_agent 查看ip' method=skill_prefix",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-011",
|
||
"dimension": "preprocessing",
|
||
"category": "skill_prefix",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0191,
|
||
"root_cause": "none",
|
||
"detail": "input='@skill:chat_only 你好' method=skill_prefix",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-012",
|
||
"dimension": "preprocessing",
|
||
"category": "skill_prefix",
|
||
"difficulty": "hard",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.1215,
|
||
"root_cause": "none",
|
||
"detail": "input='@skill:nonexistent 做点什么' method=skill_not_found_fallback",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-013",
|
||
"dimension": "preprocessing",
|
||
"category": "complex",
|
||
"difficulty": "hard",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0101,
|
||
"root_cause": "none",
|
||
"detail": "input='帮我分析这个数据并生成报告' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-014",
|
||
"dimension": "preprocessing",
|
||
"category": "complex",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0099,
|
||
"root_cause": "none",
|
||
"detail": "input='随便聊聊' method=default_react",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "prep-015",
|
||
"dimension": "preprocessing",
|
||
"category": "complex",
|
||
"difficulty": "hard",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0128,
|
||
"root_cause": "none",
|
||
"detail": "input='请帮我完成以下任务：1. 查询天气 2. 生成报告' method=default_react",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"overfitting": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.025,
|
||
"latency_p95_ms": 0.0557,
|
||
"latency_p99_ms": 0.0596,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"ip_check": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0362,
|
||
"latency_p95_ms": 0.0362,
|
||
"latency_p99_ms": 0.0362,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"search": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0243,
|
||
"latency_p95_ms": 0.0243,
|
||
"latency_p99_ms": 0.0243,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"greeting": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0606,
|
||
"latency_p95_ms": 0.0606,
|
||
"latency_p99_ms": 0.0606,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"tool_use": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0233,
|
||
"latency_p95_ms": 0.0233,
|
||
"latency_p99_ms": 0.0233,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"complex": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.025,
|
||
"latency_p95_ms": 0.025,
|
||
"latency_p99_ms": 0.025,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0243,
|
||
"latency_p95_ms": 0.035,
|
||
"latency_p99_ms": 0.036,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
},
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0606,
|
||
"latency_p95_ms": 0.0606,
|
||
"latency_p99_ms": 0.0606,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"hard": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.025,
|
||
"latency_p95_ms": 0.025,
|
||
"latency_p99_ms": 0.025,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "over-001",
|
||
"dimension": "overfitting",
|
||
"category": "ip_check",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0362,
|
||
"root_cause": "none",
|
||
"detail": "paraphrases=5 modes=['react', 'react', 'react', 'react', 'react']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "over-002",
|
||
"dimension": "overfitting",
|
||
"category": "search",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0243,
|
||
"root_cause": "none",
|
||
"detail": "paraphrases=3 modes=['react', 'react', 'react']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "over-003",
|
||
"dimension": "overfitting",
|
||
"category": "greeting",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "direct_chat",
|
||
"actual": "direct_chat",
|
||
"duration_ms": 0.0606,
|
||
"root_cause": "none",
|
||
"detail": "paraphrases=5 modes=['direct_chat', 'direct_chat', 'direct_chat', 'direct_chat', 'direct_chat']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "over-004",
|
||
"dimension": "overfitting",
|
||
"category": "tool_use",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.0233,
|
||
"root_cause": "none",
|
||
"detail": "paraphrases=3 modes=['react', 'react', 'react']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "over-005",
|
||
"dimension": "overfitting",
|
||
"category": "complex",
|
||
"difficulty": "hard",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "react",
|
||
"duration_ms": 0.025,
|
||
"root_cause": "none",
|
||
"detail": "paraphrases=3 modes=['react', 'react', 'react']",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"efficiency": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.33,
|
||
"latency_p95_ms": 0.622,
|
||
"latency_p99_ms": 0.6604,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"preprocess_latency": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.33,
|
||
"latency_p95_ms": 0.42,
|
||
"latency_p99_ms": 0.428,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
},
|
||
"tool_search_latency": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.355,
|
||
"latency_p95_ms": 0.6385,
|
||
"latency_p99_ms": 0.6637,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.165,
|
||
"latency_p95_ms": 0.2775,
|
||
"latency_p99_ms": 0.2875,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.43,
|
||
"latency_p95_ms": 0.646,
|
||
"latency_p99_ms": 0.6652,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "eff-001",
|
||
"dimension": "efficiency",
|
||
"category": "preprocess_latency",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "<=50ms",
|
||
"actual": "0.003ms",
|
||
"duration_ms": 0.29,
|
||
"root_cause": "none",
|
||
"detail": "iterations=100 avg=0.003ms threshold=50.0ms",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "eff-002",
|
||
"dimension": "efficiency",
|
||
"category": "preprocess_latency",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "<=50ms",
|
||
"actual": "0.003ms",
|
||
"duration_ms": 0.33,
|
||
"root_cause": "none",
|
||
"detail": "iterations=100 avg=0.003ms threshold=50.0ms",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "eff-003",
|
||
"dimension": "efficiency",
|
||
"category": "preprocess_latency",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "<=50ms",
|
||
"actual": "0.004ms",
|
||
"duration_ms": 0.43,
|
||
"root_cause": "none",
|
||
"detail": "iterations=100 avg=0.004ms threshold=50.0ms",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "eff-004",
|
||
"dimension": "efficiency",
|
||
"category": "tool_search_latency",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "<=10ms",
|
||
"actual": "0.007ms",
|
||
"duration_ms": 0.67,
|
||
"root_cause": "none",
|
||
"detail": "iterations=100 avg=0.007ms threshold=10.0ms",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "eff-005",
|
||
"dimension": "efficiency",
|
||
"category": "tool_search_latency",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "<=5ms",
|
||
"actual": "0.000ms",
|
||
"duration_ms": 0.04,
|
||
"root_cause": "none",
|
||
"detail": "iterations=100 avg=0.000ms threshold=5.0ms",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"tool_search": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.8333,
|
||
"recall": 0.8333,
|
||
"f1": 0.8333,
|
||
"latency_p50_ms": 0.0192,
|
||
"latency_p95_ms": 0.0278,
|
||
"latency_p99_ms": 0.0326,
|
||
"consistency": 1.0,
|
||
"total": 10,
|
||
"passed": 10,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.7225,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"exact_match": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0199,
|
||
"latency_p95_ms": 0.0203,
|
||
"latency_p99_ms": 0.0204,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"fuzzy_match": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0264,
|
||
"latency_p95_ms": 0.0331,
|
||
"latency_p99_ms": 0.0337,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"no_match": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.0118,
|
||
"latency_p95_ms": 0.0122,
|
||
"latency_p99_ms": 0.0123,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"top_k": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.016,
|
||
"latency_p95_ms": 0.016,
|
||
"latency_p99_ms": 0.016,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.8333,
|
||
"recall": 0.8333,
|
||
"f1": 0.8333,
|
||
"latency_p50_ms": 0.0194,
|
||
"latency_p95_ms": 0.0203,
|
||
"latency_p99_ms": 0.0204,
|
||
"consistency": 1.0,
|
||
"total": 7,
|
||
"passed": 7,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6457,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.019,
|
||
"latency_p95_ms": 0.0323,
|
||
"latency_p99_ms": 0.0335,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "ts-001",
|
||
"dimension": "tool_search",
|
||
"category": "exact_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "read_file",
|
||
"actual": "read_file",
|
||
"duration_ms": 0.0199,
|
||
"root_cause": "none",
|
||
"detail": "query='read file' top_k=5 results=2",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-002",
|
||
"dimension": "tool_search",
|
||
"category": "exact_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "write_file",
|
||
"actual": "write_file",
|
||
"duration_ms": 0.0204,
|
||
"root_cause": "none",
|
||
"detail": "query='write file content' top_k=5 results=2",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-003",
|
||
"dimension": "tool_search",
|
||
"category": "exact_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "web_search",
|
||
"actual": "web_search",
|
||
"duration_ms": 0.02,
|
||
"root_cause": "none",
|
||
"detail": "query='search web information' top_k=5 results=2",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-004",
|
||
"dimension": "tool_search",
|
||
"category": "exact_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "shell_exec",
|
||
"actual": "shell_exec",
|
||
"duration_ms": 0.018,
|
||
"root_cause": "none",
|
||
"detail": "query='execute shell command' top_k=5 results=1",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-005",
|
||
"dimension": "tool_search",
|
||
"category": "exact_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "http_request",
|
||
"actual": "http_request",
|
||
"duration_ms": 0.0194,
|
||
"root_cause": "none",
|
||
"detail": "query='send http request url' top_k=5 results=1",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-006",
|
||
"dimension": "tool_search",
|
||
"category": "fuzzy_match",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "read_file",
|
||
"actual": "read_file",
|
||
"duration_ms": 0.0338,
|
||
"root_cause": "none",
|
||
"detail": "query='io file' top_k=5 results=2",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-007",
|
||
"dimension": "tool_search",
|
||
"category": "fuzzy_match",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "web_search",
|
||
"actual": "web_search",
|
||
"duration_ms": 0.019,
|
||
"root_cause": "none",
|
||
"detail": "query='search query engine' top_k=5 results=1",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-008",
|
||
"dimension": "tool_search",
|
||
"category": "no_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "__none__",
|
||
"actual": "[]",
|
||
"duration_ms": 0.0112,
|
||
"root_cause": "none",
|
||
"detail": "query='' top_k=5 results=0",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-009",
|
||
"dimension": "tool_search",
|
||
"category": "no_match",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "__none__",
|
||
"actual": "[]",
|
||
"duration_ms": 0.0123,
|
||
"root_cause": "none",
|
||
"detail": "query='zzzznonexistent' top_k=5 results=0",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ts-010",
|
||
"dimension": "tool_search",
|
||
"category": "top_k",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "read_file",
|
||
"actual": "read_file",
|
||
"duration_ms": 0.016,
|
||
"root_cause": "none",
|
||
"detail": "query='file' top_k=1 results=1",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"event_model": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.057,
|
||
"latency_p95_ms": 15.9984,
|
||
"latency_p99_ms": 20.2369,
|
||
"consistency": 1.0,
|
||
"total": 6,
|
||
"passed": 6,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6097,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"sq_lifecycle": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.046,
|
||
"latency_p95_ms": 0.0982,
|
||
"latency_p99_ms": 0.1028,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
},
|
||
"eq_lifecycle": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.0681,
|
||
"latency_p95_ms": 19.1737,
|
||
"latency_p99_ms": 20.8719,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.057,
|
||
"latency_p95_ms": 15.9984,
|
||
"latency_p99_ms": 20.2369,
|
||
"consistency": 1.0,
|
||
"total": 6,
|
||
"passed": 6,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6097,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "ev-001",
|
||
"dimension": "event_model",
|
||
"category": "sq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "drained=['hello']",
|
||
"duration_ms": 0.104,
|
||
"root_cause": "none",
|
||
"detail": "task_id=09dccea9...",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ev-002",
|
||
"dimension": "event_model",
|
||
"category": "sq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "cancelled=True",
|
||
"duration_ms": 0.046,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ev-003",
|
||
"dimension": "event_model",
|
||
"category": "sq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "raised=True closed=True",
|
||
"duration_ms": 0.0115,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ev-004",
|
||
"dimension": "event_model",
|
||
"category": "eq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "received=1",
|
||
"duration_ms": 0.0681,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ev-005",
|
||
"dimension": "event_model",
|
||
"category": "eq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "events=1 closed=True",
|
||
"duration_ms": 21.2965,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "ev-006",
|
||
"dimension": "event_model",
|
||
"category": "eq_lifecycle",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "subscribers=0",
|
||
"duration_ms": 0.007,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"spec_management": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 1.3834,
|
||
"latency_p95_ms": 3.4578,
|
||
"latency_p99_ms": 4.0077,
|
||
"consistency": 1.0,
|
||
"total": 7,
|
||
"passed": 7,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6457,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"crud": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 1.3834,
|
||
"latency_p95_ms": 3.6044,
|
||
"latency_p99_ms": 4.037,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"edge": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.9497,
|
||
"latency_p95_ms": 1.7635,
|
||
"latency_p99_ms": 1.8358,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 1.3659,
|
||
"latency_p95_ms": 3.4693,
|
||
"latency_p99_ms": 4.01,
|
||
"consistency": 1.0,
|
||
"total": 6,
|
||
"passed": 6,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.6097,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 1.8539,
|
||
"latency_p95_ms": 1.8539,
|
||
"latency_p99_ms": 1.8539,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "sm-001",
|
||
"dimension": "spec_management",
|
||
"category": "crud",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "exists=True",
|
||
"duration_ms": 1.3484,
|
||
"root_cause": "none",
|
||
"detail": "path=/var/folders/6b/ljk5bdq50yxcsth24frf05200000gn/T/agentkit-benchmark-wll_nqgl/run-0/specs/sm-001/test-spec.yaml",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-002",
|
||
"dimension": "spec_management",
|
||
"category": "crud",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "steps=2",
|
||
"duration_ms": 1.3834,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-003",
|
||
"dimension": "spec_management",
|
||
"category": "crud",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "goal=Updated goal",
|
||
"duration_ms": 1.4414,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-004",
|
||
"dimension": "spec_management",
|
||
"category": "crud",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "deleted=True remaining=0",
|
||
"duration_ms": 1.0766,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-005",
|
||
"dimension": "spec_management",
|
||
"category": "crud",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "count=2",
|
||
"duration_ms": 4.1452,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-006",
|
||
"dimension": "spec_management",
|
||
"category": "edge",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "status=confirmed",
|
||
"duration_ms": 1.8539,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "sm-007",
|
||
"dimension": "spec_management",
|
||
"category": "edge",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "result=None",
|
||
"duration_ms": 0.0454,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"verification": {
|
||
"metrics": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 22.0041,
|
||
"latency_p95_ms": 411.5705,
|
||
"latency_p99_ms": 487.0649,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 5,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.5655,
|
||
"ci_upper": 1.0
|
||
},
|
||
"by_category": {
|
||
"basic": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 11.4916,
|
||
"latency_p95_ms": 11.8303,
|
||
"latency_p99_ms": 11.8604,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"retry": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 34.0985,
|
||
"latency_p95_ms": 34.0985,
|
||
"latency_p99_ms": 34.0985,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"timeout": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 505.9385,
|
||
"latency_p95_ms": 505.9385,
|
||
"latency_p99_ms": 505.9385,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"multi": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 22.0041,
|
||
"latency_p95_ms": 22.0041,
|
||
"latency_p99_ms": 22.0041,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 11.4916,
|
||
"latency_p95_ms": 11.8303,
|
||
"latency_p99_ms": 11.8604,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 34.0985,
|
||
"latency_p95_ms": 458.7545,
|
||
"latency_p99_ms": 496.5017,
|
||
"consistency": 1.0,
|
||
"total": 3,
|
||
"passed": 3,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.4385,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "vf-001",
|
||
"dimension": "verification",
|
||
"category": "basic",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "passed=True attempts=1",
|
||
"duration_ms": 11.8679,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "vf-002",
|
||
"dimension": "verification",
|
||
"category": "basic",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "passed=False errors=1",
|
||
"duration_ms": 11.1154,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "vf-003",
|
||
"dimension": "verification",
|
||
"category": "retry",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "attempts=3 callbacks=2",
|
||
"duration_ms": 34.0985,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "vf-004",
|
||
"dimension": "verification",
|
||
"category": "timeout",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "passed=False errors=1",
|
||
"duration_ms": 505.9385,
|
||
"root_cause": "none",
|
||
"detail": "errors=['Command timed out after 0.5s: sleep 10']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "vf-005",
|
||
"dimension": "verification",
|
||
"category": "multi",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "passed",
|
||
"actual": "passed=False",
|
||
"duration_ms": 22.0041,
|
||
"root_cause": "none",
|
||
"detail": "",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"llm_reasoning": {
|
||
"metrics": {
|
||
"accuracy": 0.6,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 25149.4865,
|
||
"latency_p95_ms": 30001.1677,
|
||
"latency_p99_ms": 30001.2291,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 3,
|
||
"failed": 2,
|
||
"accuracy_mean": 0.6,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2307,
|
||
"ci_upper": 0.8824
|
||
},
|
||
"by_category": {
|
||
"intent_understanding": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 21288.4177,
|
||
"latency_p95_ms": 21288.4177,
|
||
"latency_p99_ms": 21288.4177,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"tool_selection": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 5894.9682,
|
||
"latency_p95_ms": 5894.9682,
|
||
"latency_p99_ms": 5894.9682,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"multi_step": {
|
||
"accuracy": 0.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 30000.8609,
|
||
"latency_p95_ms": 30000.8609,
|
||
"latency_p99_ms": 30000.8609,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 0,
|
||
"failed": 1,
|
||
"accuracy_mean": 0.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.0,
|
||
"ci_upper": 0.7935
|
||
},
|
||
"code_generation": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 25149.4865,
|
||
"latency_p95_ms": 25149.4865,
|
||
"latency_p99_ms": 25149.4865,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"error_recovery": {
|
||
"accuracy": 0.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 30001.2444,
|
||
"latency_p95_ms": 30001.2444,
|
||
"latency_p99_ms": 30001.2444,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 0,
|
||
"failed": 1,
|
||
"accuracy_mean": 0.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.0,
|
||
"ci_upper": 0.7935
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 21288.4177,
|
||
"latency_p95_ms": 21288.4177,
|
||
"latency_p99_ms": 21288.4177,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 15522.2273,
|
||
"latency_p95_ms": 24186.7606,
|
||
"latency_p99_ms": 24956.9413,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"hard": {
|
||
"accuracy": 0.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 30001.0526,
|
||
"latency_p95_ms": 30001.2252,
|
||
"latency_p99_ms": 30001.2406,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 0,
|
||
"failed": 2,
|
||
"accuracy_mean": 0.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.0,
|
||
"ci_upper": 0.6576
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "llm-001",
|
||
"dimension": "llm_reasoning",
|
||
"category": "intent_understanding",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "mode=react tokens=1116 len=974",
|
||
"duration_ms": 21288.4177,
|
||
"root_cause": "none",
|
||
"detail": "mode=react keywords=['ip', '地址', 'ifconfig', 'hostname', '网络']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "llm-002",
|
||
"dimension": "llm_reasoning",
|
||
"category": "tool_selection",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "mode=react tokens=205 len=87",
|
||
"duration_ms": 5894.9682,
|
||
"root_cause": "none",
|
||
"detail": "mode=react keywords=['search', '搜索', 'web', '论文', 'paper', 'agent']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "llm-003",
|
||
"dimension": "llm_reasoning",
|
||
"category": "multi_step",
|
||
"difficulty": "hard",
|
||
"passed": false,
|
||
"expected": "react",
|
||
"actual": "timeout",
|
||
"duration_ms": 30000.8609,
|
||
"root_cause": "timeout",
|
||
"detail": "LLM call timed out after 30s",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "llm-004",
|
||
"dimension": "llm_reasoning",
|
||
"category": "code_generation",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "react",
|
||
"actual": "mode=react tokens=1359 len=1001",
|
||
"duration_ms": 25149.4865,
|
||
"root_cause": "none",
|
||
"detail": "mode=react keywords=['def', 'fib', 'return', 'python']",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "llm-005",
|
||
"dimension": "llm_reasoning",
|
||
"category": "error_recovery",
|
||
"difficulty": "hard",
|
||
"passed": false,
|
||
"expected": "react",
|
||
"actual": "timeout",
|
||
"duration_ms": 30001.2444,
|
||
"root_cause": "timeout",
|
||
"detail": "LLM call timed out after 30s",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
},
|
||
"gui_integration": {
|
||
"metrics": {
|
||
"accuracy": 0.8,
|
||
"precision": 0.8,
|
||
"recall": 0.8,
|
||
"f1": 0.8,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 5,
|
||
"passed": 4,
|
||
"failed": 1,
|
||
"accuracy_mean": 0.8,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3755,
|
||
"ci_upper": 0.9638
|
||
},
|
||
"by_category": {
|
||
"service_startup": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
},
|
||
"api_availability": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"websocket": {
|
||
"accuracy": 0.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 0,
|
||
"failed": 1,
|
||
"accuracy_mean": 0.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.0,
|
||
"ci_upper": 0.7935
|
||
},
|
||
"frontend": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 1,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.2065,
|
||
"ci_upper": 1.0
|
||
}
|
||
},
|
||
"by_difficulty": {
|
||
"easy": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"medium": {
|
||
"accuracy": 1.0,
|
||
"precision": 1.0,
|
||
"recall": 1.0,
|
||
"f1": 1.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 2,
|
||
"passed": 2,
|
||
"failed": 0,
|
||
"accuracy_mean": 1.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.3424,
|
||
"ci_upper": 1.0
|
||
},
|
||
"hard": {
|
||
"accuracy": 0.0,
|
||
"precision": 0.0,
|
||
"recall": 0.0,
|
||
"f1": 0.0,
|
||
"latency_p50_ms": 0.0,
|
||
"latency_p95_ms": 0.0,
|
||
"latency_p99_ms": 0.0,
|
||
"consistency": 1.0,
|
||
"total": 1,
|
||
"passed": 0,
|
||
"failed": 1,
|
||
"accuracy_mean": 0.0,
|
||
"accuracy_std": 0.0,
|
||
"ci_lower": 0.0,
|
||
"ci_upper": 0.7935
|
||
}
|
||
},
|
||
"cases": [
|
||
{
|
||
"task_id": "gui-001",
|
||
"dimension": "gui_integration",
|
||
"category": "service_startup",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "started",
|
||
"actual": "started",
|
||
"duration_ms": 0.0,
|
||
"root_cause": "none",
|
||
"detail": "port=64767 pid=20993",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "gui-002",
|
||
"dimension": "gui_integration",
|
||
"category": "api_availability",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "200",
|
||
"actual": "200",
|
||
"duration_ms": 0.0,
|
||
"root_cause": "none",
|
||
"detail": "health=200 skills=200",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "gui-003",
|
||
"dimension": "gui_integration",
|
||
"category": "api_availability",
|
||
"difficulty": "medium",
|
||
"passed": true,
|
||
"expected": "reachable",
|
||
"actual": "reachable",
|
||
"duration_ms": 0.0,
|
||
"root_cause": "none",
|
||
"detail": "status=405",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "gui-004",
|
||
"dimension": "gui_integration",
|
||
"category": "websocket",
|
||
"difficulty": "hard",
|
||
"passed": false,
|
||
"expected": "connected",
|
||
"actual": "failed",
|
||
"duration_ms": 0.0,
|
||
"root_cause": "gui_failure",
|
||
"detail": "error: server rejected WebSocket connection: HTTP 403",
|
||
"consistency": 1.0
|
||
},
|
||
{
|
||
"task_id": "gui-005",
|
||
"dimension": "gui_integration",
|
||
"category": "frontend",
|
||
"difficulty": "easy",
|
||
"passed": true,
|
||
"expected": "html",
|
||
"actual": "html",
|
||
"duration_ms": 0.0,
|
||
"root_cause": "none",
|
||
"detail": "status=200 len=465",
|
||
"consistency": 1.0
|
||
}
|
||
]
|
||
}
|
||
},
|
||
"baseline_comparison": {
|
||
"status": "compared",
|
||
"dimensions": {
|
||
"preprocessing": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"overfitting": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"efficiency": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"tool_search": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"event_model": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"spec_management": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"verification": {
|
||
"baseline_accuracy": 1.0,
|
||
"current_accuracy": 1.0,
|
||
"change": 0.0,
|
||
"direction": "—"
|
||
},
|
||
"llm_reasoning": {
|
||
"baseline_accuracy": 0.0,
|
||
"current_accuracy": 0.6,
|
||
"change": 0.6,
|
||
"direction": "↑"
|
||
},
|
||
"gui_integration": {
|
||
"baseline_accuracy": 0.0,
|
||
"current_accuracy": 0.8,
|
||
"change": 0.8,
|
||
"direction": "↑"
|
||
}
|
||
}
|
||
}
|
||
} |