{ "timestamp": "2026-06-17T03:54:43.123142+00:00", "version": "0.1.0", "runs": 1, "fast": false, "overall_accuracy": 1.0, "overall_accuracy_mean": 1.0, "overall_accuracy_std": 0.0, "summary": "All 53 tests passed across 7 dimensions.", "dimensions": { "preprocessing": { "metrics": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.016, "latency_p95_ms": 0.4208, "latency_p99_ms": 1.1294, "consistency": 1.0, "total": 15, "passed": 15, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.7961, "ci_upper": 1.0 }, "by_category": { "greeting": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0196, "latency_p95_ms": 0.0241, "latency_p99_ms": 0.0243, "consistency": 1.0, "total": 4, "passed": 4, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5101, "ci_upper": 1.0 }, "tool_query": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0153, "latency_p95_ms": 0.0162, "latency_p99_ms": 0.0164, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "skill_prefix": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0412, "latency_p95_ms": 1.1801, "latency_p99_ms": 1.2813, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 }, "complex": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0147, "latency_p95_ms": 0.0148, "latency_p99_ms": 0.0148, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.017, "latency_p95_ms": 0.0239, "latency_p99_ms": 0.0243, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "medium": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0156, "latency_p95_ms": 0.0367, "latency_p99_ms": 0.0403, "consistency": 1.0, "total": 7, "passed": 7, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6457, "ci_upper": 1.0 }, "hard": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0147, "latency_p95_ms": 1.1774, "latency_p99_ms": 1.2808, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "cases": [ { "task_id": "prep-001", "dimension": "preprocessing", "category": "greeting", "difficulty": "easy", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.0221, "root_cause": "none", "detail": "input='你好' method=regex_direct", "consistency": 1.0 }, { "task_id": "prep-002", "dimension": "preprocessing", "category": "greeting", "difficulty": "easy", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.0244, "root_cause": "none", "detail": "input='hello' method=regex_direct", "consistency": 1.0 }, { "task_id": "prep-003", "dimension": "preprocessing", "category": "greeting", "difficulty": "easy", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.017, "root_cause": "none", "detail": "input='谢谢' method=regex_direct", "consistency": 1.0 }, { "task_id": "prep-004", "dimension": "preprocessing", "category": "greeting", "difficulty": "easy", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.016, "root_cause": "none", "detail": "input='你是谁' method=regex_direct", "consistency": 1.0 }, { "task_id": "prep-005", "dimension": "preprocessing", "category": "tool_query", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0164, "root_cause": "none", "detail": "input='搜索golang教程' method=default_react", "consistency": 1.0 }, { "task_id": "prep-006", "dimension": "preprocessing", "category": "tool_query", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0156, "root_cause": "none", "detail": "input='执行ls命令' method=default_react", "consistency": 1.0 }, { "task_id": "prep-007", "dimension": "preprocessing", "category": "tool_query", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0153, "root_cause": "none", "detail": "input='翻译hello为中文' method=default_react", "consistency": 1.0 }, { "task_id": "prep-008", "dimension": "preprocessing", "category": "tool_query", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.014, "root_cause": "none", "detail": "input='什么是机器学习' method=default_react", "consistency": 1.0 }, { "task_id": "prep-009", "dimension": "preprocessing", "category": "tool_query", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0148, "root_cause": "none", "detail": "input='帮我分析数据' method=default_react", "consistency": 1.0 }, { "task_id": "prep-010", "dimension": "preprocessing", "category": "skill_prefix", "difficulty": "medium", "passed": true, "expected": "skill_react", "actual": "skill_react", "duration_ms": 0.0412, "root_cause": "none", "detail": "input='@skill:react_agent 查看ip' method=skill_prefix", "consistency": 1.0 }, { "task_id": "prep-011", "dimension": "preprocessing", "category": "skill_prefix", "difficulty": "medium", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.0262, "root_cause": "none", "detail": "input='@skill:chat_only 你好' method=skill_prefix", "consistency": 1.0 }, { "task_id": "prep-012", "dimension": "preprocessing", "category": "skill_prefix", "difficulty": "hard", "passed": true, "expected": "react", "actual": "react", "duration_ms": 1.3066, "root_cause": "none", "detail": "input='@skill:nonexistent 做点什么' method=skill_not_found_fallback", "consistency": 1.0 }, { "task_id": "prep-013", "dimension": "preprocessing", "category": "complex", "difficulty": "hard", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0147, "root_cause": "none", "detail": "input='帮我分析这个数据并生成报告' method=default_react", "consistency": 1.0 }, { "task_id": "prep-014", "dimension": "preprocessing", "category": "complex", "difficulty": "easy", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0148, "root_cause": "none", "detail": "input='随便聊聊' method=default_react", "consistency": 1.0 }, { "task_id": "prep-015", "dimension": "preprocessing", "category": "complex", "difficulty": "hard", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0132, "root_cause": "none", "detail": "input='请帮我完成以下任务：1. 查询天气 2. 生成报告' method=default_react", "consistency": 1.0 } ] }, "overfitting": { "metrics": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0295, "latency_p95_ms": 0.0396, "latency_p99_ms": 0.0401, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "by_category": { "ip_check": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0402, "latency_p95_ms": 0.0402, "latency_p99_ms": 0.0402, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "search": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0282, "latency_p95_ms": 0.0282, "latency_p99_ms": 0.0282, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "greeting": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0373, "latency_p95_ms": 0.0373, "latency_p99_ms": 0.0373, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "tool_use": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0295, "latency_p95_ms": 0.0295, "latency_p99_ms": 0.0295, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "complex": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0249, "latency_p95_ms": 0.0249, "latency_p99_ms": 0.0249, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 } }, "by_difficulty": { "medium": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0295, "latency_p95_ms": 0.0391, "latency_p99_ms": 0.04, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 }, "easy": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0373, "latency_p95_ms": 0.0373, "latency_p99_ms": 0.0373, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "hard": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0249, "latency_p95_ms": 0.0249, "latency_p99_ms": 0.0249, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 } }, "cases": [ { "task_id": "over-001", "dimension": "overfitting", "category": "ip_check", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0402, "root_cause": "none", "detail": "paraphrases=5 modes=['react', 'react', 'react', 'react', 'react']", "consistency": 1.0 }, { "task_id": "over-002", "dimension": "overfitting", "category": "search", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0282, "root_cause": "none", "detail": "paraphrases=3 modes=['react', 'react', 'react']", "consistency": 1.0 }, { "task_id": "over-003", "dimension": "overfitting", "category": "greeting", "difficulty": "easy", "passed": true, "expected": "direct_chat", "actual": "direct_chat", "duration_ms": 0.0373, "root_cause": "none", "detail": "paraphrases=5 modes=['direct_chat', 'direct_chat', 'direct_chat', 'direct_chat', 'direct_chat']", "consistency": 1.0 }, { "task_id": "over-004", "dimension": "overfitting", "category": "tool_use", "difficulty": "medium", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0295, "root_cause": "none", "detail": "paraphrases=3 modes=['react', 'react', 'react']", "consistency": 1.0 }, { "task_id": "over-005", "dimension": "overfitting", "category": "complex", "difficulty": "hard", "passed": true, "expected": "react", "actual": "react", "duration_ms": 0.0249, "root_cause": "none", "detail": "paraphrases=3 modes=['react', 'react', 'react']", "consistency": 1.0 } ] }, "efficiency": { "metrics": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.33, "latency_p95_ms": 0.602, "latency_p99_ms": 0.6404, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "by_category": { "preprocess_latency": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.33, "latency_p95_ms": 0.402, "latency_p99_ms": 0.4084, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 }, "tool_search_latency": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.345, "latency_p95_ms": 0.6195, "latency_p99_ms": 0.6439, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.16, "latency_p95_ms": 0.268, "latency_p99_ms": 0.2776, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 }, "medium": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.41, "latency_p95_ms": 0.626, "latency_p99_ms": 0.6452, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "cases": [ { "task_id": "eff-001", "dimension": "efficiency", "category": "preprocess_latency", "difficulty": "easy", "passed": true, "expected": "<=50ms", "actual": "0.003ms", "duration_ms": 0.28, "root_cause": "none", "detail": "iterations=100 avg=0.003ms threshold=50.0ms", "consistency": 1.0 }, { "task_id": "eff-002", "dimension": "efficiency", "category": "preprocess_latency", "difficulty": "medium", "passed": true, "expected": "<=50ms", "actual": "0.003ms", "duration_ms": 0.33, "root_cause": "none", "detail": "iterations=100 avg=0.003ms threshold=50.0ms", "consistency": 1.0 }, { "task_id": "eff-003", "dimension": "efficiency", "category": "preprocess_latency", "difficulty": "medium", "passed": true, "expected": "<=50ms", "actual": "0.004ms", "duration_ms": 0.41, "root_cause": "none", "detail": "iterations=100 avg=0.004ms threshold=50.0ms", "consistency": 1.0 }, { "task_id": "eff-004", "dimension": "efficiency", "category": "tool_search_latency", "difficulty": "medium", "passed": true, "expected": "<=10ms", "actual": "0.006ms", "duration_ms": 0.65, "root_cause": "none", "detail": "iterations=100 avg=0.006ms threshold=10.0ms", "consistency": 1.0 }, { "task_id": "eff-005", "dimension": "efficiency", "category": "tool_search_latency", "difficulty": "easy", "passed": true, "expected": "<=5ms", "actual": "0.000ms", "duration_ms": 0.04, "root_cause": "none", "detail": "iterations=100 avg=0.000ms threshold=5.0ms", "consistency": 1.0 } ] }, "tool_search": { "metrics": { "accuracy": 1.0, "precision": 0.8333, "recall": 0.8333, "f1": 0.8333, "latency_p50_ms": 0.0229, "latency_p95_ms": 0.0415, "latency_p99_ms": 0.0518, "consistency": 1.0, "total": 10, "passed": 10, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.7225, "ci_upper": 1.0 }, "by_category": { "exact_match": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0234, "latency_p95_ms": 0.0487, "latency_p99_ms": 0.0533, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "fuzzy_match": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0224, "latency_p95_ms": 0.0228, "latency_p99_ms": 0.0228, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 }, "no_match": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.0089, "latency_p95_ms": 0.0141, "latency_p99_ms": 0.0146, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 }, "top_k": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0184, "latency_p95_ms": 0.0184, "latency_p99_ms": 0.0184, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 0.8333, "recall": 0.8333, "f1": 0.8333, "latency_p50_ms": 0.0231, "latency_p95_ms": 0.0458, "latency_p99_ms": 0.0527, "consistency": 1.0, "total": 7, "passed": 7, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6457, "ci_upper": 1.0 }, "medium": { "accuracy": 1.0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "latency_p50_ms": 0.0219, "latency_p95_ms": 0.0227, "latency_p99_ms": 0.0228, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "cases": [ { "task_id": "ts-001", "dimension": "tool_search", "category": "exact_match", "difficulty": "easy", "passed": true, "expected": "read_file", "actual": "read_file", "duration_ms": 0.023, "root_cause": "none", "detail": "query='read file' top_k=5 results=2", "consistency": 1.0 }, { "task_id": "ts-002", "dimension": "tool_search", "category": "exact_match", "difficulty": "easy", "passed": true, "expected": "write_file", "actual": "write_file", "duration_ms": 0.0544, "root_cause": "none", "detail": "query='write file content' top_k=5 results=2", "consistency": 1.0 }, { "task_id": "ts-003", "dimension": "tool_search", "category": "exact_match", "difficulty": "easy", "passed": true, "expected": "web_search", "actual": "web_search", "duration_ms": 0.0258, "root_cause": "none", "detail": "query='search web information' top_k=5 results=2", "consistency": 1.0 }, { "task_id": "ts-004", "dimension": "tool_search", "category": "exact_match", "difficulty": "easy", "passed": true, "expected": "shell_exec", "actual": "shell_exec", "duration_ms": 0.0234, "root_cause": "none", "detail": "query='execute shell command' top_k=5 results=1", "consistency": 1.0 }, { "task_id": "ts-005", "dimension": "tool_search", "category": "exact_match", "difficulty": "easy", "passed": true, "expected": "http_request", "actual": "http_request", "duration_ms": 0.0231, "root_cause": "none", "detail": "query='send http request url' top_k=5 results=1", "consistency": 1.0 }, { "task_id": "ts-006", "dimension": "tool_search", "category": "fuzzy_match", "difficulty": "medium", "passed": true, "expected": "read_file", "actual": "read_file", "duration_ms": 0.0228, "root_cause": "none", "detail": "query='io file' top_k=5 results=2", "consistency": 1.0 }, { "task_id": "ts-007", "dimension": "tool_search", "category": "fuzzy_match", "difficulty": "medium", "passed": true, "expected": "web_search", "actual": "web_search", "duration_ms": 0.0219, "root_cause": "none", "detail": "query='search query engine' top_k=5 results=1", "consistency": 1.0 }, { "task_id": "ts-008", "dimension": "tool_search", "category": "no_match", "difficulty": "easy", "passed": true, "expected": "__none__", "actual": "[]", "duration_ms": 0.003, "root_cause": "none", "detail": "query='' top_k=5 results=0", "consistency": 1.0 }, { "task_id": "ts-009", "dimension": "tool_search", "category": "no_match", "difficulty": "easy", "passed": true, "expected": "__none__", "actual": "[]", "duration_ms": 0.0147, "root_cause": "none", "detail": "query='zzzznonexistent' top_k=5 results=0", "consistency": 1.0 }, { "task_id": "ts-010", "dimension": "tool_search", "category": "top_k", "difficulty": "medium", "passed": true, "expected": "read_file", "actual": "read_file", "duration_ms": 0.0184, "root_cause": "none", "detail": "query='file' top_k=1 results=1", "consistency": 1.0 } ] }, "event_model": { "metrics": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.0894, "latency_p95_ms": 16.7933, "latency_p99_ms": 20.5773, "consistency": 1.0, "total": 6, "passed": 6, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6097, "ci_upper": 1.0 }, "by_category": { "sq_lifecycle": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.0671, "latency_p95_ms": 0.1071, "latency_p99_ms": 0.1107, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 }, "eq_lifecycle": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 2.6035, "latency_p95_ms": 19.6313, "latency_p99_ms": 21.1449, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.0894, "latency_p95_ms": 16.7933, "latency_p99_ms": 20.5773, "consistency": 1.0, "total": 6, "passed": 6, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6097, "ci_upper": 1.0 } }, "cases": [ { "task_id": "ev-001", "dimension": "event_model", "category": "sq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "drained=['hello']", "duration_ms": 0.1116, "root_cause": "none", "detail": "task_id=5c4be886...", "consistency": 1.0 }, { "task_id": "ev-002", "dimension": "event_model", "category": "sq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "cancelled=True", "duration_ms": 0.0671, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "ev-003", "dimension": "event_model", "category": "sq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "raised=True closed=True", "duration_ms": 0.0143, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "ev-004", "dimension": "event_model", "category": "eq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "received=1", "duration_ms": 2.6035, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "ev-005", "dimension": "event_model", "category": "eq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "events=1 closed=True", "duration_ms": 21.5233, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "ev-006", "dimension": "event_model", "category": "eq_lifecycle", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "subscribers=0", "duration_ms": 0.008, "root_cause": "none", "detail": "", "consistency": 1.0 } ] }, "spec_management": { "metrics": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 1.4329, "latency_p95_ms": 2.75, "latency_p99_ms": 3.1046, "consistency": 1.0, "total": 7, "passed": 7, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6457, "ci_upper": 1.0 }, "by_category": { "crud": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 1.4329, "latency_p95_ms": 2.8609, "latency_p99_ms": 3.1268, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "edge": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 0.8834, "latency_p95_ms": 1.6324, "latency_p99_ms": 1.699, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 1.3287, "latency_p95_ms": 2.7777, "latency_p99_ms": 3.1102, "consistency": 1.0, "total": 6, "passed": 6, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.6097, "ci_upper": 1.0 }, "medium": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 1.7156, "latency_p95_ms": 1.7156, "latency_p99_ms": 1.7156, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 } }, "cases": [ { "task_id": "sm-001", "dimension": "spec_management", "category": "crud", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "exists=True", "duration_ms": 1.4329, "root_cause": "none", "detail": "path=/var/folders/6b/ljk5bdq50yxcsth24frf05200000gn/T/agentkit-benchmark-dzm9kg48/run-0/specs/sm-001/test-spec.yaml", "consistency": 1.0 }, { "task_id": "sm-002", "dimension": "spec_management", "category": "crud", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "steps=2", "duration_ms": 1.2244, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "sm-003", "dimension": "spec_management", "category": "crud", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "goal=Updated goal", "duration_ms": 1.5311, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "sm-004", "dimension": "spec_management", "category": "crud", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "deleted=True remaining=0", "duration_ms": 1.1484, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "sm-005", "dimension": "spec_management", "category": "crud", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "count=2", "duration_ms": 3.1933, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "sm-006", "dimension": "spec_management", "category": "edge", "difficulty": "medium", "passed": true, "expected": "passed", "actual": "status=confirmed", "duration_ms": 1.7156, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "sm-007", "dimension": "spec_management", "category": "edge", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "result=None", "duration_ms": 0.0512, "root_cause": "none", "detail": "", "consistency": 1.0 } ] }, "verification": { "metrics": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 24.8909, "latency_p95_ms": 411.9118, "latency_p99_ms": 487.0974, "consistency": 1.0, "total": 5, "passed": 5, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.5655, "ci_upper": 1.0 }, "by_category": { "basic": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 11.7309, "latency_p95_ms": 11.9356, "latency_p99_ms": 11.9538, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 }, "retry": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 35.984, "latency_p95_ms": 35.984, "latency_p99_ms": 35.984, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "timeout": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 505.8938, "latency_p95_ms": 505.8938, "latency_p99_ms": 505.8938, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 }, "multi": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 24.8909, "latency_p95_ms": 24.8909, "latency_p99_ms": 24.8909, "consistency": 1.0, "total": 1, "passed": 1, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.2065, "ci_upper": 1.0 } }, "by_difficulty": { "easy": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 11.7309, "latency_p95_ms": 11.9356, "latency_p99_ms": 11.9538, "consistency": 1.0, "total": 2, "passed": 2, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.3424, "ci_upper": 1.0 }, "medium": { "accuracy": 1.0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "latency_p50_ms": 35.984, "latency_p95_ms": 458.9028, "latency_p99_ms": 496.4956, "consistency": 1.0, "total": 3, "passed": 3, "failed": 0, "accuracy_mean": 1.0, "accuracy_std": 0.0, "ci_lower": 0.4385, "ci_upper": 1.0 } }, "cases": [ { "task_id": "vf-001", "dimension": "verification", "category": "basic", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "passed=True attempts=1", "duration_ms": 11.5036, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "vf-002", "dimension": "verification", "category": "basic", "difficulty": "easy", "passed": true, "expected": "passed", "actual": "passed=False errors=1", "duration_ms": 11.9583, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "vf-003", "dimension": "verification", "category": "retry", "difficulty": "medium", "passed": true, "expected": "passed", "actual": "attempts=3 callbacks=2", "duration_ms": 35.984, "root_cause": "none", "detail": "", "consistency": 1.0 }, { "task_id": "vf-004", "dimension": "verification", "category": "timeout", "difficulty": "medium", "passed": true, "expected": "passed", "actual": "passed=False errors=1", "duration_ms": 505.8938, "root_cause": "none", "detail": "errors=['Command timed out after 0.5s: sleep 10']", "consistency": 1.0 }, { "task_id": "vf-005", "dimension": "verification", "category": "multi", "difficulty": "medium", "passed": true, "expected": "passed", "actual": "passed=False", "duration_ms": 24.8909, "root_cause": "none", "detail": "", "consistency": 1.0 } ] } } }