173 lines
6.6 KiB
Python
173 lines
6.6 KiB
Python
"""Unit tests for QualityGate skill match validation (5th dimension)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from agentkit.quality.gate import QualityGate
|
|
from agentkit.skills.base import Skill, SkillConfig
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_skill(
|
|
name: str = "test_skill",
|
|
required_fields: list[str] | None = None,
|
|
min_word_count: int = 0,
|
|
) -> Skill:
|
|
"""Create a Skill with the given quality gate config."""
|
|
config = SkillConfig(
|
|
name=name,
|
|
agent_type="skill",
|
|
task_mode="llm_generate",
|
|
prompt={"identity": f"You are {name}"},
|
|
quality_gate={
|
|
"required_fields": required_fields or [],
|
|
"min_word_count": min_word_count,
|
|
},
|
|
)
|
|
return Skill(config=config)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: _check_skill_match static method
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCheckSkillMatch:
|
|
def setup_method(self) -> None:
|
|
self.gate = QualityGate()
|
|
|
|
def test_none_skill_context(self) -> None:
|
|
assert self.gate._check_skill_match({"content": "hello"}, None) is None
|
|
|
|
def test_empty_skill_context(self) -> None:
|
|
assert self.gate._check_skill_match({"content": "hello"}, {}) is None
|
|
|
|
def test_missing_intent_keywords(self) -> None:
|
|
assert self.gate._check_skill_match({"content": "hello"}, {"skill_name": "x"}) is None
|
|
|
|
def test_empty_intent_keywords(self) -> None:
|
|
assert self.gate._check_skill_match({"content": "hello"}, {"intent_keywords": []}) is None
|
|
|
|
def test_output_contains_keyword(self) -> None:
|
|
result = self.gate._check_skill_match(
|
|
{"content": "市场分析报告"},
|
|
{"intent_keywords": ["分析", "报告"]},
|
|
)
|
|
assert result is not None
|
|
assert result.passed is True
|
|
assert result.message is None
|
|
|
|
def test_output_missing_all_keywords(self) -> None:
|
|
result = self.gate._check_skill_match(
|
|
{"content": "今天天气不错"},
|
|
{"intent_keywords": ["分析", "报告"]},
|
|
)
|
|
assert result is not None
|
|
assert result.passed is True # Warning level, not blocking
|
|
assert "Warning" in (result.message or "")
|
|
|
|
def test_keyword_case_insensitive(self) -> None:
|
|
result = self.gate._check_skill_match(
|
|
{"content": "search results"},
|
|
{"intent_keywords": ["Search"]},
|
|
)
|
|
assert result is not None
|
|
assert result.passed is True
|
|
assert result.message is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: Full validate() with skill_context
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateWithSkillContext:
|
|
@pytest.mark.asyncio
|
|
async def test_no_skill_context_backward_compatible(self) -> None:
|
|
"""Without skill_context, only 4 dimensions checked."""
|
|
gate = QualityGate()
|
|
skill = _make_skill()
|
|
result = await gate.validate({"content": "hello"}, skill)
|
|
assert result.passed is True
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skill_context_with_matching_output(self) -> None:
|
|
"""Output contains keyword → skill_match passes silently."""
|
|
gate = QualityGate()
|
|
skill = _make_skill()
|
|
result = await gate.validate(
|
|
{"content": "市场分析报告"},
|
|
skill,
|
|
skill_context={"intent_keywords": ["分析"]},
|
|
)
|
|
assert result.passed is True
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 1
|
|
assert skill_match_checks[0].passed is True
|
|
assert skill_match_checks[0].message is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skill_context_warning_only(self) -> None:
|
|
"""Output missing keywords but other checks pass → warning, overall still passed."""
|
|
gate = QualityGate()
|
|
skill = _make_skill()
|
|
result = await gate.validate(
|
|
{"content": "今天天气不错"},
|
|
skill,
|
|
skill_context={"intent_keywords": ["分析"]},
|
|
)
|
|
assert result.passed is True # Warning doesn't block alone
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 1
|
|
assert "Warning" in (skill_match_checks[0].message or "")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skill_match_escalates_with_other_failure(self) -> None:
|
|
"""Output missing keywords + required field missing → skill_match escalated to failed."""
|
|
gate = QualityGate()
|
|
skill = _make_skill(required_fields=["summary"])
|
|
result = await gate.validate(
|
|
{"content": "今天天气不错"}, # missing "summary" field
|
|
skill,
|
|
skill_context={"intent_keywords": ["分析"]},
|
|
)
|
|
assert result.passed is False
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 1
|
|
assert skill_match_checks[0].passed is False # Escalated
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skill_match_no_escalation_when_matching(self) -> None:
|
|
"""Output contains keywords + required field missing → skill_match stays passed."""
|
|
gate = QualityGate()
|
|
skill = _make_skill(required_fields=["summary"])
|
|
result = await gate.validate(
|
|
{"content": "分析结果"}, # missing "summary" field
|
|
skill,
|
|
skill_context={"intent_keywords": ["分析"]},
|
|
)
|
|
assert result.passed is False # Due to required field
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 1
|
|
assert skill_match_checks[0].passed is True # Not escalated
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_intent_keywords_skips_check(self) -> None:
|
|
"""Empty intent_keywords list → skill_match check skipped entirely."""
|
|
gate = QualityGate()
|
|
skill = _make_skill()
|
|
result = await gate.validate(
|
|
{"content": "hello"},
|
|
skill,
|
|
skill_context={"intent_keywords": []},
|
|
)
|
|
skill_match_checks = [c for c in result.checks if c.name == "skill_match"]
|
|
assert len(skill_match_checks) == 0
|