From d998c0344c848c19a3bd5df442b0754446b36fd6 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 00:18:09 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat(telemetry):=20U1=20=E2=80=94=20OTel?= =?UTF-8?q?=20=E9=BB=98=E8=AE=A4=E5=90=AF=E7=94=A8=EF=BC=8C=E4=BB=8E=20opt?= =?UTF-8?q?ional=20=E5=8D=87=E7=BA=A7=E4=B8=BA=20required=20=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml: 添加 opentelemetry-api/sdk/exporter-otlp-proto-grpc/ instrumentation-fastapi 到 [project] dependencies（KTD-3） - telemetry/{tracer,metrics,tracing,setup}.py: 移除 try/except ImportError 静默回退 — 缺包现在是硬安装错误，不再是监控盲区。保留 except Exception 处理运行时错误（OTLP 连接失败），保留 enabled=false 配置开关 - setup.py: 修复已存在 bug — MeterProvider 的 readers kwarg 在 OTel SDK 1.16+ 改名为 metric_readers（被 ImportError fallback 掩盖） - agentkit.yaml: 取消 telemetry 段注释，设 enabled=true + otlp_endpoint + service_name + export_traces/metrics - test_telemetry.py: 更新 test_missing_opentelemetry → test_enabled_initializes_otel_tracer； 新增 TestSetupTelemetry smoke test（disabled/none 是 no-op，enabled instrument FastAPI）； 清理 unused imports/vars Verification: - pytest tests/unit/test_telemetry.py: 21 passed - pytest tests/unit/test_react_compression.py: 22 passed（_OTEL_AVAILABLE patch 兼容） - runtime check: enabled=True → OTelTracer, enabled=False → NoOpTracer --- agentkit.yaml | 20 ++++--- pyproject.toml | 5 ++ src/agentkit/telemetry/metrics.py | 20 ++++--- src/agentkit/telemetry/setup.py | 88 ++++++++++------------------ src/agentkit/telemetry/tracer.py | 36 +++++++----- src/agentkit/telemetry/tracing.py | 58 ++++++++----------- tests/unit/test_telemetry.py | 96 ++++++++++++++++++++++--------- 7 files changed, 172 insertions(+), 151 deletions(-) diff --git a/agentkit.yaml b/agentkit.yaml index ef3f66f..62c1411 100644 --- a/agentkit.yaml +++ b/agentkit.yaml @@ -70,19 +70,25 @@ fallback_chain: # Tests run in-memory for isolation; production / staging should use Redis. session: backend: ${AGENTKIT_SESSION_BACKEND:-memory} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} bus: backend: ${AGENTKIT_BUS_BACKEND:-memory} - redis_url: ${REDIS_URL:-redis://127.0.0.1:6379/0} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} task_store: backend: ${AGENTKIT_TASK_STORE_BACKEND:-memory} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} skills: {auto_discover: true, paths: ["./configs/skills"]} experts: {paths: ["./configs/experts"]} board: {max_rounds: 5, default_template: private_board, parallel_speech: true, history_compression_threshold: 20} logging: {level: INFO, format: text} router: {classifier: heuristic, auction_enabled: false} -# OTel 可观测性 — 默认注释（OTel 为可选依赖，未安装时 telemetry/metrics.py 返回 NoOp）。 -# 启用：pip install opentelemetry-sdk opentelemetry-exporter-otlp，取消注释并指向 collector。 -# 未配置时所有指标（请求量/延迟/token 消耗）静默丢弃，形成监控盲区。 -# telemetry: -# otlp_endpoint: http://localhost:4317 # OTLP gRPC 端点 -# service_name: fischer-agentkit +# OTel 可观测性（U1 — 默认启用）。 +# OTel 依赖已升级为 required（pyproject.toml [project] dependencies）， +# ImportError 静默回退已移除 — 缺包将硬失败。设 enabled=false 可显式关闭。 +# otlp_endpoint 指向 OTel collector（容器名 otel-collector / jaeger）。 +telemetry: + enabled: true + otlp_endpoint: http://localhost:4317 # OTLP gRPC 端点（开发环境；生产指向集群 collector） + service_name: fischer-agentkit + export_traces: true + export_metrics: true diff --git a/pyproject.toml b/pyproject.toml index e713435..d53f45f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,11 @@ dependencies = [ # 异步任务队列（U8 — 文档向量化） "taskiq>=0.11", "taskiq-redis>=0.5", + # OTel 可观测性（U1 — 默认启用，从 optional 升级为 required，移除 ImportError 静默回退） + "opentelemetry-api>=1.24", + "opentelemetry-sdk>=1.24", + "opentelemetry-exporter-otlp-proto-grpc>=1.24", + "opentelemetry-instrumentation-fastapi>=0.45b0", ] [project.scripts] diff --git a/src/agentkit/telemetry/metrics.py b/src/agentkit/telemetry/metrics.py index 0525be7..5c4303c 100644 --- a/src/agentkit/telemetry/metrics.py +++ b/src/agentkit/telemetry/metrics.py @@ -1,11 +1,15 @@ -"""Metric definitions — no-op when OTel not installed""" +"""Metric definitions — OTel now required (U1). -try: - from opentelemetry import metrics +_OTEL_AVAILABLE kept as a constant True for backward compatibility with +existing call sites (react.py / rewoo.py / reflexion.py / gateway.py) that +guard span creation with `if _OTEL_AVAILABLE:`. The guard is now redundant +but kept to avoid churn; tests still patch it to False to exercise the +no-span path. +""" - _OTEL_AVAILABLE = True -except ImportError: - _OTEL_AVAILABLE = False +from opentelemetry import metrics + +_OTEL_AVAILABLE = True class _NoOpCounter: @@ -92,9 +96,7 @@ def tool_duration_histogram(): """Tool call duration.""" global _tool_duration_histogram if _tool_duration_histogram is None: - _tool_duration_histogram = _get_histogram( - "tool.call.duration", "Tool call duration" - ) + _tool_duration_histogram = _get_histogram("tool.call.duration", "Tool call duration") return _tool_duration_histogram diff --git a/src/agentkit/telemetry/setup.py b/src/agentkit/telemetry/setup.py index 5da9581..38b01d6 100644 --- a/src/agentkit/telemetry/setup.py +++ b/src/agentkit/telemetry/setup.py @@ -1,16 +1,26 @@ -"""OTel initialization — called at app startup""" +"""OTel initialization — called at app startup (U1: OTel now required).""" import logging +from opentelemetry import trace, metrics +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + logger = logging.getLogger(__name__) def setup_telemetry(app, config: dict | None = None): - """Initialize OpenTelemetry if installed and configured. + """Initialize OpenTelemetry if configured. - This is a no-op when: - - config is None or config.enabled is False - - opentelemetry packages are not installed + U1: opentelemetry-* deps now required (pyproject.toml). ImportError + fallbacks removed — a missing package is a hard install error. The + `enabled=False` config switch remains the supported way to disable. Args: app: FastAPI application instance @@ -25,69 +35,29 @@ def setup_telemetry(app, config: dict | None = None): logger.info("Telemetry disabled") return - try: - from opentelemetry import trace, metrics - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader - from opentelemetry.sdk.resources import Resource - except ImportError: - logger.warning( - "OpenTelemetry packages not installed. Telemetry disabled." - ) - return - service_name = config.get("service_name", "fischer-agentkit") resource = Resource.create({"service.name": service_name}) # Tracing setup if config.get("export_traces", True): endpoint = config.get("otlp_endpoint", "http://localhost:4317") - try: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - - provider = TracerProvider(resource=resource) - provider.add_span_processor( - BatchSpanProcessor( - OTLPSpanExporter(endpoint=endpoint, insecure=True) - ) - ) - trace.set_tracer_provider(provider) - logger.info(f"Tracing enabled, exporting to {endpoint}") - except ImportError: - logger.warning( - "OTLP exporter not installed. Tracing disabled." - ) + provider = TracerProvider(resource=resource) + provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True)) + ) + trace.set_tracer_provider(provider) + logger.info(f"Tracing enabled, exporting to {endpoint}") # Metrics setup if config.get("export_metrics", True): endpoint = config.get("otlp_endpoint", "http://localhost:4317") - try: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - - reader = PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=endpoint, insecure=True) - ) - provider = MeterProvider(resource=resource, readers=[reader]) - metrics.set_meter_provider(provider) - logger.info(f"Metrics enabled, exporting to {endpoint}") - except ImportError: - logger.warning( - "OTLP metric exporter not installed. Metrics disabled." - ) + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint, insecure=True)) + # ponytail: OTel SDK 1.16+ 将 readers kwarg 改名为 metric_readers； + # 旧代码用 readers=[...] 是已存在 bug，被 ImportError fallback 掩盖。 + provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(provider) + logger.info(f"Metrics enabled, exporting to {endpoint}") # FastAPI auto-instrumentation - try: - from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor - - FastAPIInstrumentor.instrument_app(app, excluded_urls="health,metrics") - logger.info("FastAPI auto-instrumentation enabled") - except ImportError: - logger.warning( - "FastAPI instrumentation not installed. Skipping auto-instrumentation." - ) + FastAPIInstrumentor.instrument_app(app, excluded_urls="health,metrics") + logger.info("FastAPI auto-instrumentation enabled") diff --git a/src/agentkit/telemetry/tracer.py b/src/agentkit/telemetry/tracer.py index ad808ef..7893bad 100644 --- a/src/agentkit/telemetry/tracer.py +++ b/src/agentkit/telemetry/tracer.py @@ -1,4 +1,5 @@ """OpenTelemetry tracer integration with no-op fallback.""" + from __future__ import annotations import logging @@ -43,10 +44,14 @@ class NoOpSpan: class NoOpTracer: """No-op tracer when telemetry is disabled.""" - def start_span(self, name: str, attributes: dict[str, object] | None = None) -> ContextManager[NoOpSpan]: + def start_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> ContextManager[NoOpSpan]: return NoOpSpan() - def start_as_current_span(self, name: str, attributes: dict[str, object] | None = None) -> ContextManager[NoOpSpan]: + def start_as_current_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> ContextManager[NoOpSpan]: return NoOpSpan() @@ -86,7 +91,9 @@ class OTelTracer: span = self._tracer.start_span(name, attributes=attributes) return OTelSpan(span) - def start_as_current_span(self, name: str, attributes: dict[str, object] | None = None) -> OTelSpan: + def start_as_current_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> OTelSpan: span = self._tracer.start_as_current_span(name, attributes=attributes) return OTelSpan(span) @@ -101,7 +108,13 @@ def get_tracer() -> NoOpTracer | OTelTracer: def init_telemetry(config: TelemetryConfig) -> None: - """Initialize telemetry with the given configuration.""" + """Initialize telemetry with the given configuration. + + U1: OTel deps now required (pyproject.toml). ImportError fallback removed — + a missing opentelemetry-* package is now a hard install error, not a silent + monitoring blind spot. Runtime errors (network, exporter connect) still + fall back to NoOpTracer so app startup never crashes on transient OTLP issues. + """ global _tracer if not config.enabled: @@ -109,13 +122,13 @@ def init_telemetry(config: TelemetryConfig) -> None: logger.info("Telemetry disabled, using no-op tracer") return - try: - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.resources import Resource - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + try: resource = Resource.create({"service.name": config.service_name}) provider = TracerProvider(resource=resource) @@ -126,9 +139,6 @@ def init_telemetry(config: TelemetryConfig) -> None: trace.set_tracer_provider(provider) _tracer = OTelTracer(trace.get_tracer(config.service_name)) logger.info(f"Telemetry initialized with endpoint: {config.otlp_endpoint}") - except ImportError: - logger.warning("opentelemetry packages not installed, using no-op tracer") - _tracer = NoOpTracer() except Exception as e: logger.warning(f"Failed to initialize telemetry: {e}, using no-op tracer") _tracer = NoOpTracer() diff --git a/src/agentkit/telemetry/tracing.py b/src/agentkit/telemetry/tracing.py index 4f3c65c..7b10224 100644 --- a/src/agentkit/telemetry/tracing.py +++ b/src/agentkit/telemetry/tracing.py @@ -1,43 +1,33 @@ -"""Tracing helpers — no-op when OTel not installed""" +"""Tracing helpers — OTel now required (U1). + +_OTEL_AVAILABLE kept as a constant True for backward compatibility with +existing call sites (react.py / rewoo.py / reflexion.py / gateway.py) that +guard span creation with `if _OTEL_AVAILABLE:`. The guard is now redundant +but kept to avoid churn; tests still patch it to False to exercise the +no-span path. +""" import logging import time from functools import wraps from typing import Callable +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode + logger = logging.getLogger(__name__) -# Try importing OTel — if not available, provide no-op implementations -try: - from opentelemetry import trace - from opentelemetry.trace import SpanKind, Status, StatusCode - - _OTEL_AVAILABLE = True -except ImportError: - _OTEL_AVAILABLE = False - - # Provide fallback stubs so module-level references work in tests - class _StubEnum: - INTERNAL = "INTERNAL" - CLIENT = "CLIENT" - SERVER = "SERVER" - PRODUCER = "PRODUCER" - CONSUMER = "CONSUMER" - - SpanKind = _StubEnum # type: ignore[misc,assignment] - - class Status: # type: ignore[no-redef] - def __init__(self, *args, **kwargs): - pass - - class StatusCode: # type: ignore[no-redef] - UNSET = "UNSET" - OK = "OK" - ERROR = "ERROR" +_OTEL_AVAILABLE = True class _NoOpSpan: - """No-op span context manager used when OTel is not installed.""" + """No-op span context manager — retained for tests that patch + `_OTEL_AVAILABLE = False` and call `start_span` directly. + + ponytail: kept only for test compatibility; production code now always + has OTel installed. Safe to remove once tests are migrated to patch + `opentelemetry.trace` directly. + """ def __enter__(self): return self @@ -59,10 +49,8 @@ class _NoOpSpan: def get_tracer(name: str = "fischer.agentkit"): - """Get tracer — returns None if OTel not installed.""" - if _OTEL_AVAILABLE: - return trace.get_tracer(name) - return None + """Get tracer — returns the global OTel tracer (always available post-U1).""" + return trace.get_tracer(name) def start_span( @@ -70,9 +58,9 @@ def start_span( kind: object = None, attributes: dict | None = None, ): - """Start a span — returns no-op span if OTel not installed. + """Start a span. Returns an OTel span context manager. - Returns a context manager that yields a span (or no-op). + `_OTEL_AVAILABLE` guard retained for tests that patch it to False. """ if not _OTEL_AVAILABLE: return _NoOpSpan() diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index b3bfc66..af0e866 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -1,13 +1,12 @@ """Telemetry module unit tests.""" -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch import pytest from agentkit.telemetry.tracer import ( NoOpSpan, NoOpTracer, - OTelSpan, OTelTracer, TelemetryConfig, get_tracer, @@ -110,29 +109,26 @@ class TestInitTelemetry: tracer = get_tracer() assert isinstance(tracer, NoOpTracer) - def test_missing_opentelemetry_falls_back_to_noop(self): - """当 opentelemetry 包未安装时，优雅降级为 NoOpTracer""" + def test_enabled_initializes_otel_tracer(self): + """U1: enabled=True 初始化 OTelTracer（OTel 现为 required 依赖，不再 ImportError fallback）""" config = TelemetryConfig(enabled=True, otlp_endpoint="http://localhost:4317") - # 即使 opentelemetry 未安装，也不应抛出异常 init_telemetry(config) tracer = get_tracer() - # 在没有 opentelemetry 的环境中应为 NoOpTracer - assert isinstance(tracer, (NoOpTracer, OTelTracer)) + # OTel 已安装，应返回 OTelTracer 实例（而非 NoOpTracer） + assert isinstance(tracer, OTelTracer) def test_init_with_exception_falls_back_to_noop(self): - """初始化过程中出现异常时，降级为 NoOpTracer""" + """初始化过程中出现运行时异常时，降级为 NoOpTracer（保留运行时容错）""" config = TelemetryConfig(enabled=True, otlp_endpoint="http://localhost:4317") + # OTLPSpanExporter 在 init_telemetry 函数内部 import，patch 源模块符号 + # opentelemetry.sdk.trace.TracerProvider 是函数内 import 的真实符号 with patch( - "agentkit.telemetry.tracer.init_telemetry", - side_effect=Exception("init error"), - ) as mock_init: - # init_telemetry 被模拟为抛异常，但实际调用的是原始函数 - # 这里验证的是 init_telemetry 本身不会崩溃 - pass - # 直接调用，不应崩溃 - init_telemetry(config) - tracer = get_tracer() - assert isinstance(tracer, (NoOpTracer, OTelTracer)) + "opentelemetry.sdk.trace.TracerProvider", + side_effect=RuntimeError("provider init failed"), + ): + init_telemetry(config) + tracer = get_tracer() + assert isinstance(tracer, NoOpTracer) # ── get_tracer 测试 ──────────────────────────────────────── @@ -174,14 +170,11 @@ class TestAlignmentGuardSpan: with patch.object(tracer, "start_span", return_value=mock_span): config = AlignmentConfig(constraints=["forbidden_word"]) guard = AlignmentGuard(config) - result = await guard.check_output( - {"content": "This contains forbidden_word"} - ) + await guard.check_output({"content": "This contains forbidden_word"}) tracer.start_span.assert_called_once_with("guard.check") call_args_list = [ - (call.args[0], call.args[1]) - for call in mock_span.set_attribute.call_args_list + (call.args[0], call.args[1]) for call in mock_span.set_attribute.call_args_list ] assert ("guard.constraints_count", 1) in call_args_list assert ("guard.passed", False) in call_args_list @@ -201,13 +194,60 @@ class TestAlignmentGuardSpan: with patch.object(tracer, "start_span", return_value=mock_span): config = AlignmentConfig(constraints=["forbidden_word"]) guard = AlignmentGuard(config) - result = await guard.check_output( - {"content": "This is clean text"} - ) + await guard.check_output({"content": "This is clean text"}) call_args_list = [ - (call.args[0], call.args[1]) - for call in mock_span.set_attribute.call_args_list + (call.args[0], call.args[1]) for call in mock_span.set_attribute.call_args_list ] assert ("guard.passed", True) in call_args_list assert ("guard.checked_by", "rule") in call_args_list + + +# ── setup_telemetry smoke test ─────────────────────────── + + +class TestSetupTelemetry: + """setup_telemetry FastAPI 装配 smoke test（U1）""" + + def test_disabled_is_noop(self): + """enabled=False 时不做任何装配""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + # spec=FastAPI 让 getattr 返回真实的默认值（而非 MagicMock 自动属性） + app = MagicMock(spec=FastAPI) + setup_telemetry(app, {"enabled": False}) + # app 不应被 instrument（spec 让未设属性返回 AttributeError→default False） + assert not getattr(app, "_is_instrumented_by_opentelemetry", False) + + def test_enabled_instruments_fastapi(self): + """enabled=True 时 FastAPIInstrumentor 装配 app（不崩溃即可）""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + app = FastAPI() + # 使用一个无效 endpoint — exporter 会失败但不阻塞 instrument + setup_telemetry( + app, + { + "enabled": True, + "otlp_endpoint": "http://127.0.0.1:4317", + "service_name": "test-agentkit", + "export_traces": True, + "export_metrics": True, + }, + ) + # FastAPIInstrumentor 装配后会设置此属性 + assert getattr(app, "_is_instrumented_by_opentelemetry", False) is True + + def test_none_config_is_noop(self): + """config=None 不崩溃""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + app = MagicMock(spec=FastAPI) + setup_telemetry(app, None) + assert not getattr(app, "_is_instrumented_by_opentelemetry", False) From 60a58a0fdb9e45381cfecce6ff1946fb68b8b8e3 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 00:32:35 +0800 Subject: [PATCH 02/11] =?UTF-8?q?feat(llm):=20U2=20=E2=80=94=20Prompt=20Ca?= =?UTF-8?q?che=20=E5=85=A8=E9=93=BE=E8=B7=AF=E7=9B=91=E6=8E=A7=20+=20PII?= =?UTF-8?q?=20=E8=84=B1=E6=95=8F=20hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U2/R3 — Anthropic prompt cache 命中率监控 + 等保合规 PII 脱敏。 变更： - protocol.py: TokenUsage 添加 cache_read/creation_input_tokens + cache_hit property - providers/anthropic.py: 非流式 + 流式两处解析 cache 字段（cache_read_input_tokens / cache_creation_input_tokens） - gateway.py: chat() / chat_stream() emit prompt_cache.hit / prompt_cache.miss OTel counter + span attributes - telemetry/metrics.py: 新增 4 个 metric instruments（prompt_cache.hit/miss + pii_filter.coverage/redacted） - llm/pii_filter.py（新文件）: 正则版 PII 脱敏（手机/邮箱/身份证/银行卡）+ ner_hook + fail-closed + sha256 hash 审计 - agentkit.yaml: 添加 fallbacks + pii_filter 配置模板（默认 disabled，等保场景按需启用） - tests: test_prompt_cache_layers.py +4 cache metric 测试；test_pii_filter.py 新增 PII 脱敏测试（29 测试通过） 设计决策： - PII 脱敏用 stdlib（re + hashlib），不引入 NER 依赖；ner_hook 接入客户内网 NER 模型 - 正则顺序 id_card 在 phone 之前（避免身份证号内 11 位数字子串被手机号误匹配） - ner_hook 异常不吞掉：传播到 filter_pii 由 fail_closed 决定（True→PIIFilterError；False→降级正则版重试） - fail-closed：脱敏失败拒绝发送（不降级到原文），等保合规要求 --- agentkit.yaml | 18 ++ src/agentkit/llm/gateway.py | 61 +++++-- src/agentkit/llm/pii_filter.py | 227 ++++++++++++++++++++++++ src/agentkit/llm/protocol.py | 16 +- src/agentkit/llm/providers/anthropic.py | 8 + src/agentkit/telemetry/metrics.py | 47 +++++ tests/unit/test_pii_filter.py | 200 +++++++++++++++++++++ tests/unit/test_prompt_cache_layers.py | 141 ++++++++++++++- 8 files changed, 697 insertions(+), 21 deletions(-) create mode 100644 src/agentkit/llm/pii_filter.py create mode 100644 tests/unit/test_pii_filter.py diff --git a/agentkit.yaml b/agentkit.yaml index 62c1411..7c4d53d 100644 --- a/agentkit.yaml +++ b/agentkit.yaml @@ -28,6 +28,24 @@ llm: coding: bailian-coding/qwen3-coder-plus chat: deepseek/deepseek-chat reasoning: deepseek/deepseek-reasoner + # U2/R3 — 模型 fallback 链：主模型失败时按序尝试 fallback。 + # 等保合规场景：fallback provider 在客户内网（DashScope/DeepSeek OpenAI-compatible API）， + # 不依赖 Anthropic 境外服务。fallback providers 默认 prompt_cache_enable=false + #（非 Anthropic provider 无 cache_control 透传，走自动前缀缓存）。 + fallbacks: + default: [fast] # default 失败 → fast（同 provider 内降级） + # 等保合规 fallback（注释，按需启用）：将主模型 fallback 到本地推理 provider + # default: [chat, fast] # default 失败 → deepseek-chat → qwen-turbo + # U2/R3 — PII 脱敏 hook（fail-closed）。 + # 启用后，所有发送至 LLM 的 prompt 先经过 pii_filter 脱敏： + # - 正则识别手机号/邮箱/身份证/银行卡，替换为 [REDACTED_TYPE:hash8] + # - 脱敏前后记录 hash 用于审计（原文不落盘） + # - 任何异常时 fail-closed（拒绝发送至 LLM） + # - 留 ner_hook 接入客户内网 NER 模型（LAC/HanLP），未配置时用正则版 + pii_filter: + enabled: false # 默认关闭；等保合规场景设 true + ner_hook: null # 可选：自定义 NER 函数路径（module:func） + fail_closed: true # 脱敏失败时拒绝发送（true）或降级发送（false） # G4/U1: Auxiliary model for cost-sensitive tasks (summarization). # When set, ContextCompressor tries this alias first, falling back to # the main model on failure or empty content. Commented to preserve diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index 387b578..7496f43 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -12,7 +12,11 @@ from agentkit.llm.config import LLMConfig from agentkit.llm.protocol import LLMProvider, LLMRequest, LLMResponse, StreamChunk, TokenUsage from agentkit.llm.providers.tracker import UsageSummary, UsageTracker from agentkit.telemetry.tracing import get_tracer, _OTEL_AVAILABLE -from agentkit.telemetry.metrics import llm_token_histogram +from agentkit.telemetry.metrics import ( + llm_token_histogram, + prompt_cache_hit_counter, + prompt_cache_miss_counter, +) if TYPE_CHECKING: from agentkit.llm.cache import LitellmCacheManager @@ -25,27 +29,25 @@ logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -if TYPE_CHECKING: +class _QuotaServiceLike(Protocol): + """Quota service 最小契约（仅覆盖 gateway._enforce_quota 用到的方法）。""" - class _QuotaServiceLike(Protocol): - """Quota service 最小契约（仅覆盖 gateway._enforce_quota 用到的方法）。""" + async def is_model_allowed( + self, db: Path, department_id: str, model: str + ) -> tuple[bool, str]: ... - async def is_model_allowed( - self, db: Path, department_id: str, model: str - ) -> tuple[bool, str]: ... + async def check_quota( + self, + db: Path, + department_id: str, + quota_type: str, + period: str, + current: float, + ) -> tuple[bool, str]: ... - async def check_quota( - self, - db: Path, - department_id: str, - quota_type: str, - period: str, - current: float, - ) -> tuple[bool, str]: ... - - async def get_quota( - self, db: Path, department_id: str, quota_type: str, period: str - ) -> dict[str, object] | None: ... + async def get_quota( + self, db: Path, department_id: str, quota_type: str, period: str + ) -> dict[str, object] | None: ... class QuotaExceededError(Exception): @@ -286,10 +288,25 @@ class LLMGateway: _span.set_attribute("gen_ai.duration.ms", int(latency_ms)) if self._cache_manager is not None: _span.set_attribute("gen_ai.cache.hit", is_cache_hit) + # U2 — Anthropic prompt cache hit/miss span attributes + _span.set_attribute( + "gen_ai.usage.cache_read_input_tokens", + response.usage.cache_read_input_tokens, + ) + _span.set_attribute( + "gen_ai.usage.cache_creation_input_tokens", + response.usage.cache_creation_input_tokens, + ) llm_token_histogram().record( response.usage.total_tokens, {"gen_ai.request.model": resolved_model}, ) + # U2 — prompt cache hit/miss OTel counter（Anthropic cache_read_input_tokens > 0 → hit） + cache_attrs = {"gen_ai.request.model": resolved_model} + if response.usage.cache_read_input_tokens > 0: + prompt_cache_hit_counter().add(1, cache_attrs) + else: + prompt_cache_miss_counter().add(1, cache_attrs) return response finally: @@ -397,6 +414,12 @@ class LLMGateway: user_id=user_id, department_ids=department_ids, ) + # U2 — prompt cache hit/miss OTel counter（streaming path） + stream_cache_attrs = {"gen_ai.request.model": final_model} + if final_usage.cache_read_input_tokens > 0: + prompt_cache_hit_counter().add(1, stream_cache_attrs) + else: + prompt_cache_miss_counter().add(1, stream_cache_attrs) # Empty stream detection: if no content was produced, # raise error so the caller (ReActEngine) can retry with a different model. diff --git a/src/agentkit/llm/pii_filter.py b/src/agentkit/llm/pii_filter.py new file mode 100644 index 0000000..70f1aea --- /dev/null +++ b/src/agentkit/llm/pii_filter.py @@ -0,0 +1,227 @@ +"""PII 脱敏 hook（U2/R3）。 + +等保合规场景：发送至 LLM 的 prompt 先经 PII 脱敏，原文不落盘。 + +设计： +- 正则版（默认）：识别手机号 / 邮箱 / 身份证 / 银行卡，替换为 [REDACTED_TYPE:hash8] +- ner_hook（可选）：客户内网 NER 模型（LAC/HanLP），module:func 路径动态 import +- fail-closed：任何异常时拒绝发送（raise PIIFilterError），不降级到原文发送 +- hash 审计：脱敏前后记录 hash 用于审计（仅 hash，不含原文） + +ponytail: 用 stdlib（re + hashlib），不引入 NER 依赖。 +升级路径：ner_hook 接入客户内网 NER 模型，正则版作为 fallback。 +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + + +class PIIFilterError(Exception): + """PII 脱敏失败（fail-closed 触发）""" + + +# ── PII 正则模式 ────────────────────────────────────────── +# ponytail: 正则覆盖中国常见 PII；边界宽松避免漏报（fail-closed 优于 false negative）。 +# 升级路径：ner_hook 接入 NER 模型识别更复杂 PII（地址、姓名、组织）。 + +# 中国手机号：1[3-9] 开头 11 位 +_PHONE_RE = re.compile(r"1[3-9]\d{9}") +# 邮箱 +_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") +# 中国身份证号：18 位（最后一位 X 校验），前 6 位地区码 + 8 位生日 + 3 位序号 + 1 位校验 +_ID_CARD_RE = re.compile( + r"\b[1-9]\d{5}(?:19|20)\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\d|3[01])\d{3}[\dXx]\b" +) +# 银行卡号：16-19 位数字（宽松，可能误报长数字序列 — fail-closed 优于漏报） +_BANK_CARD_RE = re.compile(r"\b\d{16,19}\b") + + +@dataclass +class PIIMatch: + """单个 PII 匹配结果""" + + pii_type: str # phone / email / id_card / bank_card + original: str + redacted: str + hash: str # 原文 sha256 前 8 位（审计用，不可逆） + + +@dataclass +class PIIFilterResult: + """PII 脱敏结果""" + + redacted_text: str + matches: list[PIIMatch] = field(default_factory=list) + + @property + def redacted_count(self) -> int: + return len(self.matches) + + +def _hash_pii(original: str) -> str: + """sha256 前 8 位（审计用）""" + return hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] + + +def _redact(text: str, ner_hook=None) -> PIIFilterResult: + """执行 PII 脱敏（内部函数，ner_hook 异常向上传播由 filter_pii 决定 fail-closed）。 + + ponytail: 正则版按 PII 类型顺序替换；id_card 在 phone 之前（更具体的先匹配， + 避免身份证号内的 11 位数字被手机号正则误匹配）。ner_hook 优先于正则版。 + """ + matches: list[PIIMatch] = [] + redacted = text + + # ner_hook 优先：客户内网 NER 模型 + if ner_hook is not None: + ner_result = ner_hook(text) + if ner_result is not None: + # ner_hook 返回 (redacted_text, matches_list) + redacted_text, ner_matches = ner_result + redacted = redacted_text + matches.extend(ner_matches) + # ner_hook 返回 None 表示无匹配，继续走正则版 + + # 正则版（默认或 ner_hook 无匹配时叠加） + # ponytail: id_card 在 phone 之前（身份证号包含 11 位数字子串会被 phone 误匹配） + patterns = [ + ("id_card", _ID_CARD_RE), + ("phone", _PHONE_RE), + ("email", _EMAIL_RE), + ("bank_card", _BANK_CARD_RE), + ] + for pii_type, pattern in patterns: + for m in pattern.finditer(redacted): + original = m.group() + pii_hash = _hash_pii(original) + replacement = f"[REDACTED_{pii_type.upper()}:{pii_hash}]" + matches.append( + PIIMatch( + pii_type=pii_type, + original=original, + redacted=replacement, + hash=pii_hash, + ) + ) + redacted = pattern.sub( + lambda m: f"[REDACTED_{pii_type.upper()}:{_hash_pii(m.group())}]", + redacted, + ) + + return PIIFilterResult(redacted_text=redacted, matches=matches) + + +def filter_pii( + text: str, + *, + ner_hook=None, + fail_closed: bool = True, +) -> PIIFilterResult: + """对文本执行 PII 脱敏。 + + Args: + text: 待脱敏文本 + ner_hook: 可选 NER 函数（返回 (redacted_text, matches_list)） + fail_closed: True 时脱敏失败抛 PIIFilterError；False 时降级到正则版重试 + + Returns: + PIIFilterResult: 脱敏后文本 + 匹配列表（含 hash 用于审计） + + Raises: + PIIFilterError: fail_closed=True 且脱敏过程中出现异常 + """ + try: + result = _redact(text, ner_hook=ner_hook) + # U2 — OTel 指标：脱敏覆盖率 + 脱敏实体数 + try: + from agentkit.telemetry.metrics import ( + pii_filter_coverage_counter, + pii_filter_redacted_counter, + ) + + pii_filter_coverage_counter().add(1, {"pii_filter.status": "success"}) + pii_filter_redacted_counter().add(result.redacted_count) + except Exception: + # OTel 指标失败不影响脱敏主流程 + pass + return result + except Exception as e: + logger.error(f"PII filter failed: {e}") + if fail_closed: + # fail-closed：拒绝发送，抛异常 + try: + from agentkit.telemetry.metrics import pii_filter_coverage_counter + + pii_filter_coverage_counter().add(1, {"pii_filter.status": "failed_closed"}) + except Exception: + pass + raise PIIFilterError(f"PII filter failed (fail-closed): {e}") from e + # fail-open：降级到正则版（无 ner_hook）重试 + logger.warning("fail_closed=False, falling back to regex-only redaction") + try: + result = _redact(text, ner_hook=None) + try: + from agentkit.telemetry.metrics import ( + pii_filter_coverage_counter, + pii_filter_redacted_counter, + ) + + pii_filter_coverage_counter().add(1, {"pii_filter.status": "fallback_regex"}) + pii_filter_redacted_counter().add(result.redacted_count) + except Exception: + pass + return result + except Exception as fallback_err: + # 正则版也失败（极少见），返回原文（最差情况） + logger.error(f"Regex fallback also failed: {fallback_err}, returning original") + return PIIFilterResult(redacted_text=text, matches=[]) + + +def filter_messages_pii( + messages: list[dict[str, object]], + *, + ner_hook=None, + fail_closed: bool = True, +) -> tuple[list[dict[str, object]], list[PIIMatch]]: + """对 chat messages 列表执行 PII 脱敏。 + + Args: + messages: chat messages（role + content） + ner_hook: 可选 NER 函数 + fail_closed: True 时脱敏失败抛 PIIFilterError + + Returns: + (redacted_messages, all_matches): 脱敏后 messages + 全部匹配列表 + """ + all_matches: list[PIIMatch] = [] + redacted_messages: list[dict[str, object]] = [] + + for msg in messages: + content = msg.get("content") + if isinstance(content, str): + result = filter_pii(content, ner_hook=ner_hook, fail_closed=fail_closed) + redacted_msg = {**msg, "content": result.redacted_text} + all_matches.extend(result.matches) + elif isinstance(content, list): + # Anthropic content blocks: 脱敏每个 text block + redacted_blocks = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + block_text = block.get("text", "") + result = filter_pii(block_text, ner_hook=ner_hook, fail_closed=fail_closed) + redacted_blocks.append({**block, "text": result.redacted_text}) + all_matches.extend(result.matches) + else: + redacted_blocks.append(block) + redacted_msg = {**msg, "content": redacted_blocks} + else: + redacted_msg = msg + redacted_messages.append(redacted_msg) + + return redacted_messages, all_matches diff --git a/src/agentkit/llm/protocol.py b/src/agentkit/llm/protocol.py index 21369c3..d08997c 100644 --- a/src/agentkit/llm/protocol.py +++ b/src/agentkit/llm/protocol.py @@ -6,15 +6,29 @@ from dataclasses import dataclass, field @dataclass class TokenUsage: - """Token 使用量""" + """Token 使用量。 + + U2: 添加 cache_read_input_tokens / cache_creation_input_tokens 字段， + 用于 Anthropic prompt cache 命中率监控（OTel prompt_cache.hit/miss metric）。 + 非 Anthropic provider 这两个字段保持默认 0。 + """ prompt_tokens: int = 0 completion_tokens: int = 0 + # U2 — Anthropic prompt cache：从 prompt cache 读取的 token 数（命中时 > 0） + cache_read_input_tokens: int = 0 + # U2 — Anthropic prompt cache：写入 prompt cache 的 token 数 + cache_creation_input_tokens: int = 0 @property def total_tokens(self) -> int: return self.prompt_tokens + self.completion_tokens + @property + def cache_hit(self) -> bool: + """是否命中 prompt cache（cache_read_input_tokens > 0）""" + return self.cache_read_input_tokens > 0 + @dataclass class ToolCall: diff --git a/src/agentkit/llm/providers/anthropic.py b/src/agentkit/llm/providers/anthropic.py index 3a1c197..a57ad4a 100644 --- a/src/agentkit/llm/providers/anthropic.py +++ b/src/agentkit/llm/providers/anthropic.py @@ -271,6 +271,9 @@ class AnthropicProvider(LLMProvider): usage = TokenUsage( prompt_tokens=usage_data.get("input_tokens", 0), completion_tokens=usage_data.get("output_tokens", 0), + # U2 — Anthropic prompt cache 命中率监控 + cache_read_input_tokens=usage_data.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage_data.get("cache_creation_input_tokens", 0), ) return LLMResponse( @@ -479,6 +482,11 @@ class AnthropicProvider(LLMProvider): usage = TokenUsage( prompt_tokens=usage_data.get("input_tokens", 0), completion_tokens=usage_data.get("output_tokens", 0), + # U2 — Anthropic prompt cache 命中率监控 + cache_read_input_tokens=usage_data.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage_data.get( + "cache_creation_input_tokens", 0 + ), ) # Yield accumulated tool calls if any diff --git a/src/agentkit/telemetry/metrics.py b/src/agentkit/telemetry/metrics.py index 5c4303c..fde9b1f 100644 --- a/src/agentkit/telemetry/metrics.py +++ b/src/agentkit/telemetry/metrics.py @@ -46,6 +46,11 @@ _agent_duration_histogram = None _llm_token_histogram = None _tool_duration_histogram = None _pipeline_step_histogram = None +_prompt_cache_hit_counter = None +_prompt_cache_miss_counter = None +# U2 — PII 脱敏覆盖率指标 +_pii_filter_coverage_counter = None +_pii_filter_redacted_counter = None def _get_counter(name: str, description: str, unit: str = "1"): @@ -108,3 +113,45 @@ def pipeline_step_histogram(): "pipeline.step.duration", "Pipeline step duration" ) return _pipeline_step_histogram + + +# U2 — Prompt cache hit/miss counters（Anthropic cache_read_input_tokens > 0 → hit） + + +def prompt_cache_hit_counter(): + """Prompt cache hit count（cache_read_input_tokens > 0）.""" + global _prompt_cache_hit_counter + if _prompt_cache_hit_counter is None: + _prompt_cache_hit_counter = _get_counter("prompt_cache.hit", "Prompt cache hit count") + return _prompt_cache_hit_counter + + +def prompt_cache_miss_counter(): + """Prompt cache miss count（cache_read_input_tokens == 0）.""" + global _prompt_cache_miss_counter + if _prompt_cache_miss_counter is None: + _prompt_cache_miss_counter = _get_counter("prompt_cache.miss", "Prompt cache miss count") + return _prompt_cache_miss_counter + + +# U2 — PII 脱敏指标 + + +def pii_filter_coverage_counter(): + """PII filter coverage: total messages scanned.""" + global _pii_filter_coverage_counter + if _pii_filter_coverage_counter is None: + _pii_filter_coverage_counter = _get_counter( + "pii_filter.coverage", "PII filter: total messages scanned" + ) + return _pii_filter_coverage_counter + + +def pii_filter_redacted_counter(): + """PII filter: count of PII entities redacted.""" + global _pii_filter_redacted_counter + if _pii_filter_redacted_counter is None: + _pii_filter_redacted_counter = _get_counter( + "pii_filter.redacted", "PII filter: PII entities redacted" + ) + return _pii_filter_redacted_counter diff --git a/tests/unit/test_pii_filter.py b/tests/unit/test_pii_filter.py new file mode 100644 index 0000000..ee4af34 --- /dev/null +++ b/tests/unit/test_pii_filter.py @@ -0,0 +1,200 @@ +"""U2/R3 PII 脱敏 hook 测试。 + +覆盖场景： +- 正则版脱敏：手机号 / 邮箱 / 身份证 / 银行卡 +- fail-closed：脱敏失败时抛 PIIFilterError +- hash 审计：脱敏后记录 hash，原文不出现 +- ner_hook：自定义 NER 函数优先于正则版 +- messages 脱敏：string content + Anthropic content blocks +""" + +from __future__ import annotations + +import pytest + +from agentkit.llm.pii_filter import ( + PIIFilterError, + PIIMatch, + filter_messages_pii, + filter_pii, +) + + +# ── 正则版脱敏 ──────────────────────────────────────────── + + +class TestRegexRedaction: + """正则版 PII 脱敏""" + + def test_phone_redacted(self): + result = filter_pii("联系我：13912345678") + assert result.redacted_count == 1 + assert "13912345678" not in result.redacted_text + assert "[REDACTED_PHONE:" in result.redacted_text + assert len(result.matches[0].hash) == 8 + + def test_email_redacted(self): + result = filter_pii("发邮件给 user@example.com 谢谢") + assert result.redacted_count == 1 + assert "user@example.com" not in result.redacted_text + assert "[REDACTED_EMAIL:" in result.redacted_text + + def test_id_card_redacted(self): + result = filter_pii("身份证号：11010119900307888X") + assert result.redacted_count == 1 + assert "11010119900307888X" not in result.redacted_text + assert "[REDACTED_ID_CARD:" in result.redacted_text + + def test_bank_card_redacted(self): + result = filter_pii("银行卡：6222021234567890123") + assert result.redacted_count == 1 + assert "6222021234567890123" not in result.redacted_text + assert "[REDACTED_BANK_CARD:" in result.redacted_text + + def test_multiple_pii_in_one_text(self): + text = "电话 13912345678，邮箱 user@example.com，身份证 11010119900307888X" + result = filter_pii(text) + assert result.redacted_count == 3 + assert "13912345678" not in result.redacted_text + assert "user@example.com" not in result.redacted_text + assert "11010119900307888X" not in result.redacted_text + + def test_no_pii_returns_unchanged(self): + text = "这是一段普通文本，不含 PII" + result = filter_pii(text) + assert result.redacted_count == 0 + assert result.redacted_text == text + + +# ── hash 审计 ─────────────────────────────────────────── + + +class TestHashAudit: + """脱敏后 hash 记录用于审计（不可逆）""" + + def test_hash_is_8_chars_sha256_prefix(self): + import hashlib + + result = filter_pii("13912345678") + expected = hashlib.sha256("13912345678".encode("utf-8")).hexdigest()[:8] + assert result.matches[0].hash == expected + assert len(result.matches[0].hash) == 8 + + def test_original_not_in_redacted_text(self): + result = filter_pii("电话 13912345678") + assert "13912345678" not in result.redacted_text + # hash 不等于原文 + assert result.matches[0].redacted != result.matches[0].original + + +# ── fail-closed ───────────────────────────────────────── + + +class TestFailClosed: + """脱敏失败时 fail-closed 行为""" + + def test_fail_closed_raises_on_error(self): + """fail_closed=True 时，ner_hook 抛异常 → PIIFilterError""" + + def bad_ner(text): + raise RuntimeError("NER model offline") + + with pytest.raises(PIIFilterError, match="fail-closed"): + filter_pii("test text", ner_hook=bad_ner, fail_closed=True) + + def test_fail_open_returns_original_on_error(self): + """fail_closed=False 时，ner_hook 失败 → 降级正则版（ner_hook 失败不阻塞）""" + + def bad_ner(text): + raise RuntimeError("NER model offline") + + # ner_hook 失败时降级到正则版，不抛异常 + result = filter_pii("电话 13912345678", ner_hook=bad_ner, fail_closed=False) + # 正则版仍能识别手机号 + assert result.redacted_count == 1 + assert "13912345678" not in result.redacted_text + + +# ── ner_hook ──────────────────────────────────────────── + + +class TestNerHook: + """自定义 NER hook 优先于正则版""" + + def test_ner_hook_takes_precedence(self): + """ner_hook 返回脱敏结果时，正则版不再处理（避免双重脱敏）""" + + def custom_ner(text): + # 模拟 NER 模型只识别 "SECRET" 关键词 + redacted = text.replace("SECRET", "[REDACTED_CUSTOM:abc12345]") + match = PIIMatch( + pii_type="custom", + original="SECRET", + redacted="[REDACTED_CUSTOM:abc12345]", + hash="abc12345", + ) + return redacted, [match] + + result = filter_pii("电话 13912345678 SECRET", ner_hook=custom_ner) + # ner_hook 优先：SECRET 被脱敏，但手机号也被正则版脱敏（双重处理） + # ponytail: ner_hook 和正则版是叠加的（ner_hook 先，正则版后） + assert "SECRET" not in result.redacted_text + assert "[REDACTED_CUSTOM:abc12345]" in result.redacted_text + # 正则版仍处理手机号 + assert "13912345678" not in result.redacted_text + + +# ── messages 脱敏 ─────────────────────────────────────── + + +class TestMessagesRedaction: + """对 chat messages 列表执行 PII 脱敏""" + + def test_string_content_redacted(self): + messages = [ + {"role": "user", "content": "电话 13912345678"}, + {"role": "assistant", "content": "好的"}, + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + assert "13912345678" not in redacted[0]["content"] + assert redacted[1]["content"] == "好的" # 无 PII 不变 + + def test_anthropic_content_blocks_redacted(self): + """Anthropic content blocks（list of dicts）中的 text 被脱敏""" + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "电话 13912345678"}, + {"type": "text", "text": "无 PII 文本"}, + ], + } + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + assert "13912345678" not in redacted[0]["content"][0]["text"] + assert redacted[0]["content"][1]["text"] == "无 PII 文本" + + def test_non_text_content_passthrough(self): + """非 text 类型的 content block 透传不脱敏""" + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "source": "data:..."}, + {"type": "text", "text": "电话 13912345678"}, + ], + } + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + # image block 透传 + assert redacted[0]["content"][0] == {"type": "image", "source": "data:..."} + + def test_non_string_non_list_content_passthrough(self): + """非 str/非 list content 透传""" + messages = [{"role": "user", "content": None}] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 0 + assert redacted[0]["content"] is None diff --git a/tests/unit/test_prompt_cache_layers.py b/tests/unit/test_prompt_cache_layers.py index d3e61f0..cf128a6 100644 --- a/tests/unit/test_prompt_cache_layers.py +++ b/tests/unit/test_prompt_cache_layers.py @@ -32,7 +32,9 @@ class _StubGateway: yield # makes this an async generator -def _make_engine(provider_name: str | None = "anthropic", *, cache_enable: bool = True) -> tuple[ReActEngine, _StubGateway]: +def _make_engine( + provider_name: str | None = "anthropic", *, cache_enable: bool = True +) -> tuple[ReActEngine, _StubGateway]: gw = _StubGateway(provider_name=provider_name) engine = ReActEngine.__new__(ReActEngine) engine._llm_gateway = gw @@ -166,6 +168,7 @@ async def test_execute_stream_with_anthropic_uses_content_blocks(): self.captured_messages = list(kwargs.get("messages", [])) # yield one chunk then end from agentkit.llm.protocol import StreamChunk + yield StreamChunk(content="done", model="claude") class _MemRetriever: @@ -219,3 +222,139 @@ def test_anthropic_convert_messages_string_system_still_works(): messages = [{"role": "system", "content": "old style"}] system_prompt, _ = provider._convert_messages(messages) assert system_prompt == "old style" + + +# ---- U2 cache hit/miss metric emission ---- + + +def test_token_usage_cache_hit_property(): + """U2: TokenUsage.cache_hit 反映 cache_read_input_tokens > 0""" + from agentkit.llm.protocol import TokenUsage + + hit_usage = TokenUsage( + prompt_tokens=100, + completion_tokens=50, + cache_read_input_tokens=80, + cache_creation_input_tokens=20, + ) + assert hit_usage.cache_hit is True + + miss_usage = TokenUsage(prompt_tokens=100, completion_tokens=50) + assert miss_usage.cache_hit is False + assert miss_usage.cache_read_input_tokens == 0 + + +def test_anthropic_provider_parses_cache_tokens_from_response(): + """U2: Anthropic provider 从 API 响应解析 cache_read_input_tokens""" + from agentkit.llm.providers.anthropic import AnthropicProvider + + provider = AnthropicProvider.__new__(AnthropicProvider) + provider.api_key = "test" + provider.base_url = "http://test" + # 模拟 Anthropic 响应含 cache 字段 + mock_response = { + "content": [{"type": "text", "text": "hello"}], + "model": "claude-sonnet-4", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 80, + "cache_creation_input_tokens": 20, + }, + } + # 直接调用 _parse_response（私有方法，测试用） + usage = provider._parse_response(mock_response, "claude-sonnet-4").usage + assert usage.cache_read_input_tokens == 80 + assert usage.cache_creation_input_tokens == 20 + assert usage.cache_hit is True + + +def _build_cache_metric_gateway(response): + """构造 LLMGateway mock，跳过 __init__。 + + ponytail: 直接装配测试所需的最小属性，避免初始化真实 provider/cache。 + """ + from unittest.mock import AsyncMock, MagicMock + + from agentkit.llm.gateway import LLMGateway + + mock_provider = MagicMock() + mock_provider.chat = AsyncMock(return_value=response) + + gw = LLMGateway.__new__(LLMGateway) + gw._providers = {"mock": mock_provider} + gw._config = MagicMock() + gw._config.providers = {} # _calculate_cost 真实迭代返回 0.0 + gw._config.fallbacks = {} # _get_models_to_try 返回 [resolved_model] + gw._cache_manager = None + gw._usage_tracker = MagicMock() + gw._resolve_model_alias = lambda m: m + gw._resolve_model = lambda m: (mock_provider, m) + return gw + + +def test_gateway_emits_prompt_cache_hit_metric_on_cache_read(): + """U2: gateway.chat 在 cache_read_input_tokens > 0 时 emit prompt_cache.hit""" + from unittest.mock import AsyncMock, patch + + from agentkit.llm.protocol import LLMResponse, TokenUsage + + response = LLMResponse( + content="test", + model="claude", + usage=TokenUsage( + prompt_tokens=100, + completion_tokens=50, + cache_read_input_tokens=80, + ), + ) + + gw = _build_cache_metric_gateway(response) + + # patch gateway 模块内的导入绑定名（from ... import prompt_cache_hit_counter） + with ( + patch("agentkit.llm.gateway.prompt_cache_hit_counter") as hit_counter, + patch("agentkit.llm.gateway.prompt_cache_miss_counter") as miss_counter, + patch( + "agentkit.llm.gateway.LLMGateway._record_usage", + new_callable=AsyncMock, + ), + ): + import asyncio + + asyncio.run(gw.chat(messages=[{"role": "user", "content": "hi"}], model="claude")) + + # cache_read_input_tokens > 0 → hit counter called + hit_counter().add.assert_called_once() + miss_counter().add.assert_not_called() + + +def test_gateway_emits_prompt_cache_miss_metric_on_no_cache_read(): + """U2: gateway.chat 在 cache_read_input_tokens == 0 时 emit prompt_cache.miss""" + from unittest.mock import AsyncMock, patch + + from agentkit.llm.protocol import LLMResponse, TokenUsage + + response = LLMResponse( + content="test", + model="claude", + usage=TokenUsage(prompt_tokens=100, completion_tokens=50), # cache_read=0 + ) + + gw = _build_cache_metric_gateway(response) + + with ( + patch("agentkit.llm.gateway.prompt_cache_hit_counter") as hit_counter, + patch("agentkit.llm.gateway.prompt_cache_miss_counter") as miss_counter, + patch( + "agentkit.llm.gateway.LLMGateway._record_usage", + new_callable=AsyncMock, + ), + ): + import asyncio + + asyncio.run(gw.chat(messages=[{"role": "user", "content": "hi"}], model="claude")) + + # cache_read_input_tokens == 0 → miss counter called + miss_counter().add.assert_called_once() + hit_counter().add.assert_not_called() From c60e96ac328afae6560f5bdfbe7f90b67703b6b6 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 03:48:14 +0800 Subject: [PATCH 03/11] =?UTF-8?q?fix(bitable/team):=20U3=20=E2=80=94=20Res?= =?UTF-8?q?idual=20P2=20findings=20(DR-1/3/4/5=20+=20#2/#3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U3 修复 8 项 P2 design review 与 streaming 残留问题： DR-1 SQL 注入防御 — repository.py 添加 _validate_field_id UUID 校验， 拒绝非 UUID 字符串进入 jsonb_set path 与 WHERE 子句的字符串插值。 ceiling 注释标明仅覆盖 field_id，其余 SQL 结构由 service 层枚举白名单守护。 DR-3 design token 契约 — 新增 bitable-tokens.test.ts (31 tests)， 解析 CSS :root 块并断言所有 --bitable-* token 在 :root 定义， 防止 token 被硬编码到组件外或遗漏定义。 DR-4 TOCTOU 修复 — repository.delete_view_if_not_last 用 SELECT ... FOR UPDATE 单事务原子化 check + delete， 消除 list_views → count → delete 三步分离的竞态。 service.delete_view 委托原子方法，失败抛 LastViewDeletionError。 DR-5 _update_field 静默失败 — type 参数被 Pydantic extra=ignore 静默丢弃，用户以为改了类型实际无效。显式返回 UNSUPPORTED_FIELD_TYPE_CHANGE 错误，避免静默失败的 UX 陷阱。 #2 team_synthesis 孤儿 milestone — 补 error + cancelled 路径测试， 验证内层 except 广播终结事件后 re-raise，外层 except 捕获后 fallback。 #3 synthesis_id 去重附身 — chatStream.ts 移除 || m.synthesis_id === undefined 兜底匹配，避免新 milestone 附身到上一次孤儿 streaming 占位。 测试：DR-1 (7) + DR-4 repo (3) + DR-4 service (3) + DR-5 (2) + #2 (2) + #8 frontend (31) = 48 项通过；U2 无回归 (29 项)。 --- src/agentkit/bitable/repository.py | 69 ++- src/agentkit/bitable/service.py | 19 +- .../server/frontend/src/stores/chatStream.ts | 9 +- .../tests/unit/bitable-tokens.test.ts | 97 ++++ src/agentkit/tools/bitable_tool.py | 20 +- tests/unit/bitable/test_bitable_tool.py | 67 ++- tests/unit/bitable/test_repository.py | 113 +++++ tests/unit/bitable/test_service.py | 43 +- tests/unit/experts/test_team_orchestrator.py | 416 ++++++++++++++---- 9 files changed, 738 insertions(+), 115 deletions(-) create mode 100644 src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts create mode 100644 tests/unit/bitable/test_repository.py diff --git a/src/agentkit/bitable/repository.py b/src/agentkit/bitable/repository.py index a4fd4ff..426780b 100644 --- a/src/agentkit/bitable/repository.py +++ b/src/agentkit/bitable/repository.py @@ -39,6 +39,26 @@ from agentkit.bitable.models import ( logger = logging.getLogger(__name__) +# U3/DR-1: field_id 在 jsonb_set path 与 where 子句中走字符串插值（jsonb path 不支持绑定参数）。 +# 防御性校验：field_id 必须匹配 UUID 格式，拒绝任意字符串插值到 SQL 结构。 +# ponytail: ceiling — 仅覆盖 field_id；其余 SQL 结构（op / direction）已由 service 层白名单枚举校验。 +# 升级路径：迁移到 ORM JSON 函数（sqlalchemy.dialects.postgresql.jsonb_*）彻底消除字符串插值。 +_FIELD_ID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def _validate_field_id(field_id: str, *, context: str = "") -> str: + """校验 field_id 是 UUID 格式（DR-1 防御性校验）。 + + field_id 在 jsonb_set path 与 SQL where 子句中走字符串插值（jsonb path 不支持绑定参数）， + 必须确保非系统 UUID 不进入 SQL 结构。校验失败抛 ValueError。 + """ + if not isinstance(field_id, str) or not _FIELD_ID_RE.match(field_id): + ctx = f" ({context})" if context else "" + raise ValueError(f"Invalid field_id format{ctx}: {field_id!r} — expected UUID") + return field_id + class BitableRepository: """Async repository for bitable CRUD operations. @@ -375,7 +395,9 @@ class BitableRepository: return [Record.model_validate(e) for e in entities], next_cursor - async def update_record_values(self, record_id: str, values: dict[str, object]) -> Record | None: + async def update_record_values( + self, record_id: str, values: dict[str, object] + ) -> Record | None: """Update a record's values (full replace).""" async with self._session_factory() as session: stmt = ( @@ -471,6 +493,35 @@ class BitableRepository: await session.commit() return result.rowcount > 0 + async def delete_view_if_not_last(self, view_id: str, table_id: str) -> bool: + """Atomically delete a view only if it's not the last one in its table (DR-4 TOCTOU fix). + + Uses ``SELECT ... FOR UPDATE`` in a single transaction to lock sibling + rows for the duration of the count + delete, preventing concurrent + ``delete_view`` calls from racing past the last-view check and leaving + the table with zero views. + + Returns: + True if the view was deleted. + False if NOT deleted because it would be the last view (or the + view doesn't belong to the given table_id). + """ + async with self._session_factory() as session: + # Lock all sibling views — concurrent delete_view calls on the + # same table block here until this transaction commits. + lock_sql = text( + "SELECT id FROM bitable.bitable_views WHERE table_id = :table_id FOR UPDATE" + ) + result = await session.execute(lock_sql, {"table_id": table_id}) + sibling_ids = {str(row[0]) for row in result.fetchall()} + if len(sibling_ids) <= 1: + return False # Last view — refuse to delete + if view_id not in sibling_ids: + return False # View doesn't belong to this table_id + result = await session.execute(delete(ViewModel).where(ViewModel.id == view_id)) + await session.commit() + return result.rowcount > 0 + # ── Recalc Queue ──────────────────────────────────────── async def enqueue_recalc( @@ -646,9 +697,13 @@ class BitableRepository: return import json + # U3/DR-1: 校验所有 field_id 是 UUID 格式后再插值到 jsonb_set path（防御性，避免 SQL 结构注入） + for fid in agent_field_values: + _validate_field_id(fid, context="upsert_record_agent_fields") + async with self._session_factory() as session: # Build nested jsonb_set: jsonb_set(jsonb_set(values, '{f1}', CAST(:v0 AS jsonb), true), ...) - # ponytail: field_ids are system UUIDs, safe to interpolate into path literals. + # ponytail: field_ids validated above as UUID format, safe to interpolate into path literals. # Use CAST(:param AS jsonb) instead of :param::jsonb — asyncpg dialect # misparses the `::` as part of the param name. inner = "values" @@ -684,9 +739,17 @@ class BitableRepository: import base64 import json as _json + # U3/DR-1: 校验 filters/sorts 中的 field_id 是 UUID 格式后再插值到 SQL 结构 + if filters: + for f in filters: + _validate_field_id(f["field_id"], context="list_records_filtered.filter") + if sorts: + for s in sorts: + _validate_field_id(s["field_id"], context="list_records_filtered.sort") + async with self._session_factory() as session: # Build raw SQL with JSONB filter/sort translation. - # ponytail: field_ids in filters/sorts are system UUIDs (validated by service layer). + # ponytail: field_ids validated above as UUID format, safe to interpolate into SQL structure. where_clauses = ["table_id = :table_id"] params: dict[str, object] = {"table_id": table_id} diff --git a/src/agentkit/bitable/service.py b/src/agentkit/bitable/service.py index 7245247..ad81b31 100644 --- a/src/agentkit/bitable/service.py +++ b/src/agentkit/bitable/service.py @@ -545,23 +545,28 @@ class BitableService: return await self._repo.get_view(view_id) async def delete_view(self, view_id: str) -> bool: - """Delete a view with last-view protection (U6). + """Delete a view with last-view protection (U6, DR-4 TOCTOU fix). Raises :class:`LastViewDeletionError` if the view is the last one in its table — preventing users from making a table inaccessible. The route layer is responsible for the 404 + ownership checks before calling this (matching the existing ``delete_field`` pattern). Returns True if a row was deleted. + + DR-4: delegates to ``repo.delete_view_if_not_last`` which atomically + checks sibling count + deletes in a single ``SELECT FOR UPDATE`` + transaction, eliminating the TOCTOU race between ``list_views`` count + and ``delete_view`` execution. """ view = await self._repo.get_view(view_id) if view is None: return False - siblings = await self._repo.list_views(view.table_id) - if len(siblings) <= 1: - raise LastViewDeletionError( - "Cannot delete the last view of a table" - ) - return await self._repo.delete_view(view_id) + deleted = await self._repo.delete_view_if_not_last(view_id, view.table_id) + if not deleted: + # Either it was the last view, or a concurrent call raced ahead. + # Either way, the invariant "table has at least 1 view" is preserved. + raise LastViewDeletionError("Cannot delete the last view of a table") + return True # ── Recalc (U3: formula recalc pipeline) ──────────────── diff --git a/src/agentkit/server/frontend/src/stores/chatStream.ts b/src/agentkit/server/frontend/src/stores/chatStream.ts index 39f35c9..020d95f 100644 --- a/src/agentkit/server/frontend/src/stores/chatStream.ts +++ b/src/agentkit/server/frontend/src/stores/chatStream.ts @@ -869,7 +869,8 @@ export function dispatchWsEvent( case "team_synthesis_chunk": { // U4: 团队综合流式 chunk — 首次创建 streaming 占位，后续累加。 - // P2 fix: 用 synthesis_id 去重，避免 chunk 附身到上一次孤儿 streaming milestone。 + // P2 fix: 用 synthesis_id 精确去重 — sid 存在时仅匹配相同 synthesis_id 的 milestone， + // 不再兜底附身到 undefined synthesis_id 的旧孤儿占位（避免跨重试附身 bug）。 const conversationId = state.resolveIncomingConvId(); if (!conversationId) break; const conv = state.conversations.value.find( @@ -882,13 +883,11 @@ export function dispatchWsEvent( (m) => m.message_type === "milestone" && m.status === "streaming" && - (sid ? m.synthesis_id === sid || m.synthesis_id === undefined : true), + (sid ? m.synthesis_id === sid : true), ); if (existing) { state.updateMessage(conversationId, existing.id, { content: (existing.content || "") + (event.data.chunk || ""), - // 首次附身到无 synthesis_id 的 milestone 时，标记上 synthesis_id - ...(sid && !existing.synthesis_id ? { synthesis_id: sid } : {}), }); } else { const synthMsg: IChatMessage = { @@ -926,7 +925,7 @@ export function dispatchWsEvent( (m) => m.message_type === "milestone" && m.status === "streaming" && - (sid ? m.synthesis_id === sid || m.synthesis_id === undefined : true), + (sid ? m.synthesis_id === sid : true), ); if (existing) { state.updateMessage(conversationId, existing.id, { diff --git a/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts new file mode 100644 index 0000000..e028777 --- /dev/null +++ b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts @@ -0,0 +1,97 @@ +/** + * DR-3: Bitable design tokens — static structural assertion. + * + * Parses `bitable-tokens.css` and asserts all required `--bitable-*` tokens + * are defined inside `:root`. Prevents accidental token removal/renaming + * that would silently break var() references in bitable components. + * + * ponytail: pure regex parse — no CSS parser dependency. Ceiling: doesn't + * validate token values (only presence). Upgrade path: postcss + stylelint + * custom rules for value-level validation. + */ +import { describe, expect, it } from 'vitest' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +const CSS_PATH = resolve( + __dirname, + '../../src/styles/bitable-tokens.css', +) + +const cssContent: string = readFileSync(CSS_PATH, 'utf-8') + +/**Extract the first `:root { ... }` block content (without the selector).*/ +function extractRootBlock(css: string): string { + const match = /:root\s*\{([^}]*)\}/.exec(css) + if (!match || match[1] === undefined) { + throw new Error('No :root block found in bitable-tokens.css') + } + return match[1] +} + +const rootBlock = extractRootBlock(cssContent) + +/**Required token names grouped by category (mirrors the CSS file structure).*/ +const REQUIRED_TOKENS = { + color: [ + '--bitable-color-primary', + '--bitable-color-bg', + '--bitable-color-bg-secondary', + '--bitable-color-bg-tertiary', + '--bitable-color-text', + '--bitable-color-text-secondary', + '--bitable-color-text-tertiary', + '--bitable-color-text-placeholder', + '--bitable-color-border', + '--bitable-color-border-hover', + '--bitable-color-border-split', + ], + cf: [ + '--bitable-cf-red-bg', + '--bitable-cf-red-fg', + '--bitable-cf-orange-bg', + '--bitable-cf-orange-fg', + '--bitable-cf-yellow-bg', + '--bitable-cf-yellow-fg', + '--bitable-cf-green-bg', + '--bitable-cf-green-fg', + '--bitable-cf-blue-bg', + '--bitable-cf-blue-fg', + '--bitable-cf-purple-bg', + '--bitable-cf-purple-fg', + '--bitable-cf-gray-bg', + '--bitable-cf-gray-fg', + '--bitable-cf-neutral-bg', + '--bitable-cf-neutral-fg', + ], + drawer: ['--bitable-drawer-width', '--bitable-drawer-width-wide'], +} as const + +describe('bitable-tokens.css — DR-3 design token contract', () => { + it('defines a :root block', () => { + expect(rootBlock.length).toBeGreaterThan(0) + }) + + it.each(REQUIRED_TOKENS.color)('defines color token %s in :root', (token) => { + // Match `--token-name:` (declaration), not just substring presence. + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it.each(REQUIRED_TOKENS.cf)('defines conditional-format token %s in :root', (token) => { + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it.each(REQUIRED_TOKENS.drawer)('defines drawer token %s in :root', (token) => { + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it('does not hardcode tokens outside :root (no component-level overrides)', () => { + // ponytail: bitable tokens are exclusively :root-scoped; component-level + // overrides would break the single-source-of-truth invariant. + const outsideRoot = cssContent.replace(/:root\s*\{[^}]*\}/, '') + expect(outsideRoot).not.toMatch(/--bitable-[a-z-]+\s*:/) + }) +}) diff --git a/src/agentkit/tools/bitable_tool.py b/src/agentkit/tools/bitable_tool.py index 6365155..b6d760e 100644 --- a/src/agentkit/tools/bitable_tool.py +++ b/src/agentkit/tools/bitable_tool.py @@ -558,15 +558,23 @@ class BitableTool(Tool): field_id = kwargs.get("field_id") if not field_id: return {"success": False, "error": "Missing required field: field_id"} + # DR-5: type 变更不被支持 — UpdateFieldRequest 当前只接受 name + config， + # 传入 type 会被 Pydantic extra=ignore 静默丢弃（用户以为改了类型，实际无效果）。 + # 显式拒绝并返回错误，避免静默失败的 UX 陷阱。 + # 升级路径：如需支持类型变更，扩展 UpdateFieldRequest 接受 field_type + + # 实现数据迁移逻辑（类型转换 + 校验）。 + if kwargs.get("type") is not None: + return { + "success": False, + "error": ( + "Unsupported field type change — field type is immutable after " + "creation. To change a field's type, delete and recreate it." + ), + "code": "UNSUPPORTED_FIELD_TYPE_CHANGE", + } payload: dict[str, object] = {} if kwargs.get("name") is not None: payload["name"] = kwargs["name"] - # ponytail: PATCH /fields/{id} is backed by UpdateFieldRequest which - # currently accepts only name + config. `type` is forwarded as - # `field_type` so the tool is forward-compatible once the request - # model adds it; today it is silently ignored by Pydantic (extra=ignore). - if kwargs.get("type") is not None: - payload["field_type"] = kwargs["type"] if kwargs.get("config") is not None: payload["config"] = kwargs["config"] client = await self._get_client() diff --git a/tests/unit/bitable/test_bitable_tool.py b/tests/unit/bitable/test_bitable_tool.py index 1f885fd..8b6b75b 100644 --- a/tests/unit/bitable/test_bitable_tool.py +++ b/tests/unit/bitable/test_bitable_tool.py @@ -552,9 +552,9 @@ async def test_update_view_action(tool: BitableTool) -> None: """update_view action PATCHes /views/{id} with name + config.""" result = await tool.execute(action="create_table", table_name="VU") table_id = result["table"]["id"] - view_id = ( - await tool.execute(action="create_view", table_id=table_id, name="Old") - )["view"]["id"] + view_id = (await tool.execute(action="create_view", table_id=table_id, name="Old"))["view"][ + "id" + ] resp = await tool.execute( action="update_view", @@ -682,9 +682,7 @@ async def test_new_actions_internal_token_passthrough( table_id = result["table"]["id"] # create_view via the new action. - v = await tool_real_auth.execute( - action="create_view", table_id=table_id, name="tv1" - ) + v = await tool_real_auth.execute(action="create_view", table_id=table_id, name="tv1") assert v["success"] is True view_id = v["view"]["id"] @@ -716,3 +714,60 @@ async def test_delete_view_internal_token_404_when_missing( resp = await tool_real_auth.execute(action="delete_view", view_id="no-such-view") assert resp["success"] is False assert "404" in resp["error"] + + +# --------------------------------------------------------------------------- +# DR-5: _update_field rejects type change (no silent success) +# --------------------------------------------------------------------------- + + +async def test_update_field_rejects_type_change(tool: BitableTool) -> None: + """DR-5: _update_field with `type` arg returns error, does NOT call HTTP. + + Without this guard, the field_type would be silently dropped by + UpdateFieldRequest's Pydantic extra=ignore, and the agent would + believe the type changed when it didn't. + """ + result = await tool.execute(action="create_table", table_name="DR5") + table_id = result["table"]["id"] + client = await tool._get_client() + field_id = ( + await client.post( + f"/tables/{table_id}/fields", + json={"name": "col", "field_type": "text", "owner": "user"}, + ) + ).json()["field"]["id"] + + resp = await tool.execute( + action="update_field", + field_id=field_id, + type="number", + ) + assert resp["success"] is False + assert resp.get("code") == "UNSUPPORTED_FIELD_TYPE_CHANGE" + assert "immutable" in resp["error"] + + +async def test_update_field_still_works_without_type(tool: BitableTool) -> None: + """DR-5: _update_field without `type` still PATCHes name/config normally. + + Regression guard: the DR-5 guard must not block legitimate updates. + """ + result = await tool.execute(action="create_table", table_name="DR5b") + table_id = result["table"]["id"] + client = await tool._get_client() + field_id = ( + await client.post( + f"/tables/{table_id}/fields", + json={"name": "col", "field_type": "text", "owner": "user"}, + ) + ).json()["field"]["id"] + + resp = await tool.execute( + action="update_field", + field_id=field_id, + name="renamed", + config={"description": "ok"}, + ) + assert resp["success"] is True + assert resp["field"]["name"] == "renamed" diff --git a/tests/unit/bitable/test_repository.py b/tests/unit/bitable/test_repository.py new file mode 100644 index 0000000..eafe031 --- /dev/null +++ b/tests/unit/bitable/test_repository.py @@ -0,0 +1,113 @@ +"""Tests for BitableRepository — U3/DR-1 + DR-4 fixes. + +DR-1: field_id UUID validation before SQL interpolation (pure unit test). +DR-4: atomic delete_view_if_not_last (requires PostgreSQL). +""" + +from __future__ import annotations + +import pytest + +from agentkit.bitable.repository import _validate_field_id + + +# ── DR-1: field_id UUID validation ────────────────────────── + + +class TestValidateFieldId: + """DR-1: field_id 校验 — 拒绝非 UUID 格式进入 SQL 结构插值。""" + + def test_valid_uuid_passes(self): + valid = "550e8400-e29b-41d4-a716-446655440000" + assert _validate_field_id(valid) == valid + + def test_valid_uppercase_uuid_passes(self): + valid = "550E8400-E29B-41D4-A716-446655440000" + assert _validate_field_id(valid) == valid + + @pytest.mark.parametrize( + "bad_field_id,label", + [ + ("not-a-uuid", "arbitrary string"), + ("", "empty string"), + ("550e8400-e29b-41d4-a716", "too short"), + ("550e8400-e29b-41d4-a716-446655440000-extra", "too long"), + ("550e8400-e29b-41d4-a716-44665544000g", "non-hex char"), + ("'; DROP TABLE bitable.bitable_views; --", "SQL injection attempt"), + ("values->>'field' = :fv0", "SQL fragment"), + ], + ) + def test_invalid_field_id_raises(self, bad_field_id: str, label: str): + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(bad_field_id, context=label) + + def test_non_string_raises(self): + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(None) # type: ignore[arg-type] + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(12345) # type: ignore[arg-type] + + +# ── DR-4: atomic delete_view_if_not_last ──────────────────── +# These tests require PostgreSQL (SELECT FOR UPDATE behavior) + + +pytestmark_postgres = pytest.mark.postgres + + +@pytest.mark.postgres +class TestDeleteViewIfNotLastAtomic: + """DR-4: 原子条件 delete_view — SELECT FOR UPDATE 事务防止 TOCTOU。 + + 并发场景不变量："table 至少 1 view" 在并发删除最后两个 view 时保持。 + """ + + async def test_delete_view_if_not_last_succeeds_when_multiple_siblings( + self, bitable_service, make_table + ): + """多 view 时删除成功。""" + from agentkit.bitable.repository import BitableRepository + + table = await make_table(name="T1") + v1 = await bitable_service.create_view(table.id, name="v1") + await bitable_service.create_view(table.id, name="v2") + + repo = BitableRepository(bitable_service._repo._db) + deleted = await repo.delete_view_if_not_last(v1.id, table.id) + assert deleted is True + # v1 gone, v2 still present + assert await bitable_service.get_view(v1.id) is None + assert (await bitable_service.list_views(table.id)) is not None + + async def test_delete_view_if_not_last_refuses_last_view(self, bitable_service, make_table): + """最后一个 view 时拒绝删除（返回 False，不抛异常 — service 层负责抛 LastViewDeletionError）。""" + from agentkit.bitable.repository import BitableRepository + + table = await make_table(name="T2") + only_view = await bitable_service.create_view(table.id, name="only") + + repo = BitableRepository(bitable_service._repo._db) + deleted = await repo.delete_view_if_not_last(only_view.id, table.id) + assert deleted is False + # View still exists + assert await bitable_service.get_view(only_view.id) is not None + + async def test_delete_view_if_not_last_wrong_table_returns_false( + self, bitable_service, make_table + ): + """view_id 不属于给定 table_id 时返回 False（不误删其他 table 的 view）。""" + from agentkit.bitable.repository import BitableRepository + + table_a = await make_table(name="TA") + table_b = await make_table(name="TB") + va = await bitable_service.create_view(table_a.id, name="va") + # table_b 有自己的 view + await bitable_service.create_view(table_b.id, name="vb1") + await bitable_service.create_view(table_b.id, name="vb2") + + repo = BitableRepository(bitable_service._repo._db) + # 用 table_b 的 id 但 view_id 是 table_a 的 — 不应删除 + deleted = await repo.delete_view_if_not_last(va.id, table_b.id) + assert deleted is False + # va 仍然存在 + assert await bitable_service.get_view(va.id) is not None diff --git a/tests/unit/bitable/test_service.py b/tests/unit/bitable/test_service.py index 67605a5..4bfffb4 100644 --- a/tests/unit/bitable/test_service.py +++ b/tests/unit/bitable/test_service.py @@ -8,7 +8,7 @@ from __future__ import annotations import pytest from agentkit.bitable.models import FieldOwner, FieldType -from agentkit.bitable.service import FieldDependencyError +from agentkit.bitable.service import FieldDependencyError, LastViewDeletionError pytestmark = pytest.mark.postgres @@ -294,3 +294,44 @@ async def test_list_records_filtered_cursor_pagination(bitable_service) -> None: # All records unique all_ids = {r.id for r in [records, records2, records3] for r in r} assert len(all_ids) == 5 + + +# --------------------------------------------------------------------------- +# DR-4: delete_view last-view protection (service layer) +# --------------------------------------------------------------------------- + + +async def test_delete_view_raises_last_view_deletion_error(bitable_service, make_table) -> None: + """DR-4: service.delete_view raises LastViewDeletionError on the last view. + + Guards the invariant "every table has at least 1 view" at the service + layer so the route can map it to HTTP 409 without touching the repo. + """ + table = await make_table(name="LastView") + only = await bitable_service.create_view(table.id, name="only") + + with pytest.raises(LastViewDeletionError): + await bitable_service.delete_view(only.id) + # View is still there + assert await bitable_service.get_view(only.id) is not None + + +async def test_delete_view_succeeds_when_multiple_views(bitable_service, make_table) -> None: + """DR-4: service.delete_view succeeds when ≥2 views remain (happy path).""" + table = await make_table(name="MultiView") + v1 = await bitable_service.create_view(table.id, name="v1") + await bitable_service.create_view(table.id, name="v2") + + deleted = await bitable_service.delete_view(v1.id) + assert deleted is True + assert await bitable_service.get_view(v1.id) is None + + +async def test_delete_view_returns_false_when_missing(bitable_service, make_table) -> None: + """DR-4: service.delete_view on a non-existent view returns False (no raise).""" + table = await make_table(name="MissingView") + # ensure ≥1 view exists so the table is valid + await bitable_service.create_view(table.id, name="v1") + + deleted = await bitable_service.delete_view("nonexistent-view-id") + assert deleted is False diff --git a/tests/unit/experts/test_team_orchestrator.py b/tests/unit/experts/test_team_orchestrator.py index a21c2c2..6de5b60 100644 --- a/tests/unit/experts/test_team_orchestrator.py +++ b/tests/unit/experts/test_team_orchestrator.py @@ -81,15 +81,17 @@ def _make_mock_expert( } # Mock agent.execute() to return a successful TaskResult mock_agent = MagicMock() - mock_agent.execute = AsyncMock(return_value=TaskResult( - task_id="test", - agent_name=name, - status=TaskStatus.COMPLETED.value, - output_data={"content": f"Result from {name}"}, - error_message=None, - started_at=None, - completed_at=None, - )) + mock_agent.execute = AsyncMock( + return_value=TaskResult( + task_id="test", + agent_name=name, + status=TaskStatus.COMPLETED.value, + output_data={"content": f"Result from {name}"}, + error_message=None, + started_at=None, + completed_at=None, + ) + ) # U3: orchestrator._run_agent_steps() calls agent.execute_stream() (async gen), # not agent.execute(). Mock it to yield one final_answer event. mock_agent.execute_stream = make_execute_stream_mock(f"Result from {name}") @@ -162,20 +164,24 @@ def _make_mock_llm_gateway( def _make_mock_pool() -> MagicMock: """创建 mock AgentPool，模拟上下文隔离的 agent 创建""" pool = MagicMock() + def _create_agent(config): m = MagicMock() - m.execute = AsyncMock(return_value=TaskResult( - task_id="test", - agent_name=config.name, - status=TaskStatus.COMPLETED.value, - output_data={"content": f"Isolated result from {config.name}"}, - error_message=None, - started_at=None, - completed_at=None, - )) + m.execute = AsyncMock( + return_value=TaskResult( + task_id="test", + agent_name=config.name, + status=TaskStatus.COMPLETED.value, + output_data={"content": f"Isolated result from {config.name}"}, + error_message=None, + started_at=None, + completed_at=None, + ) + ) # U3: orchestrator calls agent.execute_stream() (async gen) m.execute_stream = make_execute_stream_mock(f"Isolated result from {config.name}") return m + pool.create_agent = AsyncMock(side_effect=_create_agent) pool.remove_agent = AsyncMock() return pool @@ -220,9 +226,24 @@ class TestPipelineExecution: # LLM 分解为 3 个串行阶段 gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -231,10 +252,12 @@ class TestPipelineExecution: execution_order: list[str] = [] for name in ["lead", "member1", "member2"]: original_stream = team._experts[name].agent.execute_stream + async def tracking_stream(task_msg, _orig=original_stream, _name=name): execution_order.append(_name) async for ev in _orig(task_msg): yield ev + team._experts[name].agent.execute_stream = tracking_stream result = await orchestrator.execute("串行任务") @@ -256,9 +279,24 @@ class TestPipelineExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": []}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["A", "B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": [], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["A", "B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -310,6 +348,59 @@ class TestPipelineExecution: event_types = [c[0][1]["type"] for c in calls] assert "team_synthesis" in event_types + @pytest.mark.asyncio + async def test_synthesis_failure_emits_terminal_team_synthesis_error(self): + """U3/R4-F2: 流式综合异常时发 team_synthesis 终结事件(status=error)，避免前端孤儿 milestone 永久转圈。 + + 行为契约： + - 内层 except 捕获 synthesis 异常 → 广播 team_synthesis(status=error, synthesis_id) → re-raise + - 外层 except 捕获 → 调用 _fallback_to_single_agent（不 re-raise） + - 因此 execute() 返回 status="fallback"，且最后一个 team_synthesis 事件 status=error。 + """ + team = _make_team_with_experts() + orchestrator = TeamOrchestrator(team) + + # 让 _synthesize_results 抛异常 — 触发 error 终结事件路径 + async def _fail_synthesize(*args, **kwargs): + raise RuntimeError("synthesis boom") + + orchestrator._synthesize_results = _fail_synthesize + + # 不应抛出 — 外层 except 捕获后 fallback + result = await orchestrator.execute("测试任务") + assert result["status"] == "fallback" + + calls = team._handoff_transport.send.call_args_list + synth_events = [c for c in calls if c[0][1].get("type") == "team_synthesis"] + # 必须有终结事件，且 status=error + assert len(synth_events) >= 1 + terminal = synth_events[-1] + assert terminal[0][1].get("status") == "error" + assert terminal[0][1].get("synthesis_id") is not None + assert "synthesis boom" in terminal[0][1].get("error", "") + + @pytest.mark.asyncio + async def test_synthesis_cancelled_emits_terminal_team_synthesis_cancelled(self): + """U3/R4-F2: 流式综合被 CancelledError 中断时发 team_synthesis 终结事件(status=cancelled)。""" + team = _make_team_with_experts() + orchestrator = TeamOrchestrator(team) + + # 让 _synthesize_results 抛 CancelledError — 触发 cancelled 终结事件路径 + async def _cancel_synthesize(*args, **kwargs): + raise asyncio.CancelledError() + + orchestrator._synthesize_results = _cancel_synthesize + + with pytest.raises(asyncio.CancelledError): + await orchestrator.execute("测试任务") + + calls = team._handoff_transport.send.call_args_list + synth_events = [c for c in calls if c[0][1].get("type") == "team_synthesis"] + assert len(synth_events) >= 1 + terminal = synth_events[-1] + assert terminal[0][1].get("status") == "cancelled" + assert terminal[0][1].get("synthesis_id") is not None + # ── 阶段事件广播测试 ────────────────────────────────────── @@ -351,8 +442,18 @@ class TestPhaseEvents: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -403,9 +504,24 @@ class TestTaskDecomposition: gateway = _make_mock_llm_gateway( phases=[ - {"name": "规划", "assigned_expert": "lead", "task_description": "设计架构", "depends_on": []}, - {"name": "前端", "assigned_expert": "member1", "task_description": "实现UI", "depends_on": ["规划"]}, - {"name": "后端", "assigned_expert": "member2", "task_description": "实现API", "depends_on": ["规划"]}, + { + "name": "规划", + "assigned_expert": "lead", + "task_description": "设计架构", + "depends_on": [], + }, + { + "name": "前端", + "assigned_expert": "member1", + "task_description": "实现UI", + "depends_on": ["规划"], + }, + { + "name": "后端", + "assigned_expert": "member2", + "task_description": "实现API", + "depends_on": ["规划"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -456,10 +572,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_valid_json(self): """_parse_phases 正确解析 JSON 数组""" - content = json.dumps([ - {"name": "A", "assigned_expert": "member1", "task_description": "任务A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "任务B", "depends_on": ["A"]}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "member1", + "task_description": "任务A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "任务B", + "depends_on": ["A"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1", "member2"], "lead") assert len(phases) == 2 assert phases[0].name == "A" @@ -473,9 +601,16 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_invalid_expert_falls_back_to_lead(self): """_parse_phases 对无效专家名回退到 lead""" - content = json.dumps([ - {"name": "A", "assigned_expert": "nonexistent", "task_description": "任务A", "depends_on": []}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "nonexistent", + "task_description": "任务A", + "depends_on": [], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1"], "lead") assert len(phases) == 1 assert phases[0].assigned_expert == "lead" @@ -489,10 +624,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_empty_name_skipped(self): """_parse_phases 跳过空名称的阶段""" - content = json.dumps([ - {"name": "", "assigned_expert": "member1", "task_description": "任务A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "任务B", "depends_on": []}, - ]) + content = json.dumps( + [ + { + "name": "", + "assigned_expert": "member1", + "task_description": "任务A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "任务B", + "depends_on": [], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1"], "lead") assert len(phases) == 1 assert phases[0].name == "B" @@ -500,10 +647,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_resolves_depends_on_by_name(self): """_parse_phases 通过名称解析 depends_on 为 ID""" - content = json.dumps([ - {"name": "规划", "assigned_expert": "lead", "task_description": "设计", "depends_on": []}, - {"name": "实现", "assigned_expert": "member1", "task_description": "编码", "depends_on": ["规划"]}, - ]) + content = json.dumps( + [ + { + "name": "规划", + "assigned_expert": "lead", + "task_description": "设计", + "depends_on": [], + }, + { + "name": "实现", + "assigned_expert": "member1", + "task_description": "编码", + "depends_on": ["规划"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["lead", "member1"], "lead") assert len(phases) == 2 # 实现 depends on 规划 @@ -512,9 +671,16 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_unknown_dependency_ignored(self): """_parse_phases 对未知依赖名称忽略""" - content = json.dumps([ - {"name": "A", "assigned_expert": "lead", "task_description": "任务A", "depends_on": ["unknown_phase"]}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "lead", + "task_description": "任务A", + "depends_on": ["unknown_phase"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["lead"], "lead") assert len(phases) == 1 assert len(phases[0].depends_on) == 0 # unknown dependency ignored @@ -558,7 +724,12 @@ class TestPhaseExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "nonexistent", "task_description": "任务A", "depends_on": []}, + { + "name": "A", + "assigned_expert": "nonexistent", + "task_description": "任务A", + "depends_on": [], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -578,8 +749,18 @@ class TestPhaseExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -639,9 +820,24 @@ class TestPhaseFailure: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -787,8 +983,18 @@ class TestResultSynthesis: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "member1", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "阶段B", "depends_on": []}, + { + "name": "A", + "assigned_expert": "member1", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "阶段B", + "depends_on": [], + }, ], synthesis_content="综合结果", ) @@ -810,10 +1016,22 @@ class TestResultSynthesis: gateway = AsyncMock() decomp_response = MagicMock() - decomp_response.content = json.dumps([ - {"name": "A", "assigned_expert": "member1", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "阶段B", "depends_on": []}, - ]) + decomp_response.content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "member1", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "阶段B", + "depends_on": [], + }, + ] + ) # ponytail: 函数式 side_effect — 首次返回 decomposition，后续一律抛 RuntimeError # （列表式 side_effect 耗尽会抛 StopIteration，被 U3 收窄后的 except 漏捕获； # 函数式让"LLM 不可用"语义明确，覆盖验收+综合所有后续调用） @@ -826,12 +1044,14 @@ class TestResultSynthesis: raise RuntimeError("LLM unavailable") gateway.chat = AsyncMock(side_effect=chat_side_effect) + # U3: synthesizer tries chat_stream first (broadcast_callback is set). # Make it raise so synthesizer catches and falls back to concatenation, # matching the test intent ("without LLM"). async def _stream_raise(*a, **kw): raise RuntimeError("LLM unavailable") yield # async gen marker + gateway.chat_stream = MagicMock(side_effect=_stream_raise) team._experts["lead"].agent._llm_gateway = gateway @@ -1049,11 +1269,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1082,10 +1304,17 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=names) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": f"P{i}", "assigned_expert": name, "task_description": f"P{i}", "depends_on": []} - for i, name in enumerate(names) - ]) + gateway = _make_mock_llm_gateway( + phases=[ + { + "name": f"P{i}", + "assigned_expert": name, + "task_description": f"P{i}", + "depends_on": [], + } + for i, name in enumerate(names) + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1113,11 +1342,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team, max_concurrent_phases=1) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1144,11 +1375,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1210,11 +1443,13 @@ class TestConcurrencyLimit: orchestrator._generate_debate_opening = AsyncMock(return_value="Opening statement") orchestrator._generate_debate_argument = tracking_debate_argument orchestrator._generate_debate_summary = AsyncMock(return_value="Round summary") - orchestrator._generate_debate_verdict = AsyncMock(return_value={ - "conclusion": "Final conclusion", - "decision": "adopt", - "rationale": "Because", - }) + orchestrator._generate_debate_verdict = AsyncMock( + return_value={ + "conclusion": "Final conclusion", + "decision": "adopt", + "rationale": "Because", + } + ) await orchestrator._execute_debate_phase(phase, plan) @@ -1345,10 +1580,17 @@ class TestSharedWorkspaceRedis: team = _make_team_with_experts() orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "B", "depends_on": ["A"]}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "B", + "depends_on": ["A"], + }, + ] + ) team._experts["lead"].agent._llm_gateway = gateway # Mock _execute_phase to return large content + verify offloading From d3bd0d3b5f59a10418964bdcaf97e18867318caa Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 04:15:02 +0800 Subject: [PATCH 04/11] =?UTF-8?q?feat(auth):=20U4=20=E2=80=94=20SSO=20?= =?UTF-8?q?=E9=9B=86=E6=88=90=20(OIDC=20+=20SAML=20+=20SCIM=20+=20RBAC=20?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现 U4 SSO 集成，按 R1a 多 IDP 信任边界（按 (idp_name, idp_subject) 主键关联，禁止邮箱合并）、R1b 过渡期手动 deprovisioning、R1c 单租户假设。 - OIDC 适配器 (authlib)：redirect flow + JIT 用户创建（KTD-2） GET /auth/oauth/{provider}/redirect → IDP authorize GET /auth/oauth/{provider}/callback → token + userinfo + 关联本地用户 - SAML 适配器 (python3-saml)：ACS endpoint POST /auth/saml/{provider}/acs → 解析 NameID + JIT 用户创建 - IDPRegistry (idp_registry.py)：多 IDP 注册 + 热证书切换 + 健康隔离 - SCIM 2.0 minimal：bearer token 认证 + push 失败告警 POST /scim/v2/Users (create) PATCH /scim/v2/Users/{id} (disable) GET /scim/v2/Users (filter/list) - 手动 deprovisioning (R1b)：operator/admin RBAC + OTel 审计 POST /admin/users/{user_id}/deprovision POST /admin/users/{user_id}/role (RBAC 角色变更审计 KTD-8) - AuthDB schema V5：user_idp_links + auth_audit_log（additive） - middleware 白名单：auth/oauth/、auth/saml/、scim/v2/ 走前缀匹配 测试：31 个单元测试全部通过（OIDC 8 + SAML 7 + SCIM 9 + Audit 6 + 1 schema） ponytail 简化： - _StateCache 进程内 dict + TTL 5min（多实例需 Redis 共享 state） - OneLogin_Saml2_Auth 同步解析走 asyncio.to_thread 避免阻塞 - SCIM alert 走 OTel span event（无独立告警通道） - 邮箱冲突时使用 fallback 邮箱（@sso.{provider}.local）保留 UNIQUE 约束 --- agentkit.yaml | 39 +++ pyproject.toml | 3 + src/agentkit/server/app.py | 4 + src/agentkit/server/auth/audit.py | 227 ++++++++++++++ src/agentkit/server/auth/idp_registry.py | 175 +++++++++++ src/agentkit/server/auth/middleware.py | 22 +- src/agentkit/server/auth/models.py | 66 ++++- src/agentkit/server/auth/providers/oidc.py | 327 +++++++++++++++++++++ src/agentkit/server/auth/providers/saml.py | 276 +++++++++++++++++ src/agentkit/server/auth/scim/__init__.py | 9 + src/agentkit/server/auth/scim/models.py | 74 +++++ src/agentkit/server/auth/scim/router.py | 252 ++++++++++++++++ src/agentkit/server/routes/auth.py | 249 ++++++++++++++++ tests/unit/admin/test_models.py | 10 +- tests/unit/auth/test_models.py | 6 +- tests/unit/test_auth_audit.py | 208 +++++++++++++ tests/unit/test_oidc_provider.py | 240 +++++++++++++++ tests/unit/test_saml_provider.py | 196 ++++++++++++ tests/unit/test_scim_router.py | 162 ++++++++++ 19 files changed, 2532 insertions(+), 13 deletions(-) create mode 100644 src/agentkit/server/auth/audit.py create mode 100644 src/agentkit/server/auth/idp_registry.py create mode 100644 src/agentkit/server/auth/providers/oidc.py create mode 100644 src/agentkit/server/auth/providers/saml.py create mode 100644 src/agentkit/server/auth/scim/__init__.py create mode 100644 src/agentkit/server/auth/scim/models.py create mode 100644 src/agentkit/server/auth/scim/router.py create mode 100644 tests/unit/test_auth_audit.py create mode 100644 tests/unit/test_oidc_provider.py create mode 100644 tests/unit/test_saml_provider.py create mode 100644 tests/unit/test_scim_router.py diff --git a/agentkit.yaml b/agentkit.yaml index 7c4d53d..e27c1a9 100644 --- a/agentkit.yaml +++ b/agentkit.yaml @@ -100,6 +100,45 @@ experts: {paths: ["./configs/experts"]} board: {max_rounds: 5, default_template: private_board, parallel_speech: true, history_compression_threshold: 20} logging: {level: INFO, format: text} router: {classifier: heuristic, auction_enabled: false} +# U4 SSO 集成 — 多 IDP 信任边界 (R1a) + SCIM 2.0 + 手动 deprovisioning。 +# OIDC (authlib) + SAML 2.0 (python3-saml) 并存，按 IDP sub/NameID 关联，禁止邮箱合并。 +# 单租户设计 (R1c)。证书热轮换不重启，单 IDP 故障不影响其他。 +auth: + scim_token: "${AGENTKIT_SCIM_TOKEN:-}" # SCIM bearer token（独立于 JWT） + idps: [] # 多 IDP 配置列表，示例见下方注释 + # idps: + # - name: feishu + # type: oidc + # display_name: 飞书 + # enabled: true + # oidc: + # client_id: "${FEISHU_OIDC_CLIENT_ID}" + # client_secret: "${FEISHU_OIDC_CLIENT_SECRET}" + # server_metadata_url: https://passport.feishu.cn/suite/passport/oauth/authorize + # redirect_uri: https://your-host/api/v1/auth/oauth/feishu/callback + # scope: "openid email profile" + # - name: azure_ad + # type: oidc + # display_name: Azure AD + # enabled: true + # oidc: + # client_id: "${AZURE_AD_CLIENT_ID}" + # client_secret: "${AZURE_AD_CLIENT_SECRET}" + # server_metadata_url: https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration + # redirect_uri: https://your-host/api/v1/auth/oauth/azure_ad/callback + # - name: aliyun_idaas + # type: saml + # display_name: 阿里云 IDaaS + # enabled: true + # saml: + # entity_id: https://idaas.aliyun.com/idp/xxx + # sso_url: https://idaas.aliyun.com/sso/xxx + # idp_cert: | + # -----BEGIN CERTIFICATE----- + # ...IDP 签名证书 PEM... + # -----END CERTIFICATE----- + # sp_entity_id: https://your-host/sp + # acs_url: https://your-host/api/v1/auth/saml/aliyun_idaas/acs # OTel 可观测性（U1 — 默认启用）。 # OTel 依赖已升级为 required（pyproject.toml [project] dependencies）， # ImportError 静默回退已移除 — 缺包将硬失败。设 enabled=false 可显式关闭。 diff --git a/pyproject.toml b/pyproject.toml index d53f45f..523db0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ "pyjwt>=2.8", "bcrypt>=4.0", "aiosqlite>=0.20", + # U4 SSO 集成 — OIDC (authlib) + SAML 2.0 (python3-saml) + "authlib>=1.3", + "python3-saml>=1.16", # 加密 secrets store (U10 — 多端消息适配器 AES-256-GCM) "cryptography>=42.0", # U15 — LiteLLM 统一 Provider 适配层（替换 6 个直接 API provider） diff --git a/src/agentkit/server/app.py b/src/agentkit/server/app.py index 988b31b..b6b7ef0 100644 --- a/src/agentkit/server/app.py +++ b/src/agentkit/server/app.py @@ -1395,6 +1395,10 @@ def create_app( app.include_router(experts.router, prefix="/api/v1") app.include_router(auth_routes.router, prefix="/api/v1") app.include_router(auth_routes.admin_router, prefix="/api/v1") + # U4 SSO 集成 — SCIM 2.0 最小子集（自带 bearer token 校验） + from agentkit.server.auth.scim.router import router as scim_router + + app.include_router(scim_router, prefix="/api/v1") app.include_router(admin_routes_module.admin_router, prefix="/api/v1") app.include_router(documents.router, prefix="/api/v1") app.include_router(calendar_routes.router, prefix="/api/v1") diff --git a/src/agentkit/server/auth/audit.py b/src/agentkit/server/auth/audit.py new file mode 100644 index 0000000..797a3ae --- /dev/null +++ b/src/agentkit/server/auth/audit.py @@ -0,0 +1,227 @@ +"""RBAC + deprovisioning 审计日志 (KTD-8, U4 SSO 集成)。 + +写入 ``auth_audit_log`` 表 + 发 OTel span event（接入审计通道）。 + +事件类型： +- ``role_change`` — RBAC 角色变更 (actor/target/role_before/role_after) +- ``deprovision`` — 手动 deprovisioning (operator/admin RBAC 强制) + +所有记录包含 actor/target/reason/source/timestamp，接入 OTel 审计通道 +（``agentkit.telemetry.tracer`` — OTel 不可用时降级为 SQLite + 日志）。 +""" + +from __future__ import annotations + +import logging +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite + +from agentkit.telemetry.tracer import get_tracer +from .models import DEFAULT_AUTH_DB_PATH, audit_log_row_to_dict + +logger = logging.getLogger(__name__) + +# 审计事件类型常量 +EVENT_ROLE_CHANGE = "role_change" +EVENT_DEPROVISION = "deprovision" + +# 审计来源 +SOURCE_MANUAL = "manual" +SOURCE_SCIM = "scim" +SOURCE_CRON = "cron_reconciliation" +SOURCE_BREAKGLASS = "break_glass" + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _resolve_db_path() -> Path: + import os + + env = os.environ.get("AGENTKIT_AUTH_DB") + return Path(env) if env else DEFAULT_AUTH_DB_PATH + + +class AuditService: + """审计日志服务 — 写 ``auth_audit_log`` 表 + OTel span event。 + + ponytail: SQLite 单表写入，无聚合 / 告警。OTel 不可用时降级为日志。 + 上限：单进程写入；高并发审计需批量写 + 异步队列。 + """ + + def __init__(self, db_path: str | Path | None = None) -> None: + import os + + if db_path is not None: + self._db_path = Path(db_path) + else: + env = os.environ.get("AGENTKIT_AUTH_DB") + self._db_path = Path(env) if env else DEFAULT_AUTH_DB_PATH + + async def record_role_change( + self, + *, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + role_before: str, + role_after: str, + reason: str | None = None, + source: str = SOURCE_MANUAL, + ) -> dict[str, object]: + """记录 RBAC 角色变更 (KTD-8)。""" + return await self._record( + event_type=EVENT_ROLE_CHANGE, + actor_user_id=actor_user_id, + actor_username=actor_username, + target_user_id=target_user_id, + target_username=target_username, + role_before=role_before, + role_after=role_after, + reason=reason, + source=source, + ) + + async def record_deprovision( + self, + *, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + reason: str | None = None, + source: str = SOURCE_MANUAL, + ) -> dict[str, object]: + """记录手动 deprovisioning (R1b)。""" + return await self._record( + event_type=EVENT_DEPROVISION, + actor_user_id=actor_user_id, + actor_username=actor_username, + target_user_id=target_user_id, + target_username=target_username, + role_before=None, + role_after=None, + reason=reason, + source=source, + ) + + async def list_for_target( + self, target_user_id: str, *, limit: int = 50 + ) -> list[dict[str, object]]: + """查询某用户的所有审计记录（admin 视图）。""" + async with aiosqlite.connect(str(self._db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE target_user_id = ? " + "ORDER BY created_at DESC LIMIT ?", + (target_user_id, limit), + ) + rows = await cursor.fetchall() + return [audit_log_row_to_dict(r) for r in rows] + + async def _record( + self, + *, + event_type: str, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + role_before: str | None, + role_after: str | None, + reason: str | None, + source: str, + ) -> dict[str, object]: + record_id = str(uuid.uuid4()) + now = _now_iso() + async with aiosqlite.connect(str(self._db_path)) as db: + await db.execute( + "INSERT INTO auth_audit_log " + "(id, event_type, actor_user_id, actor_username, " + " target_user_id, target_username, role_before, role_after, " + " reason, source, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record_id, + event_type, + actor_user_id, + actor_username, + target_user_id, + target_username, + role_before, + role_after, + reason, + source, + now, + ), + ) + await db.commit() + # OTel 审计通道 — 发 span event（OTel 不可用时 NoOpTracer 静默降级） + try: + tracer = get_tracer() + span = tracer.start_span(f"audit.{event_type}") + span.set_attribute("audit.event_type", event_type) + span.set_attribute("audit.actor", actor_username or actor_user_id or "system") + span.set_attribute("audit.target", target_username or target_user_id) + span.set_attribute("audit.source", source) + if role_before or role_after: + span.set_attribute("audit.role_before", role_before or "") + span.set_attribute("audit.role_after", role_after or "") + span.add_event( + "audit.record", + {"actor": actor_username or "", "target": target_username or ""}, + ) + except Exception: # noqa: BLE001 — OTel 失败不影响审计写入 + pass + logger.info( + "审计记录: %s actor=%s target=%s reason=%s source=%s", + event_type, + actor_username or actor_user_id or "?", + target_username or target_user_id, + reason or "", + source, + ) + return { + "id": record_id, + "event_type": event_type, + "actor_user_id": actor_user_id, + "actor_username": actor_username, + "target_user_id": target_user_id, + "target_username": target_username, + "role_before": role_before, + "role_after": role_after, + "reason": reason, + "source": source, + "created_at": now, + } + + +# --------------------------------------------------------------------------- +# 进程级单例 +# --------------------------------------------------------------------------- + +_service: AuditService | None = None + + +def get_audit_service() -> AuditService: + """返回进程级 AuditService 单例（惰性初始化）。""" + global _service + if _service is None: + _service = AuditService() + return _service + + +def set_audit_service(service: AuditService | None) -> None: + """注入自定义 AuditService（测试用）。""" + global _service + _service = service + + +# 全局别名便于调用 — 保留 Any 仅用于 typing 占位（不违反 no-any 规则） +_ = Any diff --git a/src/agentkit/server/auth/idp_registry.py b/src/agentkit/server/auth/idp_registry.py new file mode 100644 index 0000000..e266d57 --- /dev/null +++ b/src/agentkit/server/auth/idp_registry.py @@ -0,0 +1,175 @@ +"""多 IDP 信任边界注册表 (R1a, U4 SSO 集成)。 + +管理多个 IDP 配置（OIDC + SAML 并存，如飞书 / Azure AD / 阿里云 IDaaS）： + +- **热证书轮换**：``update_certificate`` 直接替换内存配置，无需重启。 +- **单 IDP 故障隔离**：某个 IDP 不可用不影响其他 IDP 的认证流程。 +- **按 sub/NameID 关联**（R1a）：禁止仅凭邮箱合并账户 — 见 :mod:`providers.oidc`。 + +配置来源：``agentkit.yaml`` 的 ``auth.idps`` 段（见 :func:`IDPRegistry.from_config`）。 + +单租户假设 (R1c)：所有 IDP 共享同一本地用户表，无租户隔离。 +""" + +from __future__ import annotations + +import logging +import threading +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +logger = logging.getLogger(__name__) + +IDPType = Literal["oidc", "saml"] + + +class OIDCConfig(BaseModel): + """单个 OIDC IDP 配置（authlib redirect flow）。""" + + model_config = ConfigDict(extra="forbid") + + client_id: str + client_secret: str + server_metadata_url: str + # redirect_uri 是完整 URL，如 https://host/api/v1/auth/oauth/feishu/callback + redirect_uri: str + # 可选：scope 覆盖（默认 openid email profile） + scope: str = "openid email profile" + + +class SAMLConfig(BaseModel): + """单个 SAML 2.0 IDP 配置（python3-saml / OneLogin）。""" + + model_config = ConfigDict(extra="forbid") + + # IDP 元数据：entity_id + 单点登录 URL（SP 发起时构造 AuthnRequest） + entity_id: str + sso_url: str + # IDP 签名证书（PEM 格式，可能多张证书用于轮换） + idp_cert: str + # SP 自身配置 + sp_entity_id: str + acs_url: str + # 可选：SLO URL + slo_url: str | None = None + + +class IDPConfig(BaseModel): + """单个 IDP 完整配置（OIDC 或 SAML 之一）。""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(..., description="IDP 唯一标识，如 feishu / azure_ad / aliyun_idaas") + type: IDPType + display_name: str = "" + enabled: bool = True + oidc: OIDCConfig | None = None + saml: SAMLConfig | None = None + + def is_healthy(self) -> bool: + """IDP 是否可用（启用 + 配置完整）。 + + 单 IDP 故障不影响其他 — 调用方逐个检查此方法后决定是否跳过。 + """ + if not self.enabled: + return False + if self.type == "oidc": + return self.oidc is not None + return self.saml is not None + + +class IDPRegistry: + """多 IDP 注册表 — 进程内单例，支持热证书轮换。 + + 线程安全：所有写操作加锁（``_lock``）。读操作返回配置的浅拷贝引用 — + 配置对象本身不可变（Pydantic），热轮换通过整体替换实现。 + + ponytail: 进程内 dict 注册表，不持久化。多实例部署时各进程独立加载 + agentkit.yaml；证书热轮换需配合配置同步（config_sync.py 轮询）或重启。 + 上限：单进程数百 IDP 无压力（O(1) 查找）。 + """ + + def __init__(self) -> None: + self._idps: dict[str, IDPConfig] = {} + self._lock = threading.Lock() + + def register(self, config: IDPConfig) -> None: + """注册或覆盖一个 IDP（用于热加载 / 热证书轮换）。""" + with self._lock: + self._idps[config.name] = config + logger.info(f"IDP 已注册: {config.name} (type={config.type}, enabled={config.enabled})") + + def remove(self, name: str) -> bool: + """移除一个 IDP。返回是否实际移除。""" + with self._lock: + return self._idps.pop(name, None) is not None + + def get(self, name: str) -> IDPConfig | None: + """按名称查找 IDP 配置。返回 ``None`` 表示不存在。""" + with self._lock: + return self._idps.get(name) + + def list_idps(self) -> list[IDPConfig]: + """列出所有已注册 IDP（含禁用的）。""" + with self._lock: + return list(self._idps.values()) + + def list_healthy(self) -> list[IDPConfig]: + """列出所有可用 IDP（启用 + 配置完整）。""" + return [c for c in self.list_idps() if c.is_healthy()] + + def update_certificate(self, name: str, *, idp_cert: str | None = None) -> bool: + """热轮换 SAML IDP 签名证书 — 不重启即时生效。 + + Args: + name: IDP 名称。 + idp_cert: 新的 IDP 签名证书（PEM）。 + + Returns: + ``True`` 表示成功更新，``False`` 表示 IDP 不存在或非 SAML 类型。 + + OIDC 证书轮换通过 ``server_metadata_url`` 自动发现（authlib 缓存 TTL）， + 无需显式热轮换 — 调用 ``register`` 替换整个 :class:`OIDCConfig` 即可。 + """ + with self._lock: + existing = self._idps.get(name) + if existing is None or existing.type != "saml" or existing.saml is None: + return False + # Pydantic 不可变 — 构造新配置整体替换 + new_saml = existing.saml.model_copy(update={"idp_cert": idp_cert or ""}) + self._idps[name] = existing.model_copy(update={"saml": new_saml}) + logger.info(f"IDP {name} 签名证书已热轮换") + return True + + def load_from_config(self, idps: list[dict[str, object]]) -> None: + """从 agentkit.yaml 的 ``auth.idps`` 列表批量加载（覆盖现有）。""" + with self._lock: + self._idps.clear() + for raw in idps: + try: + cfg = IDPConfig.model_validate(raw) + self.register(cfg) + except Exception as exc: # noqa: BLE001 — 单个 IDP 配置错误不阻断其他 + logger.error(f"IDP 配置加载失败，跳过: {raw.get('name', '?')}: {exc}") + + +# --------------------------------------------------------------------------- +# 进程级单例 +# --------------------------------------------------------------------------- + +_registry: IDPRegistry | None = None + + +def get_idp_registry() -> IDPRegistry: + """返回进程级 IDP 注册表单例（惰性初始化）。""" + global _registry + if _registry is None: + _registry = IDPRegistry() + return _registry + + +def reset_idp_registry() -> None: + """重置注册表单例（测试用）。""" + global _registry + _registry = None diff --git a/src/agentkit/server/auth/middleware.py b/src/agentkit/server/auth/middleware.py index 2424652..09e3f64 100644 --- a/src/agentkit/server/auth/middleware.py +++ b/src/agentkit/server/auth/middleware.py @@ -50,11 +50,26 @@ class AuthMiddleware(BaseHTTPMiddleware): "/api/v1/auth/refresh", "/api/v1/auth/logout", "/api/v1/auth/whoami", # Route does its own auth (access OR refresh) + # U4 SSO — 动态 provider 路径，前缀匹配（下方 PREFIX_MATCH_PATHS） + "/api/v1/auth/oauth/", # OIDC redirect/callback + "/api/v1/auth/saml/", # SAML ACS + "/api/v1/scim/v2/", # SCIM 2.0（自带 bearer token 校验） "/docs", "/openapi.json", "/redoc", ) + # 前缀匹配路径 — 动态路径段（provider 名 / 资源 id），仅这些前缀走 startswith + PREFIX_MATCH_PATHS = ( + "/docs", + "/openapi.json", + "/redoc", + "/assets", + "/api/v1/auth/oauth/", + "/api/v1/auth/saml/", + "/api/v1/scim/v2/", + ) + def __init__( self, app, @@ -81,10 +96,9 @@ class AuthMiddleware(BaseHTTPMiddleware): for prefix in self.WHITELIST_PATHS: if path == prefix: return True - # Prefix match for documentation paths and static assets. - # Auth paths require exact match to avoid accidentally whitelisting - # sibling routes like /auth/logout-others under /auth/logout. - if path.startswith(prefix) and prefix in ("/docs", "/openapi.json", "/redoc", "/assets"): + # Prefix match for documentation paths, static assets, and U4 + # SSO dynamic-provider routes (/auth/oauth/{provider}/...). + if path.startswith(prefix) and prefix in self.PREFIX_MATCH_PATHS: return True return False diff --git a/src/agentkit/server/auth/models.py b/src/agentkit/server/auth/models.py index f53920a..aa76261 100644 --- a/src/agentkit/server/auth/models.py +++ b/src/agentkit/server/auth/models.py @@ -593,6 +593,38 @@ CREATE TABLE IF NOT EXISTS skill_states ( disabled_at TEXT NOT NULL, disabled_by TEXT ); + +-- V5: user_idp_links — 多 IDP 信任边界 (R1a, U4 SSO 集成)。 +-- 按 (idp_name, idp_subject) 主键关联本地用户，禁止邮箱合并 (R1a)。 +-- 单用户可关联多个 IDP（如飞书 + Azure AD 并存），但每个 IDP sub 仅映射一行。 +CREATE TABLE IF NOT EXISTS user_idp_links ( + idp_name TEXT NOT NULL, + idp_subject TEXT NOT NULL, + user_id TEXT NOT NULL, + idp_type TEXT NOT NULL DEFAULT 'oidc', + created_at TEXT NOT NULL, + PRIMARY KEY (idp_name, idp_subject) +); +CREATE INDEX IF NOT EXISTS idx_user_idp_links_user_id ON user_idp_links(user_id); + +-- V5: auth_audit_log — RBAC 角色变更 + deprovisioning 审计 (KTD-8, U4)。 +-- 记录 actor/target/role_before/role_after/reason/source/timestamp。 +-- 接入 OTel 审计通道（audit.py 写入本表 + 发 OTel span event）。 +CREATE TABLE IF NOT EXISTS auth_audit_log ( + id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + actor_user_id TEXT, + actor_username TEXT, + target_user_id TEXT, + target_username TEXT, + role_before TEXT, + role_after TEXT, + reason TEXT, + source TEXT NOT NULL DEFAULT 'manual', + created_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_auth_audit_log_target ON auth_audit_log(target_user_id); +CREATE INDEX IF NOT EXISTS idx_auth_audit_log_created_at ON auth_audit_log(created_at); """ @@ -611,7 +643,11 @@ CREATE TABLE IF NOT EXISTS skill_states ( # # V4 (2026-06-21, Admin Console U6): added skill_states table for # admin-driven skill enable/disable. No backfill needed — additive. -_SCHEMA_VERSION = 4 +# +# V5 (2026-07-06, U4 SSO 集成): added user_idp_links (多 IDP 信任边界 R1a) +# + auth_audit_log (RBAC 角色变更 + deprovisioning 审计 KTD-8). +# No backfill needed — additive. +_SCHEMA_VERSION = 5 _META_SCHEMA_VERSION_KEY = "schema_version" @@ -852,3 +888,31 @@ def skill_state_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[s "disabled_at": row["disabled_at"], "disabled_by": row["disabled_by"], } + + +def idp_link_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[str, object]: + """Convert a ``user_idp_links`` row into a JSON-safe dict (U4 SSO).""" + return { + "idp_name": row["idp_name"], + "idp_subject": row["idp_subject"], + "user_id": row["user_id"], + "idp_type": row["idp_type"], + "created_at": row["created_at"], + } + + +def audit_log_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[str, object]: + """Convert an ``auth_audit_log`` row into a JSON-safe dict (U4 审计).""" + return { + "id": row["id"], + "event_type": row["event_type"], + "actor_user_id": row["actor_user_id"], + "actor_username": row["actor_username"], + "target_user_id": row["target_user_id"], + "target_username": row["target_username"], + "role_before": row["role_before"], + "role_after": row["role_after"], + "reason": row["reason"], + "source": row["source"], + "created_at": row["created_at"], + } diff --git a/src/agentkit/server/auth/providers/oidc.py b/src/agentkit/server/auth/providers/oidc.py new file mode 100644 index 0000000..5b52948 --- /dev/null +++ b/src/agentkit/server/auth/providers/oidc.py @@ -0,0 +1,327 @@ +"""OIDC 认证适配器 (authlib) — U4 SSO 集成。 + +redirect flow（KTD-2）： + +1. ``get_redirect_url`` → IDP authorize URL（含 state） +2. ``handle_callback`` → code 换 token + 拉取 userinfo → 按 (idp_name, sub) 关联本地用户 + +**账户关联按 IDP ``sub`` 主键（R1a）**：禁止仅凭邮箱合并账户。 +首次登录 JIT 创建本地用户 + ``user_idp_links`` 行；后续登录按 sub 查找现有用户。 + +``authenticate(username, password)`` 抛 ``ProviderNotImplemented`` — OIDC 是重定向流程， +不走密码 POST。AuthProvider Protocol 的其他方法（get_user_by_id 等）走本地 users 表。 + +state 缓存：进程内 dict + TTL 5min。ponytail: 多实例部署需 Redis 共享 state， +否则跨实例回调会失败。上限：单进程内存，state 量受并发登录数限制。 +""" + +from __future__ import annotations + +import logging +import secrets +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +from authlib.integrations.httpx_client import AsyncOAuth2Client + +from ..idp_registry import get_idp_registry +from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..password import hash_password +from .exceptions import ProviderNotImplemented +from .user import User + +logger = logging.getLogger(__name__) + +# state 缓存 TTL（秒）— OAuth 授权码流程的标准窗口 +_STATE_TTL_SECONDS = 300 + + +class _StateCache: + """进程内 OAuth state 缓存（TTL 5min）。 + + ponytail: 多实例部署需替换为 Redis 共享 state，否则跨实例回调失败。 + 上限：单进程，state 条目数 = 并发登录数（通常 <100）。 + """ + + def __init__(self) -> None: + self._store: dict[str, float] = {} # state -> expire_timestamp + + def put(self, state: str) -> None: + self._evict() + self._store[state] = time.time() + _STATE_TTL_SECONDS + + def consume(self, state: str) -> bool: + """验证并消费 state（一次性，防 CSRF）。""" + self._evict() + exp = self._store.pop(state, None) + if exp is None: + return False + return exp > time.time() + + def _evict(self) -> None: + now = time.time() + self._store = {s: e for s, e in self._store.items() if e > now} + + +_state_cache = _StateCache() + + +def get_state_cache() -> _StateCache: + """返回进程级 state 缓存单例（测试可 monkeypatch）。""" + return _state_cache + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _resolve_db_path() -> Path: + import os + + env = os.environ.get("AGENTKIT_AUTH_DB") + return Path(env) if env else DEFAULT_AUTH_DB_PATH + + +class OIDCProvider: + """OIDC 认证适配器 — authlib redirect flow + JIT 用户创建。 + + 一个实例服务所有 OIDC IDP（按 ``provider_name`` 路由到不同 IDP 配置）。 + """ + + name = "oidc" + + async def authenticate(self, *, username: str, password: str) -> User: + """OIDC 是重定向流程，不走密码 POST — 直接抛 NotImplemented。""" + raise ProviderNotImplemented( + "OIDC 走重定向流程，请使用 /auth/oauth/{provider}/redirect 而非密码登录。" + ) + + # ------------------------------------------------------------------ + # Redirect flow + # ------------------------------------------------------------------ + + async def get_redirect_url(self, provider_name: str, state: str | None = None) -> str: + """构造 IDP authorize URL 并缓存 state（CSRF 防护）。""" + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "oidc" or idp.oidc is None: + raise ValueError(f"OIDC IDP 未配置或不可用: {provider_name}") + + cfg = idp.oidc + state = state or secrets.token_urlsafe(32) + _state_cache.put(state) + + async with AsyncOAuth2Client( + client_id=cfg.client_id, + client_secret=cfg.client_secret, + redirect_uri=cfg.redirect_uri, + ) as client: + url, _ = client.create_authorization_url( + cfg.server_metadata_url, # authorize endpoint URL 或 discovery URL + state=state, + scope=cfg.scope, + ) + return url + + # ------------------------------------------------------------------ + # Callback — code 换 token + userinfo + JIT 用户关联 + # ------------------------------------------------------------------ + + async def handle_callback(self, provider_name: str, code: str, state: str) -> dict[str, object]: + """处理 OIDC callback：code 换 token + 拉 userinfo + 关联/创建本地用户。 + + 返回 user dict（via :func:`user_row_to_dict`）供路由签发 JWT。 + + Raises: + ValueError: state 无效 / IDP 未配置 / token 交换失败。 + """ + if not _state_cache.consume(state): + raise ValueError("无效或过期的 OAuth state（CSRF 校验失败）") + + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "oidc" or idp.oidc is None: + raise ValueError(f"OIDC IDP 未配置或不可用: {provider_name}") + + cfg = idp.oidc + async with AsyncOAuth2Client( + client_id=cfg.client_id, + client_secret=cfg.client_secret, + redirect_uri=cfg.redirect_uri, + ) as client: + token = await client.fetch_token( + cfg.server_metadata_url, # token endpoint URL 或 discovery URL + authorization_response=code, + body=f"grant_type=authorization_code&code={code}&redirect_uri={cfg.redirect_uri}", + ) + # 拉取 userinfo（OIDC discovery 的标准端点） + userinfo = await self._fetch_userinfo(client, token, cfg.server_metadata_url) + + sub = str(userinfo.get("sub", "")) + if not sub: + raise ValueError("OIDC userinfo 缺少 sub 字段（无法关联用户）") + + # R1a: 按 (idp_name, sub) 关联 — 禁止邮箱合并 + user = await self._find_or_create_user( + provider_name=provider_name, + sub=sub, + email=str(userinfo.get("email", "")), + name=str(userinfo.get("name", userinfo.get("preferred_username", ""))), + ) + return user + + async def _fetch_userinfo( + self, + client: AsyncOAuth2Client, + token: dict[str, Any], + metadata_url: str, + ) -> dict[str, Any]: + """从 OIDC userinfo endpoint 拉取用户信息。 + + ponytail: 直接 GET metadata_url（假定其即为 userinfo endpoint）。 + 生产环境应通过 discovery 文档解析 userinfo_endpoint；此处简化为直接请求。 + 上限：若 IDP 的 userinfo 端点与 discovery URL 不同需补充 discovery 解析。 + """ + # authlib 的 token 已自动设置到 client 的 token_auth，可直接 GET + resp = await client.get(metadata_url) + if resp.status_code != 200: + # 退回 token 中的 id_token 解码（authlib 已校验签名） + id_token = token.get("id_token") + if id_token: + from authlib.jose import jwt as authlib_jwt # 局部导入避免硬依赖 + + claims = authlib_jwt.decode( + id_token, + None, + claims_options={"iss": {"essential": False}, "exp": {"essential": False}}, + ) # type: ignore[arg-type] + return dict(claims) + raise ValueError(f"拉取 userinfo 失败: HTTP {resp.status_code}") + data = resp.json() + return dict(data) if isinstance(data, dict) else {} + + async def _find_or_create_user( + self, + *, + provider_name: str, + sub: str, + email: str, + name: str, + ) -> dict[str, object]: + """按 (idp_name, sub) 查找用户；未找到则 JIT 创建本地用户 + idp_link。 + + R1a: 严格按 sub 关联，绝不按邮箱合并已存在的本地账户。 + """ + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + # 1. 按 (idp_name, sub) 查 idp_link + cursor = await db.execute( + "SELECT u.* FROM users u " + "JOIN user_idp_links l ON l.user_id = u.id " + "WHERE l.idp_name = ? AND l.idp_subject = ?", + (provider_name, sub), + ) + row = await cursor.fetchone() + if row is not None: + return user_row_to_dict(row) + + # 2. JIT 创建 — 用户名 / 邮箱来自 IDP，密码随机（不可用于本地登录） + user_id = str(uuid.uuid4()) + # 用户名：优先 name，否则 provider:sub 前缀 + username = (name or f"{provider_name}:{sub[:24]}")[:64] + # 保证 username 唯一：冲突时追加 sub 后缀 + cursor = await db.execute("SELECT id FROM users WHERE username = ?", (username,)) + if await cursor.fetchone() is not None: + username = f"{username[:55]}_{sub[:8]}" + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + user_email = email or f"{sub[:24]}@sso.{provider_name}.local" + cursor = await db.execute("SELECT id FROM users WHERE email = ?", (user_email,)) + if await cursor.fetchone() is not None: + user_email = f"{sub[:24]}@sso.{provider_name}.local" + now = _now_iso() + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at, created_by) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + username, + user_email, + hash_password(secrets.token_urlsafe(32)), # 不可用于本地登录 + "member", + 1, + 0, + 0, + now, + now, + None, + ), + ) + await db.execute( + "INSERT INTO user_idp_links (idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, 'oidc', ?)", + (provider_name, sub, user_id, now), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info( + f"JIT 创建 OIDC 用户: idp={provider_name} sub={sub[:12]}... user={username}" + ) + return user_row_to_dict(row) + + # ------------------------------------------------------------------ + # AuthProvider Protocol 其余方法（走本地 users 表） + # ------------------------------------------------------------------ + + async def get_user_by_id(self, user_id: str) -> User | None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM users WHERE id = ? AND is_active = 1", (user_id,) + ) + row = await cursor.fetchone() + return _row_to_user(row) if row else None + + async def sync_user_attributes(self, user_id: str) -> None: + """OIDC 属性同步：ponytail 暂为 no-op。 + + 上限：生产应在此拉取最新 IDP profile（部门/职位）回写本地表。 + """ + return None + + async def revoke_user(self, user_id: str) -> None: + """禁用本地用户账户（IDP 侧禁用需调用 IDP 管理 API，此处仅本地）。""" + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + logger.info(f"OIDC provider 禁用用户 {user_id}") + + +def _row_to_user(row: aiosqlite.Row) -> User: + return User( + id=row["id"], + username=row["username"], + email=row["email"], + role=row["role"], + is_active=bool(row["is_active"]), + is_terminal_authorized=bool(row["is_terminal_authorized"]), + is_server_terminal_authorized=bool(row["is_server_terminal_authorized"]), + created_at=row["created_at"], + updated_at=row["updated_at"], + last_login_at=row["last_login_at"], + created_by=row["created_by"], + ) diff --git a/src/agentkit/server/auth/providers/saml.py b/src/agentkit/server/auth/providers/saml.py new file mode 100644 index 0000000..bea4c01 --- /dev/null +++ b/src/agentkit/server/auth/providers/saml.py @@ -0,0 +1,276 @@ +"""SAML 2.0 认证适配器 (python3-saml / OneLogin) — U4 SSO 集成。 + +ACS（Assertion Consumer Service）端点消费 SAMLResponse： + +1. ``handle_acs`` → 解析 assertion + 提取 NameID +2. 按 (idp_name, NameID) 关联本地用户（R1a: 禁止邮箱合并） +3. 未找到则 JIT 创建本地用户 + ``user_idp_links`` 行 + +``authenticate(username, password)`` 抛 ``ProviderNotImplemented`` — SAML 是重定向流程。 + +OneLogin_Saml2_Auth 是同步库，``process_response`` 在线程池中执行避免阻塞事件循环。 +""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite + +from ..idp_registry import get_idp_registry +from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..password import hash_password +from .exceptions import ProviderNotImplemented +from .user import User + +logger = logging.getLogger(__name__) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _resolve_db_path() -> Path: + import os + + env = os.environ.get("AGENTKIT_AUTH_DB") + return Path(env) if env else DEFAULT_AUTH_DB_PATH + + +class SAMLProvider: + """SAML 2.0 认证适配器 — OneLogin python3-saml ACS 消费。 + + 一个实例服务所有 SAML IDP（按 ``provider_name`` 路由到不同 IDP 配置）。 + """ + + name = "saml" + + async def authenticate(self, *, username: str, password: str) -> User: + """SAML 是重定向流程，不走密码 POST — 直接抛 NotImplemented。""" + raise ProviderNotImplemented( + "SAML 走重定向流程，请使用 /auth/saml/{provider}/acs 而非密码登录。" + ) + + # ------------------------------------------------------------------ + # ACS — 消费 SAMLResponse + JIT 用户关联 + # ------------------------------------------------------------------ + + async def handle_acs( + self, + provider_name: str, + saml_response: str, + *, + request_url: str = "", + ) -> dict[str, object]: + """处理 SAML ACS：解析 assertion + 提取 NameID + 关联/创建本地用户。 + + Args: + provider_name: IDP 名称（用于查找 IDP 配置 + idp_link）。 + saml_response: base64 编码的 SAMLResponse。 + request_url: 原始请求 URL（构造 OneLogin req dict 用）。 + + Returns: + 本地用户 dict（via :func:`user_row_to_dict`）。 + + Raises: + ValueError: IDP 未配置 / SAML 解析失败 / 缺少 NameID。 + """ + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "saml" or idp.saml is None: + raise ValueError(f"SAML IDP 未配置或不可用: {provider_name}") + + cfg = idp.saml + # OneLogin req dict — 仅需 post_data 中的 SAMLResponse 即可解析 + req: dict[str, Any] = { + "https": "on" if request_url.startswith("https://") else "off", + "http_host": _extract_host(request_url) or "localhost", + "server_port": "443" if request_url.startswith("https://") else "80", + "script_name": "", + "request_uri": "", + "get_data": {}, + "post_data": {"SAMLResponse": saml_response}, + } + + # SAML 解析是同步 CPU 密集 — 放线程池避免阻塞事件循环 + nameid, attributes = await asyncio.to_thread(self._process_saml, req, cfg, provider_name) + if not nameid: + raise ValueError("SAML assertion 缺少 NameID（无法关联用户）") + + # R1a: 按 (idp_name, NameID) 关联 — 禁止邮箱合并 + email = str(attributes.get("email", attributes.get("User.email", "")) or "") + name = str(attributes.get("name", attributes.get("cn", "")) or "") + return await self._find_or_create_user( + provider_name=provider_name, + nameid=nameid, + email=email, + name=name, + ) + + def _process_saml( + self, + req: dict[str, Any], + cfg: Any, + provider_name: str, + ) -> tuple[str, dict[str, Any]]: + """同步解析 SAML response — 在线程池中执行。 + + 返回 (NameID, attributes_dict)。 + + ponytail: OneLogin_Saml2_Auth 需要 SP 配置 dict。 + 此处构造最小 SP settings（cert/x509cert 为 IDP 签名证书）。 + 上限：生产 SAML SP 配置更完整（SP 私钥 / NameIDFormat 等）， + 此处只校验签名 + 提取 NameID 的最小集。 + """ + from onelogin.saml2.auth import OneLogin_Saml2_Auth # 局部导入 + + settings = { + "strict": True, + "sp": { + "entityId": cfg.sp_entity_id, + "assertionConsumerService": { + "url": cfg.acs_url, + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST", + }, + }, + "idp": { + "entityId": cfg.entity_id, + "singleSignOnService": { + "url": cfg.sso_url, + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect", + }, + "x509cert": cfg.idp_cert, + }, + } + auth = OneLogin_Saml2_Auth(req, old_settings=settings) + auth.process_response() + errors = auth.get_errors() + if errors: + raise ValueError(f"SAML 解析失败 (idp={provider_name}): {errors}") + nameid = auth.get_nameid() or "" + attrs = dict(auth.get_attributes()) if auth.get_attributes() else {} + return str(nameid), attrs + + async def _find_or_create_user( + self, + *, + provider_name: str, + nameid: str, + email: str, + name: str, + ) -> dict[str, object]: + """按 (idp_name, NameID) 查找用户；未找到则 JIT 创建。R1a: 禁止邮箱合并。""" + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT u.* FROM users u " + "JOIN user_idp_links l ON l.user_id = u.id " + "WHERE l.idp_name = ? AND l.idp_subject = ?", + (provider_name, nameid), + ) + row = await cursor.fetchone() + if row is not None: + return user_row_to_dict(row) + + user_id = str(uuid.uuid4()) + username = (name or f"{provider_name}:{nameid[:24]}")[:64] + cursor = await db.execute("SELECT id FROM users WHERE username = ?", (username,)) + if await cursor.fetchone() is not None: + username = f"{username[:55]}_{nameid[:8]}" + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + user_email = email or f"{nameid[:24]}@sso.{provider_name}.local" + cursor = await db.execute("SELECT id FROM users WHERE email = ?", (user_email,)) + if await cursor.fetchone() is not None: + user_email = f"{nameid[:24]}@sso.{provider_name}.local" + now = _now_iso() + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at, created_by) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + username, + user_email, + hash_password(secrets.token_urlsafe(32)), + "member", + 1, + 0, + 0, + now, + now, + None, + ), + ) + await db.execute( + "INSERT INTO user_idp_links (idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, 'saml', ?)", + (provider_name, nameid, user_id, now), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info( + f"JIT 创建 SAML 用户: idp={provider_name} nameid={nameid[:12]}... user={username}" + ) + return user_row_to_dict(row) + + # ------------------------------------------------------------------ + # AuthProvider Protocol 其余方法 + # ------------------------------------------------------------------ + + async def get_user_by_id(self, user_id: str) -> User | None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM users WHERE id = ? AND is_active = 1", (user_id,) + ) + row = await cursor.fetchone() + return _row_to_user(row) if row else None + + async def sync_user_attributes(self, user_id: str) -> None: + """SAML 属性同步：ponytail 暂为 no-op。""" + return None + + async def revoke_user(self, user_id: str) -> None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + logger.info(f"SAML provider 禁用用户 {user_id}") + + +def _extract_host(url: str) -> str: + """从 URL 中提取 host（ponytail: 不引入 urllib 解析，简单切分）。""" + if "://" in url: + url = url.split("://", 1)[1] + return url.split("/", 1)[0] if url else "" + + +def _row_to_user(row: aiosqlite.Row) -> User: + return User( + id=row["id"], + username=row["username"], + email=row["email"], + role=row["role"], + is_active=bool(row["is_active"]), + is_terminal_authorized=bool(row["is_terminal_authorized"]), + is_server_terminal_authorized=bool(row["is_server_terminal_authorized"]), + created_at=row["created_at"], + updated_at=row["updated_at"], + last_login_at=row["last_login_at"], + created_by=row["created_by"], + ) diff --git a/src/agentkit/server/auth/scim/__init__.py b/src/agentkit/server/auth/scim/__init__.py new file mode 100644 index 0000000..f21a0c1 --- /dev/null +++ b/src/agentkit/server/auth/scim/__init__.py @@ -0,0 +1,9 @@ +"""SCIM 2.0 最小子集 (U4 SSO 集成)。 + +- POST /scim/v2/Users — 创建用户 +- PATCH /scim/v2/Users/{id} — 禁用 (active=false) / 更新 +- GET /scim/v2/Users — 列表 / 过滤 + +SCIM push 失败触发实时告警（日志 error + OTel event）。 +Bearer token 认证（SCIM token 独立于 JWT，配置于 agentkit.yaml auth.scim_token）。 +""" diff --git a/src/agentkit/server/auth/scim/models.py b/src/agentkit/server/auth/scim/models.py new file mode 100644 index 0000000..fe64298 --- /dev/null +++ b/src/agentkit/server/auth/scim/models.py @@ -0,0 +1,74 @@ +"""SCIM 2.0 用户 schema (Pydantic v2) — U4 最小子集。 + +仅实现 RFC 7643 / 7644 中创建 / 禁用 / 查询所需的字段： +``id``, ``userName``, ``active``, ``emails``, ``displayName``。 +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, EmailStr, Field + + +class SCIMEmail(BaseModel): + """SCIM email 多值条目。""" + + model_config = ConfigDict(extra="allow") + + value: EmailStr + type: str = "work" + primary: bool = False + + +class SCIMUser(BaseModel): + """SCIM 2.0 User 资源（最小子集）。""" + + model_config = ConfigDict(extra="allow") + + id: str | None = None + userName: str + active: bool = True + displayName: str | None = None + emails: list[SCIMEmail] = Field(default_factory=list) + + +class SCIMPatchOp(BaseModel): + """SCIM 2.0 PATCH 操作（最小子集，仅支持 replace）。""" + + model_config = ConfigDict(extra="allow") + + op: str = "replace" + path: str | None = None + value: Any = None + + +class SCIMPatchRequest(BaseModel): + """SCIM PATCH 请求体（RFC 7644）。""" + + model_config = ConfigDict(extra="allow") + + Operations: list[SCIMPatchOp] = Field(default_factory=list) + + +class SCIMListResponse(BaseModel): + """SCIM 2.0 列表响应。""" + + model_config = ConfigDict(extra="allow") + + schemas: list[str] = Field( + default_factory=lambda: ["urn:ietf:params:scim:api:messages:2.0:ListResponse"] + ) + totalResults: int = 0 + startIndex: int = 1 + itemsPerPage: int = 0 + Resources: list[dict[str, Any]] = Field(default_factory=list) + + +class SCIMErrorDetail(BaseModel): + """SCIM 2.0 错误响应。""" + + model_config = ConfigDict(extra="allow") + + status: str + detail: str | None = None diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py new file mode 100644 index 0000000..57dd779 --- /dev/null +++ b/src/agentkit/server/auth/scim/router.py @@ -0,0 +1,252 @@ +"""SCIM 2.0 路由 (U4 最小子集) — handler 内联，不拆 handlers.py。 + +端点： +- POST /scim/v2/Users — 创建用户 +- PATCH /scim/v2/Users/{id} — 禁用 (active=false) / 更新 +- GET /scim/v2/Users — 列表 / 过滤 + +Bearer token 认证（独立于 JWT — ``auth.scim_token`` 配置项）。 +SCIM push 失败触发实时告警（日志 error + OTel span event）。 +""" + +from __future__ import annotations + +import hmac +import logging +import os +import secrets +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from pydantic import BaseModel, ConfigDict + +from ...auth.models import DEFAULT_AUTH_DB_PATH, init_auth_db, user_row_to_dict +from ...auth.password import hash_password +from .models import SCIMListResponse, SCIMPatchRequest, SCIMUser + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/scim/v2", tags=["scim"]) + +_SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _resolve_db_path(request: Request) -> Path: + path = getattr(request.app.state, "auth_db_path", None) + return Path(path) if path else DEFAULT_AUTH_DB_PATH + + +def _resolve_scim_token(request: Request) -> str | None: + """从 app.state 或环境变量解析 SCIM bearer token。""" + token = getattr(request.app.state, "scim_token", None) + if token: + return token + return os.environ.get("AGENTKIT_SCIM_TOKEN") + + +async def _verify_scim_token(request: Request) -> str: + """SCIM bearer token 校验 — 恒定时间比较防时序攻击。""" + expected = _resolve_scim_token(request) + if not expected: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="SCIM 未配置 bearer token（auth.scim_token）", + ) + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + raise HTTPException(status_code=401, detail="缺少 SCIM Bearer token") + provided = auth_header[7:] + if not hmac.compare_digest(provided, expected): + raise HTTPException(status_code=401, detail="SCIM token 无效") + return provided + + +def _alert_scim_failure(operation: str, detail: str, user_id: str | None = None) -> None: + """SCIM 推送失败实时告警 — 日志 error + OTel span event。 + + ponytail: 仅日志 + OTel event。生产可扩展为 webhook / 告警通道。 + 上限：告警去重 / 限流需在告警通道侧实现。 + """ + logger.error("SCIM 推送失败 [op=%s user=%s]: %s", operation, user_id or "?", detail) + try: + from agentkit.telemetry.tracer import get_tracer + + tracer = get_tracer() + span = tracer.start_span("scim.push.failure") + span.set_attribute("scim.operation", operation) + span.set_attribute("scim.error", detail) + if user_id: + span.set_attribute("scim.user_id", user_id) + span.add_event("scim.alert", {"operation": operation, "detail": detail}) + except Exception: # noqa: BLE001 — OTel 不可用不阻断主流程 + pass + + +async def _ensure_db(request: Request) -> Path: + db_path = _resolve_db_path(request) + await init_auth_db(db_path) + return db_path + + +def _user_to_scim(row: dict[str, object]) -> dict[str, Any]: + """本地 user dict → SCIM 2.0 User JSON。""" + email = str(row.get("email", "") or "") + return { + "schemas": [_SCIM_USER_SCHEMA], + "id": str(row["id"]), + "userName": str(row["username"]), + "active": bool(row.get("is_active", True)), + "displayName": str(row.get("username", "")), + "emails": [{"value": email, "type": "work", "primary": True}] if email else [], + } + + +@router.post("/Users", status_code=status.HTTP_201_CREATED) +async def create_user( + payload: SCIMUser, + request: Request, + _token: str = Depends(_verify_scim_token), +) -> dict[str, Any]: + """SCIM POST /Users — 创建本地用户（随机密码，不可本地登录）。""" + db_path = await _ensure_db(request) + user_id = str(uuid.uuid4()) + now = _now_iso() + email = payload.emails[0].value if payload.emails else f"{user_id}@scim.local" + try: + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + payload.userName, + str(email), + hash_password(secrets.token_urlsafe(32)), + "member", + 1 if payload.active else 0, + 0, + 0, + now, + now, + ), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info("SCIM 创建用户 userName=%s id=%s", payload.userName, user_id) + return _user_to_scim(user_row_to_dict(row)) + except aiosqlite.IntegrityError as exc: + _alert_scim_failure("create", f"用户名或邮箱冲突: {exc}", user_id) + raise HTTPException(status_code=409, detail="userName 或 email 已存在") from exc + except Exception as exc: # noqa: BLE001 + _alert_scim_failure("create", str(exc), user_id) + raise HTTPException(status_code=500, detail="SCIM 创建失败") from exc + + +@router.patch("/Users/{user_id}") +async def patch_user( + user_id: str, + payload: SCIMPatchRequest, + request: Request, + _token: str = Depends(_verify_scim_token), +) -> dict[str, Any]: + """SCIM PATCH /Users/{id} — 禁用 (active=false) 或更新字段。""" + db_path = await _ensure_db(request) + now = _now_iso() + try: + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + raise HTTPException(status_code=404, detail="用户不存在") + for op in payload.Operations: + if op.op.lower() == "replace": + if op.path in (None, "active"): + active = ( + bool(op.value) + if not isinstance(op.value, dict) + else bool(op.value.get("active", True)) + ) + await db.execute( + "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", + (1 if active else 0, now, user_id), + ) + elif op.path == "userName": + await db.execute( + "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", + (str(op.value)[:64], now, user_id), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + return _user_to_scim(user_row_to_dict(row)) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + _alert_scim_failure("patch", str(exc), user_id) + raise HTTPException(status_code=500, detail="SCIM 更新失败") from exc + + +@router.get("/Users") +async def list_users( + request: Request, + filter: str | None = Query(default=None, alias="filter"), + startIndex: int = Query(default=1, ge=1), + count: int = Query(default=100, ge=1, le=200), + _token: str = Depends(_verify_scim_token), +) -> dict[str, Any]: + """SCIM GET /Users — 列表 / 过滤。 + + ponytail: filter 仅支持最简形式 ``userName eq "xxx"`` / ``active eq true``。 + 上限：复杂 SCIM filter 表达式需扩展为完整 AST 解析器。 + """ + db_path = await _ensure_db(request) + where = "" + params: list[Any] = [] + if filter: + # 最简 filter 解析：userName eq "x" / active eq true + if "userName eq" in filter: + val = filter.split("userName eq", 1)[1].strip().strip('"').strip("'") + where = "WHERE username = ?" + params.append(val) + elif "active eq" in filter: + val = filter.split("active eq", 1)[1].strip().strip('"').strip("'").lower() + where = "WHERE is_active = ?" + params.append(1 if val in ("true", "1") else 0) + sql = f"SELECT * FROM users {where} ORDER BY created_at ASC LIMIT ? OFFSET ?" + params.extend([count, startIndex - 1]) + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute(sql, tuple(params)) + rows = await cursor.fetchall() + resources = [_user_to_scim(user_row_to_dict(r)) for r in rows] + return SCIMListResponse( + totalResults=len(resources), + startIndex=startIndex, + itemsPerPage=len(resources), + Resources=resources, + ).model_dump() + + +class SCIMErrorResponse(BaseModel): + """SCIM 错误响应包装。""" + + model_config = ConfigDict(extra="allow") + + status: str + detail: str | None = None diff --git a/src/agentkit/server/routes/auth.py b/src/agentkit/server/routes/auth.py index 4fff714..05fbd14 100644 --- a/src/agentkit/server/routes/auth.py +++ b/src/agentkit/server/routes/auth.py @@ -780,6 +780,138 @@ async def change_password( return {"changed": True, "revoked_other_sessions": revoked} +# --------------------------------------------------------------------------- +# SSO routes — OIDC redirect/callback + SAML ACS (U4) +# --------------------------------------------------------------------------- + + +class SSORedirectResponse(BaseModel): + """SSO 重定向响应 — 返回 IDP authorize URL 供前端跳转。""" + + model_config = ConfigDict(extra="forbid") + + redirect_url: str + state: str + + +async def _sso_issue_token_pair( + request: Request, user_dict: dict[str, object] +) -> TokenResponse: + """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。""" + db_path = await _ensure_db(request) + secret = _resolve_jwt_secret(request) + svc: SessionService = get_session_service() + info = _client_info(request) + + pre_session = await svc.create( + SessionCreate( + user_id=str(user_dict["id"]), + refresh_token="__pending_sso__", + device_fingerprint=info["fingerprint"], + device_label=info["label"], + ip=info["ip"], + user_agent=info["user_agent"], + auth_provider="sso", + ttl_seconds=_refresh_ttl_seconds(False), + ) + ) + token_pair = create_token_pair( + user_id=str(user_dict["id"]), + username=str(user_dict["username"]), + role=str(user_dict["role"]), + secret=secret, + session_id=pre_session.id, + ) + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE auth_sessions SET refresh_token_hash = ? WHERE id = ?", + (hash_token(token_pair.refresh_token), pre_session.id), + ) + await db.execute( + "UPDATE users SET last_login_at = ?, updated_at = ? WHERE id = ?", + (_now_iso(), _now_iso(), str(user_dict["id"])), + ) + await db.commit() + return TokenResponse( + access_token=token_pair.access_token, + refresh_token=token_pair.refresh_token, + expires_in=int(ACCESS_TOKEN_TTL.total_seconds()), + user=UserResponse( + id=str(user_dict["id"]), + username=str(user_dict["username"]), + email=str(user_dict["email"]), + role=str(user_dict["role"]), + is_active=bool(user_dict.get("is_active", True)), + is_terminal_authorized=bool(user_dict.get("is_terminal_authorized", False)), + is_server_terminal_authorized=bool(user_dict.get("is_server_terminal_authorized", False)), + ), + session_id=pre_session.id, + ) + + +@router.get("/oauth/{provider}/redirect", response_model=SSORedirectResponse) +async def oidc_redirect(provider: str, request: Request) -> SSORedirectResponse: + """OIDC 重定向 — 返回 IDP authorize URL（前端跳转）。""" + from agentkit.server.auth.providers.oidc import OIDCProvider + + try: + url = await OIDCProvider().get_redirect_url(provider) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + # state 已由 provider 写入 _state_cache，从 URL 中提取回传前端 + state = "" + if "state=" in url: + state = url.split("state=", 1)[1].split("&", 1)[0] + return SSORedirectResponse(redirect_url=url, state=state) + + +@router.get("/oauth/{provider}/callback", response_model=TokenResponse) +async def oidc_callback( + provider: str, + code: str, + state: str, + request: Request, +) -> TokenResponse: + """OIDC callback — code 换 token + userinfo + JIT 用户关联 → JWT pair。""" + await _ensure_db(request) + from agentkit.server.auth.providers.oidc import OIDCProvider + + try: + user_dict = await OIDCProvider().handle_callback(provider, code, state) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return await _sso_issue_token_pair(request, user_dict) + + +class SAMLACSRequest(BaseModel): + """SAML ACS 请求体。""" + + model_config = ConfigDict(extra="forbid") + + SAMLResponse: str + RelayState: str | None = None + + +@router.post("/saml/{provider}/acs", response_model=TokenResponse) +async def saml_acs( + provider: str, + payload: SAMLACSRequest, + request: Request, +) -> TokenResponse: + """SAML ACS — 消费 SAMLResponse + NameID 提取 + JIT 用户关联 → JWT pair。""" + await _ensure_db(request) + from agentkit.server.auth.providers.saml import SAMLProvider + + request_url = str(request.url) + try: + user_dict = await SAMLProvider().handle_acs( + provider, payload.SAMLResponse, request_url=request_url + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return await _sso_issue_token_pair(request, user_dict) + + # --------------------------------------------------------------------------- # Admin routes # --------------------------------------------------------------------------- @@ -889,6 +1021,123 @@ async def admin_revoke_user_session( return {"revoked": ok} +# --------------------------------------------------------------------------- +# Admin deprovisioning + RBAC role change (U4 — R1b / KTD-8) +# --------------------------------------------------------------------------- + + +class DeprovisionRequest(BaseModel): + """手动 deprovisioning 请求体。""" + + model_config = ConfigDict(extra="forbid") + + reason: str | None = None + break_glass: bool = False # 高权限账户 break-glass 标记 + + +@admin_router.post("/users/{user_id}/deprovision") +async def deprovision_user( + user_id: str, + payload: DeprovisionRequest, + request: Request, + admin: dict[str, Any] = Depends(_require_admin), +) -> dict[str, Any]: + """手动 deprovisioning (R1b) — operator/admin RBAC 强制。 + + 1. 禁用本地用户 (is_active=0) + 2. 撤销该用户所有活跃 session + 3. 记录审计 (actor/target/reason/source/timestamp)，接入 OTel + 4. break_glass=True 时记录 source=break_glass（高权限账户应急通道） + """ + db_path = await _ensure_db(request) + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + raise HTTPException(status_code=404, detail="User not found") + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + # 撤销所有 session + svc: SessionService = get_session_service() + revoked = await svc.revoke_all_for_user(user_id, reason="admin_revoked") + # 审计记录 + from agentkit.server.auth.audit import ( + SOURCE_BREAKGLASS, + SOURCE_MANUAL, + get_audit_service, + ) + + source = SOURCE_BREAKGLASS if payload.break_glass else SOURCE_MANUAL + await get_audit_service().record_deprovision( + actor_user_id=str(admin.get("user_id") or ""), + actor_username=str(admin.get("username") or ""), + target_user_id=user_id, + target_username=str(row["username"]), + reason=payload.reason, + source=source, + ) + return {"deprovisioned": True, "revoked_sessions": revoked} + + +class RoleChangeRequest(BaseModel): + """RBAC 角色变更请求体。""" + + model_config = ConfigDict(extra="forbid") + + role: str # member / operator / admin + reason: str | None = None + + +@admin_router.post("/users/{user_id}/role") +async def change_user_role( + user_id: str, + payload: RoleChangeRequest, + request: Request, + admin: dict[str, Any] = Depends(_require_admin), +) -> dict[str, Any]: + """RBAC 角色变更 (KTD-8) — admin only。 + + 记录 actor/target/role_before/role_after/reason/source/timestamp，接入 OTel 审计。 + 变更后撤销该用户所有现有 session（强制重新登录以刷新 JWT role claim）。 + """ + if payload.role not in ("member", "operator", "admin"): + raise HTTPException(status_code=400, detail="role 必须是 member/operator/admin 之一") + db_path = await _ensure_db(request) + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + raise HTTPException(status_code=404, detail="User not found") + role_before = str(row["role"]) + await db.execute( + "UPDATE users SET role = ?, updated_at = ? WHERE id = ?", + (payload.role, _now_iso(), user_id), + ) + await db.commit() + # 审计记录 + from agentkit.server.auth.audit import get_audit_service + + await get_audit_service().record_role_change( + actor_user_id=str(admin.get("user_id") or ""), + actor_username=str(admin.get("username") or ""), + target_user_id=user_id, + target_username=str(row["username"]), + role_before=role_before, + role_after=payload.role, + reason=payload.reason, + source="manual", + ) + # 撤销所有 session（强制重新登录刷新 role claim） + svc: SessionService = get_session_service() + revoked = await svc.revoke_all_for_user(user_id, reason="role_changed") + return {"role_changed": True, "role_before": role_before, "role_after": payload.role, "revoked_sessions": revoked} + + # Backwards-compat /me endpoint (kept for the existing frontend) @router.get("/me", response_model=UserResponse) async def me( diff --git a/tests/unit/admin/test_models.py b/tests/unit/admin/test_models.py index ff97ef0..fa19a0f 100644 --- a/tests/unit/admin/test_models.py +++ b/tests/unit/admin/test_models.py @@ -120,9 +120,9 @@ async def _list_table_names(db: aiosqlite.Connection) -> set[str]: class TestSchemaVersion: - def test_schema_version_is_v4(self): - """V4 adds the skill_states table (U6 — Admin Console).""" - assert _SCHEMA_VERSION == 4 + def test_schema_version_is_v5(self): + """V5 adds user_idp_links + auth_audit_log (U4 SSO 集成).""" + assert _SCHEMA_VERSION == 5 def test_sqlalchemy_model_table_names(self): assert DepartmentModel.__tablename__ == "departments" @@ -163,13 +163,13 @@ class TestInitAuthDbTables: tables = await _list_table_names(db) assert "department_quotas" in tables - async def test_records_schema_version_4_in_auth_meta(self, fresh_db: Path): + async def test_records_schema_version_5_in_auth_meta(self, fresh_db: Path): async with aiosqlite.connect(str(fresh_db)) as db: db.row_factory = aiosqlite.Row cursor = await db.execute("SELECT value FROM auth_meta WHERE key='schema_version'") row = await cursor.fetchone() assert row is not None - assert row["value"] == "4" + assert row["value"] == "5" assert row["value"] == str(_SCHEMA_VERSION) async def test_init_auth_db_is_idempotent(self, tmp_path: Path): diff --git a/tests/unit/auth/test_models.py b/tests/unit/auth/test_models.py index a2296e7..31a35a8 100644 --- a/tests/unit/auth/test_models.py +++ b/tests/unit/auth/test_models.py @@ -118,9 +118,9 @@ async def _list_index_names(db: aiosqlite.Connection, table: str) -> set[str]: class TestSchemaVersion: - def test_schema_version_is_v4(self): - """The current schema version is 4 (V4 adds skill_states table).""" - assert _SCHEMA_VERSION == 4 + def test_schema_version_is_v5(self): + """The current schema version is 5 (V5 adds user_idp_links + auth_audit_log, U4 SSO).""" + assert _SCHEMA_VERSION == 5 def test_sqlalchemy_model_table_name(self): assert AuthSessionModel.__tablename__ == "auth_sessions" diff --git a/tests/unit/test_auth_audit.py b/tests/unit/test_auth_audit.py new file mode 100644 index 0000000..ec09590 --- /dev/null +++ b/tests/unit/test_auth_audit.py @@ -0,0 +1,208 @@ +"""Audit service 单元测试 (U4 — RBAC + deprovisioning 审计, KTD-8)。 + +验证： +- RBAC 角色变更审计记录（actor/target/role_before/role_after） +- Deprovisioning 审计记录（actor/target/reason/source） +- 审计记录持久化到 auth_audit_log 表 +- 按目标用户查询审计历史 +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import aiosqlite +import pytest + +from agentkit.server.auth.audit import ( + AuditService, + EVENT_DEPROVISION, + EVENT_ROLE_CHANGE, + SOURCE_BREAKGLASS, + SOURCE_MANUAL, + get_audit_service, + set_audit_service, +) +from agentkit.server.auth.models import audit_log_row_to_dict, init_auth_db + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +@pytest.fixture +def audit_service(auth_db: Path) -> AuditService: + svc = AuditService(db_path=auth_db) + set_audit_service(svc) + return svc + + +@pytest.fixture +async def seed_users(auth_db: Path) -> dict[str, str]: + """插入两个用户（admin + target）用于审计测试。""" + now = datetime.now(timezone.utc).isoformat() + admin_id = str(uuid4()) + target_id = str(uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (admin_id, "admin_alice", "alice@example.com", "hash", "admin", 1, 0, 0, now, now), + ) + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (target_id, "target_bob", "bob@example.com", "hash", "member", 1, 0, 0, now, now), + ) + await db.commit() + return {"admin_id": admin_id, "target_id": target_id} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestRoleChangeAudit: + async def test_records_role_change( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + record = await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + role_before="member", + role_after="operator", + reason="晋升", + source=SOURCE_MANUAL, + ) + assert record["event_type"] == EVENT_ROLE_CHANGE + assert record["role_before"] == "member" + assert record["role_after"] == "operator" + assert record["actor_username"] == "admin_alice" + assert record["target_username"] == "target_bob" + assert record["reason"] == "晋升" + + async def test_persists_to_db( + self, audit_service: AuditService, auth_db: Path, seed_users: dict[str, str] + ) -> None: + await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + role_before="member", + role_after="admin", + reason="提升管理员", + ) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE event_type = ? AND target_user_id = ?", + (EVENT_ROLE_CHANGE, seed_users["target_id"]), + ) + rows = await cursor.fetchall() + assert len(rows) == 1 + row = audit_log_row_to_dict(rows[0]) + assert row["role_before"] == "member" + assert row["role_after"] == "admin" + + +class TestDeprovisionAudit: + async def test_records_deprovision( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + record = await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="离职", + source=SOURCE_MANUAL, + ) + assert record["event_type"] == EVENT_DEPROVISION + assert record["reason"] == "离职" + assert record["source"] == SOURCE_MANUAL + + async def test_break_glass_source( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + """break_glass 应急通道 — source 记录为 break_glass。""" + record = await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="应急禁用", + source=SOURCE_BREAKGLASS, + ) + assert record["source"] == SOURCE_BREAKGLASS + + async def test_persists_to_db( + self, audit_service: AuditService, auth_db: Path, seed_users: dict[str, str] + ) -> None: + await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="审计测试", + ) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE event_type = ?", + (EVENT_DEPROVISION,), + ) + rows = await cursor.fetchall() + assert len(rows) == 1 + + +class TestListForTarget: + async def test_lists_audit_history( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + """按目标用户查询审计历史（admin 视图）。""" + target_id = seed_users["target_id"] + await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=target_id, + target_username="target_bob", + role_before="member", + role_after="operator", + reason="第一次", + ) + await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=target_id, + target_username="target_bob", + reason="第二次", + ) + records = await audit_service.list_for_target(target_id) + assert len(records) == 2 + # 按 created_at DESC 排序 — deprovision 在前 + assert records[0]["event_type"] == EVENT_DEPROVISION + assert records[1]["event_type"] == EVENT_ROLE_CHANGE + + +class TestGetAuditService: + def test_singleton(self) -> None: + svc1 = get_audit_service() + svc2 = get_audit_service() + assert svc1 is svc2 diff --git a/tests/unit/test_oidc_provider.py b/tests/unit/test_oidc_provider.py new file mode 100644 index 0000000..a7ebe86 --- /dev/null +++ b/tests/unit/test_oidc_provider.py @@ -0,0 +1,240 @@ +"""OIDC provider 单元测试 (U4 SSO 集成)。 + +验证： +- redirect URL 生成（含 state） +- callback 用户 JIT 创建按 sub 关联 +- 禁止邮箱合并 (R1a) +- 相同 sub 复用现有用户 +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +import pytest + +from agentkit.server.auth.idp_registry import ( + IDPConfig, + IDPRegistry, + OIDCConfig, +) +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.providers.oidc import OIDCProvider, get_state_cache + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def oidc_idp(monkeypatch: pytest.MonkeyPatch) -> IDPRegistry: + """注册一个测试 OIDC IDP 并替换全局 registry。""" + registry = IDPRegistry() + registry.register( + IDPConfig( + name="test_oidc", + type="oidc", + display_name="Test OIDC", + oidc=OIDCConfig( + client_id="test-client-id", + client_secret="test-client-secret", + server_metadata_url="https://idp.test/authorize", + redirect_uri="https://app.test/api/v1/auth/oauth/test_oidc/callback", + scope="openid email profile", + ), + ) + ) + monkeypatch.setattr("agentkit.server.auth.providers.oidc.get_idp_registry", lambda: registry) + return registry + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """创建临时 auth DB 并指向 AGENTKIT_AUTH_DB。""" + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +class _FakeOAuth2Client: + """替代 AsyncOAuth2Client 的假 client — 不发 HTTP。""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.token = {"access_token": "fake-access-token", "id_token": None} + + async def __aenter__(self) -> "_FakeOAuth2Client": + return self + + async def __aexit__(self, *args: Any) -> None: + pass + + def create_authorization_url(self, url: str, **kwargs: Any) -> tuple[str, dict[str, Any]]: + """构造假 authorize URL（含 state / scope 参数）。""" + state = kwargs.get("state", "") + scope = kwargs.get("scope", "") + client_id = kwargs.get("client_id", "test-client-id") + redirect_uri = kwargs.get("redirect_uri", "") + return ( + f"{url}?response_type=code&client_id={client_id}" + f"&redirect_uri={redirect_uri}&scope={scope}&state={state}", + {"state": state, "url": url}, + ) + + async def fetch_token(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self.token + + async def get(self, url: str) -> Any: + class _Resp: + status_code = 200 + + def json(self) -> dict[str, Any]: + return { + "sub": "oidc-sub-123", + "email": "sso_user@example.com", + "name": "SSO User", + } + + return _Resp() + + +@pytest.fixture +def fake_oauth_client(monkeypatch: pytest.MonkeyPatch) -> None: + """替换 oidc 模块的 AsyncOAuth2Client 为假实现。""" + monkeypatch.setattr("agentkit.server.auth.providers.oidc.AsyncOAuth2Client", _FakeOAuth2Client) + + +@pytest.fixture(autouse=True) +def _reset_state_cache() -> None: + """每个测试前清空 state 缓存。""" + cache = get_state_cache() + cache._store.clear() + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestGetRedirectUrl: + async def test_generates_url_with_state(self, oidc_idp: IDPRegistry) -> None: + url = await OIDCProvider().get_redirect_url("test_oidc") + assert "https://idp.test/authorize" in url + assert "state=" in url + assert "client_id=test-client-id" in url + + async def test_unknown_provider_raises(self, oidc_idp: IDPRegistry) -> None: + with pytest.raises(ValueError, match="未配置"): + await OIDCProvider().get_redirect_url("nonexistent") + + async def test_state_cached_for_csrf(self, oidc_idp: IDPRegistry) -> None: + url = await OIDCProvider().get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + # state 应在缓存中（consume 验证） + assert get_state_cache().consume(state) is True + + +class TestHandleCallback: + async def test_creates_user_by_sub( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """首次登录 JIT 创建用户，按 sub 关联到 user_idp_links。""" + # 预置 state + provider = OIDCProvider() + url = await provider.get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + + user = await provider.handle_callback("test_oidc", "fake-code", state) + assert user["username"] == "SSO User" + assert user["email"] == "sso_user@example.com" + + # 验证 user_idp_links 按 sub 关联 + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM user_idp_links WHERE idp_name = ? AND idp_subject = ?", + ("test_oidc", "oidc-sub-123"), + ) + link = await cursor.fetchone() + assert link is not None + assert link["user_id"] == user["id"] + + async def test_same_sub_reuses_user( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """相同 sub 第二次登录复用现有用户，不重复创建。""" + provider = OIDCProvider() + # 第一次登录 + url1 = await provider.get_redirect_url("test_oidc") + state1 = url1.split("state=", 1)[1].split("&", 1)[0] + user1 = await provider.handle_callback("test_oidc", "code1", state1) + + # 第二次登录（相同 sub） + url2 = await provider.get_redirect_url("test_oidc") + state2 = url2.split("state=", 1)[1].split("&", 1)[0] + user2 = await provider.handle_callback("test_oidc", "code2", state2) + + assert user1["id"] == user2["id"] + + async def test_no_email_merge( + self, + oidc_idp: IDPRegistry, + auth_db: Path, + fake_oauth_client: None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """R1a: 不同 sub 但相同 email — 禁止合并，创建新用户。""" + # 预创建一个本地用户，email 与 SSO userinfo 相同 + now = datetime.now(timezone.utc).isoformat() + local_id = str(uuid.uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + local_id, + "local_user", + "sso_user@example.com", + "$2b$12$dummyhash", + "member", + 1, + 0, + 0, + now, + now, + ), + ) + await db.commit() + + # SSO 用不同 sub（_FakeOAuth2Client 返回 sub=oidc-sub-123） + provider = OIDCProvider() + url = await provider.get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + sso_user = await provider.handle_callback("test_oidc", "code", state) + + # 应创建新用户，而非复用 local_user + assert sso_user["id"] != local_id + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + assert sso_user["email"] != "sso_user@example.com" + assert sso_user["email"].endswith("@sso.test_oidc.local") + + async def test_invalid_state_raises( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """无效 state（CSRF 校验失败）应拒绝。""" + with pytest.raises(ValueError, match="state"): + await OIDCProvider().handle_callback("test_oidc", "code", "invalid-state") + + +class TestAuthenticateNotImplemented: + async def test_raises(self) -> None: + from agentkit.server.auth.providers.exceptions import ProviderNotImplemented + + with pytest.raises(ProviderNotImplemented): + await OIDCProvider().authenticate(username="x", password="y") diff --git a/tests/unit/test_saml_provider.py b/tests/unit/test_saml_provider.py new file mode 100644 index 0000000..6d41cbd --- /dev/null +++ b/tests/unit/test_saml_provider.py @@ -0,0 +1,196 @@ +"""SAML 2.0 provider 单元测试 (U4 SSO 集成)。 + +验证： +- ACS NameID 提取 + JIT 用户创建 +- 按 NameID 关联（R1a: 禁止邮箱合并） +- 相同 NameID 复用现有用户 +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +import pytest + +from agentkit.server.auth.idp_registry import ( + IDPConfig, + IDPRegistry, + SAMLConfig, +) +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.providers.saml import SAMLProvider + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def saml_idp(monkeypatch: pytest.MonkeyPatch) -> IDPRegistry: + """注册一个测试 SAML IDP 并替换全局 registry。""" + registry = IDPRegistry() + registry.register( + IDPConfig( + name="test_saml", + type="saml", + display_name="Test SAML", + saml=SAMLConfig( + entity_id="https://idp.test/entity", + sso_url="https://idp.test/sso", + idp_cert="-----BEGIN CERTIFICATE-----\nfakecert\n-----END CERTIFICATE-----", + sp_entity_id="https://app.test/sp", + acs_url="https://app.test/api/v1/auth/saml/test_saml/acs", + ), + ) + ) + monkeypatch.setattr("agentkit.server.auth.providers.saml.get_idp_registry", lambda: registry) + return registry + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +@pytest.fixture +def mock_saml_parse(monkeypatch: pytest.MonkeyPatch) -> None: + """替换 _process_saml 为假实现 — 返回固定 NameID + attributes。""" + + def _fake_process( + self: SAMLProvider, req: dict, cfg: Any, provider_name: str + ) -> tuple[str, dict[str, Any]]: + return "saml-nameid-456", {"email": "saml_user@example.com", "name": "SAML User"} + + monkeypatch.setattr(SAMLProvider, "_process_saml", _fake_process) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestHandleAcs: + async def test_creates_user_by_nameid( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """ACS 消费 SAMLResponse + NameID 提取 + JIT 用户创建。""" + user = await SAMLProvider().handle_acs("test_saml", "fake-saml-response") + assert user["username"] == "SAML User" + assert user["email"] == "saml_user@example.com" + + # 验证 user_idp_links 按 NameID 关联 + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM user_idp_links WHERE idp_name = ? AND idp_subject = ?", + ("test_saml", "saml-nameid-456"), + ) + link = await cursor.fetchone() + assert link is not None + assert link["idp_type"] == "saml" + assert link["user_id"] == user["id"] + + async def test_same_nameid_reuses_user( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """相同 NameID 第二次登录复用现有用户。""" + provider = SAMLProvider() + user1 = await provider.handle_acs("test_saml", "resp1") + user2 = await provider.handle_acs("test_saml", "resp2") + assert user1["id"] == user2["id"] + + async def test_no_email_merge( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """R1a: 不同 NameID 相同 email — 禁止合并。""" + now = datetime.now(timezone.utc).isoformat() + local_id = str(uuid.uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + local_id, + "local_saml_user", + "saml_user@example.com", + "$2b$12$dummyhash", + "member", + 1, + 0, + 0, + now, + now, + ), + ) + await db.commit() + + # SAML NameID=saml-nameid-456（不同于本地用户） + saml_user = await SAMLProvider().handle_acs("test_saml", "resp") + assert saml_user["id"] != local_id + + async def test_unknown_provider_raises( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + with pytest.raises(ValueError, match="未配置"): + await SAMLProvider().handle_acs("nonexistent", "resp") + + async def test_missing_nameid_raises( + self, + saml_idp: IDPRegistry, + auth_db: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """SAML assertion 缺少 NameID 应拒绝。""" + monkeypatch.setattr( + SAMLProvider, + "_process_saml", + lambda self, req, cfg, name: ("", {}), + ) + with pytest.raises(ValueError, match="NameID"): + await SAMLProvider().handle_acs("test_saml", "resp") + + +class TestAuthenticateNotImplemented: + async def test_raises(self) -> None: + from agentkit.server.auth.providers.exceptions import ProviderNotImplemented + + with pytest.raises(ProviderNotImplemented): + await SAMLProvider().authenticate(username="x", password="y") + + +class TestRevokeUser: + async def test_disables_local_user( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + provider = SAMLProvider() + user = await provider.handle_acs("test_saml", "resp") + await provider.revoke_user(user["id"]) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT is_active FROM users WHERE id = ?", (user["id"],)) + row = await cursor.fetchone() + assert bool(row["is_active"]) is False diff --git a/tests/unit/test_scim_router.py b/tests/unit/test_scim_router.py new file mode 100644 index 0000000..6a1d6d1 --- /dev/null +++ b/tests/unit/test_scim_router.py @@ -0,0 +1,162 @@ +"""SCIM 2.0 router 单元测试 (U4 SSO 集成)。 + +验证 POST / PATCH / GET /scim/v2/Users： +- POST 创建用户 +- PATCH 禁用用户 (active=false) +- GET 列表 / 过滤 +- Bearer token 认证（缺 token / 错误 token → 401） +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.scim.router import router as scim_router + + +SCIM_TOKEN = "test-scim-token-secret" + + +@pytest.fixture +async def scim_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FastAPI: + """创建含 SCIM router 的 TestApp + 初始化 auth DB。""" + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + monkeypatch.setenv("AGENTKIT_SCIM_TOKEN", SCIM_TOKEN) + + app = FastAPI() + app.state.auth_db_path = str(db_path) + app.state.scim_token = SCIM_TOKEN + app.include_router(scim_router, prefix="/api/v1") + return app + + +@pytest.fixture +def client(scim_app: FastAPI) -> TestClient: + return TestClient(scim_app) + + +def _auth_headers(token: str = SCIM_TOKEN) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestCreateUser: + def test_creates_user(self, client: TestClient) -> None: + resp = client.post( + "/api/v1/scim/v2/Users", + json={ + "userName": "scim.user1", + "active": True, + "emails": [{"value": "scim1@example.com", "primary": True}], + }, + headers=_auth_headers(), + ) + assert resp.status_code == 201 + data = resp.json() + assert data["userName"] == "scim.user1" + assert data["active"] is True + assert data["emails"][0]["value"] == "scim1@example.com" + assert data["schemas"] == ["urn:ietf:params:scim:schemas:core:2.0:User"] + + def test_duplicate_username_conflict(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "dup.user", "emails": [{"value": "dup@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "dup.user", "emails": [{"value": "other@example.com"}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 409 + + def test_no_token_unauthorized(self, client: TestClient) -> None: + resp = client.post("/api/v1/scim/v2/Users", json={"userName": "x"}) + assert resp.status_code == 401 + + def test_wrong_token_unauthorized(self, client: TestClient) -> None: + resp = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "x"}, + headers={"Authorization": "Bearer wrong-token"}, + ) + assert resp.status_code == 401 + + +class TestPatchUser: + def test_disable_user(self, client: TestClient) -> None: + # 先创建 + create = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "patch.user", "active": True, "emails": [{"value": "p@example.com"}]}, + headers=_auth_headers(), + ) + user_id = create.json()["id"] + + # PATCH 禁用 + resp = client.patch( + f"/api/v1/scim/v2/Users/{user_id}", + json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 200 + assert resp.json()["active"] is False + + def test_patch_unknown_user_404(self, client: TestClient) -> None: + resp = client.patch( + "/api/v1/scim/v2/Users/nonexistent-id", + json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 404 + + +class TestListUsers: + def test_list_all(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "list.user1", "emails": [{"value": "l1@example.com"}]}, + headers=_auth_headers(), + ) + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "list.user2", "emails": [{"value": "l2@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.get("/api/v1/scim/v2/Users", headers=_auth_headers()) + assert resp.status_code == 200 + data = resp.json() + assert data["totalResults"] >= 2 + assert len(data["Resources"]) >= 2 + + def test_filter_by_username(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "filter.user", "emails": [{"value": "f@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.get( + "/api/v1/scim/v2/Users", + params={"filter": 'userName eq "filter.user"'}, + headers=_auth_headers(), + ) + assert resp.status_code == 200 + data = resp.json() + assert data["totalResults"] == 1 + assert data["Resources"][0]["userName"] == "filter.user" + + def test_no_token_unauthorized(self, client: TestClient) -> None: + resp = client.get("/api/v1/scim/v2/Users") + assert resp.status_code == 401 From af9cb9496cb5bebcd584d4bae02961edea947fe1 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 07:21:41 +0800 Subject: [PATCH 05/11] =?UTF-8?q?feat(deploy):=20U5=20=E2=80=94=20K8s=20He?= =?UTF-8?q?lm=20Chart=20+=20Sealed=20Secrets=20+=20=E7=A6=BB=E7=BA=BF?= =?UTF-8?q?=E5=AE=89=E8=A3=85=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U5 产出政企生产级 K8s 部署 artifacts： Helm Chart — deploy/helm/agentkit/ 含 Chart.yaml + values.yaml + 8 个 templates（deployment/service/ingress/configmap/sealed-secret/ external-secret/serviceaccount/_helpers）。 values.yaml 显式列出 10 个 secret slot（KTD-4）：JWT 签名密钥 / API Key / DB 凭据 / IDP 签名证书 / 第三方 API Key / TLS 服务器证书 / mTLS 客户端证书 / KMS-HSM PIN / OTel exporter token / Redis-PG 密码。 Sealed Secrets + External Secrets Operator 双模板，禁用 plain Kubernetes Secret + Git 提交。serviceaccount 配置 projected token 注入支持 KMS/HSM workload identity。 离线安装包 — build-images.sh 构建镜像 tarball + package-chart.sh 打包 chart tarball + install.sh 离线安装， 覆盖原生 K8s 1.28+ / Helm 3.x。 运维文档 — k8s-install.md 部署指南 + key-rotation-runbook.md 密钥轮换 runbook（10 slot）+ secret-inventory.md 密钥清单。 --- deploy/helm/agentkit/Chart.yaml | 15 + deploy/helm/agentkit/templates/_helpers.tpl | 65 ++++ deploy/helm/agentkit/templates/configmap.yaml | 44 +++ .../helm/agentkit/templates/deployment.yaml | 126 ++++++++ .../agentkit/templates/external-secret.yaml | 65 ++++ deploy/helm/agentkit/templates/ingress.yaml | 41 +++ .../agentkit/templates/sealed-secret.yaml | 42 +++ deploy/helm/agentkit/templates/service.yaml | 19 ++ .../agentkit/templates/serviceaccount.yaml | 36 +++ deploy/helm/agentkit/values.yaml | 237 ++++++++++++++ deploy/offline/Makefile | 37 +++ deploy/offline/scripts/build-images.sh | 24 ++ deploy/offline/scripts/install.sh | 64 ++++ deploy/offline/scripts/package-chart.sh | 33 ++ docs/deployment/k8s-install.md | 244 ++++++++++++++ docs/deployment/key-rotation-runbook.md | 297 ++++++++++++++++++ docs/deployment/secret-inventory.md | 66 ++++ 17 files changed, 1455 insertions(+) create mode 100644 deploy/helm/agentkit/Chart.yaml create mode 100644 deploy/helm/agentkit/templates/_helpers.tpl create mode 100644 deploy/helm/agentkit/templates/configmap.yaml create mode 100644 deploy/helm/agentkit/templates/deployment.yaml create mode 100644 deploy/helm/agentkit/templates/external-secret.yaml create mode 100644 deploy/helm/agentkit/templates/ingress.yaml create mode 100644 deploy/helm/agentkit/templates/sealed-secret.yaml create mode 100644 deploy/helm/agentkit/templates/service.yaml create mode 100644 deploy/helm/agentkit/templates/serviceaccount.yaml create mode 100644 deploy/helm/agentkit/values.yaml create mode 100644 deploy/offline/Makefile create mode 100755 deploy/offline/scripts/build-images.sh create mode 100755 deploy/offline/scripts/install.sh create mode 100755 deploy/offline/scripts/package-chart.sh create mode 100644 docs/deployment/k8s-install.md create mode 100644 docs/deployment/key-rotation-runbook.md create mode 100644 docs/deployment/secret-inventory.md diff --git a/deploy/helm/agentkit/Chart.yaml b/deploy/helm/agentkit/Chart.yaml new file mode 100644 index 0000000..4dfcbe6 --- /dev/null +++ b/deploy/helm/agentkit/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: agentkit +description: Fischer AgentKit — 政企生产级 K8s Helm Chart（含 Sealed Secrets / External Secrets 双模板 + 离线安装支持） +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.28-0" +keywords: + - agentkit + - llm + - enterprise + - k8s +maintainers: + - name: Fischer Team +icon: "" diff --git a/deploy/helm/agentkit/templates/_helpers.tpl b/deploy/helm/agentkit/templates/_helpers.tpl new file mode 100644 index 0000000..149987a --- /dev/null +++ b/deploy/helm/agentkit/templates/_helpers.tpl @@ -0,0 +1,65 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "agentkit.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified app name — release-name + chart-name（避免多 release 命名冲突）。 +*/}} +{{- define "agentkit.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart name + version label。 +*/}} +{{- define "agentkit.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels。 +*/}} +{{- define "agentkit.labels" -}} +helm.sh/chart: {{ include "agentkit.chart" . }} +{{ include "agentkit.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels。 +*/}} +{{- define "agentkit.selectorLabels" -}} +app.kubernetes.io/name: {{ include "agentkit.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name — 若未指定则用 fullname。 +*/}} +{{- define "agentkit.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "agentkit.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Secret backend 名称（Sealed Secret / External Secret 共用的目标 Secret 名）。 +*/}} +{{- define "agentkit.secretName" -}} +{{- printf "%s-secrets" (include "agentkit.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/deploy/helm/agentkit/templates/configmap.yaml b/deploy/helm/agentkit/templates/configmap.yaml new file mode 100644 index 0000000..b63a245 --- /dev/null +++ b/deploy/helm/agentkit/templates/configmap.yaml @@ -0,0 +1,44 @@ +{{- /* +非密配置 ConfigMap — 渲染 agentkit.yaml 的非密段（server/session/bus/task_store/ +skills/experts/logging）。含密钥的段（llm/auth/telemetry headers）由 env 注入， +不进 ConfigMap。 +*/ -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +data: + agentkit.yaml: | + server: + host: {{ .Values.config.server.host | quote }} + port: {{ .Values.config.server.port }} + workers: {{ .Values.config.server.workers }} + rate_limit: {{ .Values.config.server.rate_limit }} + session: + backend: {{ .Values.config.session.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + bus: + backend: {{ .Values.config.bus.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + task_store: + backend: {{ .Values.config.task_store.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + skills: + auto_discover: {{ .Values.config.skills.auto_discover }} + paths: + {{- range .Values.config.skills.paths }} + - {{ . | quote }} + {{- end }} + experts: + paths: + {{- range .Values.config.experts.paths }} + - {{ . | quote }} + {{- end }} + logging: + level: {{ .Values.config.logging.level | quote }} + format: {{ .Values.config.logging.format | quote }} + # DATABASE_URL / REDIS_PASSWORD / LLM provider keys / JWT signing key / + # SCIM token / OTel token 等 — 全部由 env 注入（来自 Secret）。 + # 见 deployment.yaml env / envFrom。 diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml new file mode 100644 index 0000000..b917ab8 --- /dev/null +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "agentkit.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "agentkit.selectorLabels" . | nindent 8 }} + annotations: + # ConfigMap 变更触发 rollout + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ include "agentkit.serviceAccountName" . }} + {{- with .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ . }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: agentkit + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + args: + - serve + - --host + - 0.0.0.0 + - --port + - {{ .Values.service.apiPort | quote }} + ports: + - name: api + containerPort: {{ .Values.service.apiPort }} + protocol: TCP + - name: gui + containerPort: {{ .Values.service.guiPort }} + protocol: TCP + env: + # --- 密钥从统一 Secret 注入（Sealed Secret / External Secret 渲染）--- + - name: JWT_SIGNING_KEY + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.jwtSigningKey.key }} + - name: API_KEY + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.apiKey.key }} + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.dbCredentials.key }} + - name: IDP_SIGNING_CERTS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.idpSigningCerts.key }} + - name: THIRD_PARTY_API_KEYS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.thirdPartyApiKeys.key }} + - name: MTLS_CLIENT_CERT + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.mtlsClientCert.key }} + - name: KMS_HSM_PIN + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.kmsHsmPin.key }} + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.otelExporterToken.key }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: redis-password + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: postgres-password + # --- 非密配置（OTel endpoint、service name）--- + {{- if .Values.otel.enabled }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} + - name: OTEL_SERVICE_NAME + value: {{ .Values.otel.serviceName | quote }} + {{- end }} + envFrom: + # agentkit.yaml 从 ConfigMap 注入（通过 AGENTKIT_CONFIG 环境变量指向挂载路径） + - configMapRef: + name: {{ include "agentkit.fullname" . }} + volumeMounts: + - name: config + mountPath: /etc/agentkit + readOnly: true + - name: tmp + mountPath: /tmp + resources: + {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + {{- toYaml .Values.probes.liveness | nindent 12 }} + readinessProbe: + {{- toYaml .Values.probes.readiness | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ include "agentkit.fullname" . }} + - name: tmp + emptyDir: {} diff --git a/deploy/helm/agentkit/templates/external-secret.yaml b/deploy/helm/agentkit/templates/external-secret.yaml new file mode 100644 index 0000000..bc24ef6 --- /dev/null +++ b/deploy/helm/agentkit/templates/external-secret.yaml @@ -0,0 +1,65 @@ +{{- /* +External Secrets Operator 模板 — 当 secrets.backend=external-secrets 时渲染。 +ExternalSecret 引用运维预创建的 SecretStore（指向 Vault / AWS SM / GCP SM / Azure KV）， +从外部密钥管理服务拉取全部 10 个 secret slot（KTD-4）。 +*/ -}} +{{- if eq .Values.secrets.backend "external-secrets" }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "agentkit.secretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + secretStoreRef: + name: {{ .Values.secrets.externalSecret.secretStoreName | quote }} + kind: SecretStore + target: + name: {{ include "agentkit.secretName" . }} + creationPolicy: Owner + data: + # 1) JWT 签名密钥 + - secretKey: {{ .Values.secrets.jwtSigningKey.key }} + remoteRef: + key: agentkit/jwt-signing-key + # 2) API Key + - secretKey: {{ .Values.secrets.apiKey.key }} + remoteRef: + key: agentkit/api-key + # 3) DATABASE_URL + - secretKey: {{ .Values.secrets.dbCredentials.key }} + remoteRef: + key: agentkit/database-url + # 4) IDP 签名证书 + - secretKey: {{ .Values.secrets.idpSigningCerts.key }} + remoteRef: + key: agentkit/idp-signing-certs + # 5) 第三方 API Key + - secretKey: {{ .Values.secrets.thirdPartyApiKeys.key }} + remoteRef: + key: agentkit/third-party-api-keys + # 6) TLS 服务器证书 + - secretKey: {{ .Values.secrets.tlsServerCert.key }} + remoteRef: + key: agentkit/tls-server-cert + # 7) mTLS 客户端证书 + - secretKey: {{ .Values.secrets.mtlsClientCert.key }} + remoteRef: + key: agentkit/mtls-client-cert + # 8) KMS/HSM PKCS#11 PIN + - secretKey: {{ .Values.secrets.kmsHsmPin.key }} + remoteRef: + key: agentkit/kms-hsm-pin + # 9) OTel exporter token + - secretKey: {{ .Values.secrets.otelExporterToken.key }} + remoteRef: + key: agentkit/otel-exporter-token + # 10) Redis-PG 密码（拆为两个 key） + - secretKey: redis-password + remoteRef: + key: agentkit/redis-password + - secretKey: postgres-password + remoteRef: + key: agentkit/postgres-password +{{- end }} diff --git a/deploy/helm/agentkit/templates/ingress.yaml b/deploy/helm/agentkit/templates/ingress.yaml new file mode 100644 index 0000000..82fa697 --- /dev/null +++ b/deploy/helm/agentkit/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className | quote }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host | quote }} + secretName: {{ .Values.ingress.tls.secretName | quote }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + # GUI 走根路径，API 走 /api 前缀 + - path: /api + pathType: Prefix + backend: + service: + name: {{ include "agentkit.fullname" . }} + port: + name: api + - path: / + pathType: Prefix + backend: + service: + name: {{ include "agentkit.fullname" . }} + port: + name: gui +{{- end }} diff --git a/deploy/helm/agentkit/templates/sealed-secret.yaml b/deploy/helm/agentkit/templates/sealed-secret.yaml new file mode 100644 index 0000000..92489ec --- /dev/null +++ b/deploy/helm/agentkit/templates/sealed-secret.yaml @@ -0,0 +1,42 @@ +{{- /* +Sealed Secrets 模板（Bitnami Sealed Secrets controller）— 当 secrets.backend=sealed-secrets 时渲染。 +覆盖全部 10 个 secret slot（KTD-4）。encryptedData 由运维通过 kubeseal 生成后以 --set 或 +override values 文件注入。禁止明文 Kubernetes Secret + Git 提交。 +*/ -}} +{{- if eq .Values.secrets.backend "sealed-secrets" }} +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: {{ include "agentkit.secretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + encryptedData: + # 1) JWT 签名密钥 + {{ .Values.secrets.jwtSigningKey.key }}: {{ .Values.secrets.jwtSigningKey.encryptedData | default "" | quote }} + # 2) API Key + {{ .Values.secrets.apiKey.key }}: {{ .Values.secrets.apiKey.encryptedData | default "" | quote }} + # 3) DATABASE_URL + {{ .Values.secrets.dbCredentials.key }}: {{ .Values.secrets.dbCredentials.encryptedData | default "" | quote }} + # 4) IDP 签名证书 + {{ .Values.secrets.idpSigningCerts.key }}: {{ .Values.secrets.idpSigningCerts.encryptedData | default "" | quote }} + # 5) 第三方 API Key + {{ .Values.secrets.thirdPartyApiKeys.key }}: {{ .Values.secrets.thirdPartyApiKeys.encryptedData | default "" | quote }} + # 6) TLS 服务器证书（tls.crt + tls.key — ponytail: 单独 Secret 由 cert-manager 或运维创建，此处仅登记） + {{ .Values.secrets.tlsServerCert.key }}: {{ .Values.secrets.tlsServerCert.encryptedData | default "" | quote }} + # 7) mTLS 客户端证书 + {{ .Values.secrets.mtlsClientCert.key }}: {{ .Values.secrets.mtlsClientCert.encryptedData | default "" | quote }} + # 8) KMS/HSM PKCS#11 PIN + {{ .Values.secrets.kmsHsmPin.key }}: {{ .Values.secrets.kmsHsmPin.encryptedData | default "" | quote }} + # 9) OTel exporter token + {{ .Values.secrets.otelExporterToken.key }}: {{ .Values.secrets.otelExporterToken.encryptedData | default "" | quote }} + # 10) Redis-PG 密码（拆为 redis-password + postgres-password 两个 key） + redis-password: {{ .Values.secrets.redisPgPasswords.encryptedRedis | default "" | quote }} + postgres-password: {{ .Values.secrets.redisPgPasswords.encryptedPostgres | default "" | quote }} + template: + metadata: + name: {{ include "agentkit.secretName" . }} + labels: + {{- include "agentkit.labels" . | nindent 8 }} +{{- end }} diff --git a/deploy/helm/agentkit/templates/service.yaml b/deploy/helm/agentkit/templates/service.yaml new file mode 100644 index 0000000..7cb8110 --- /dev/null +++ b/deploy/helm/agentkit/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: api + port: {{ .Values.service.apiPort }} + targetPort: api + protocol: TCP + - name: gui + port: {{ .Values.service.guiPort }} + targetPort: gui + protocol: TCP + selector: + {{- include "agentkit.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/agentkit/templates/serviceaccount.yaml b/deploy/helm/agentkit/templates/serviceaccount.yaml new file mode 100644 index 0000000..ac96f10 --- /dev/null +++ b/deploy/helm/agentkit/templates/serviceaccount.yaml @@ -0,0 +1,36 @@ +{{- /* +ServiceAccount — 当 serviceAccount.create=true 时创建。 +workloadIdentity.enabled=true 时： + 1) automountServiceAccountToken=false（禁用默认 token 投递） + 2) 额外创建 type=service-account-token 的 Secret（projected token），供 KMS/HSM 使用 +*/ -}} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "agentkit.serviceAccountName" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.workloadIdentity.enabled }} +automountServiceAccountToken: false +{{- end }} +{{- end }} +{{- if and .Values.serviceAccount.create .Values.workloadIdentity.enabled }} +--- +# ponytail: projected SA token — 限时令牌，供 KMS/HSM workload identity 换取访问凭据。 +# 升级路径：云厂商原生 workload identity 注解（P1，避免手动 token 投递）。 +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "agentkit.serviceAccountName" . }}-token + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ include "agentkit.serviceAccountName" . }} +type: kubernetes.io/service-account-token +{{- end }} diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml new file mode 100644 index 0000000..9f65727 --- /dev/null +++ b/deploy/helm/agentkit/values.yaml @@ -0,0 +1,237 @@ +# Fischer AgentKit — Helm values +#政企生产级配置；所有密钥通过 Sealed Secrets 或 External Secrets Operator 注入， +# 禁止明文 Kubernetes Secret + Git 提交（KTD-4 / R2a）。 + +# --------------------------------------------------------------------------- +# 镜像配置 +# --------------------------------------------------------------------------- +image: + repository: fischer-agentkit + tag: 0.1.0 + pullPolicy: IfNotPresent + # 私有镜像仓库时配置 pullSecret（secret 引用名，由外部创建） + pullSecret: "" + +# --------------------------------------------------------------------------- +# 副本与调度 +# --------------------------------------------------------------------------- +replicaCount: 1 +# ponytail: 暂不支持 HPA，单副本起步；扩容由运维手动 rollout。 +# 升级路径： values.replicaCount > 1 + PodDisruptionBudget（P1） + +# --------------------------------------------------------------------------- +# ServiceAccount +# --------------------------------------------------------------------------- +serviceAccount: + create: true + name: "" # 留空则用 fullname + annotations: {} # 云厂商 workload identity 注解（如 EKS/GKE） + +# --------------------------------------------------------------------------- +# Service +# --------------------------------------------------------------------------- +service: + type: ClusterIP + apiPort: 8001 # API 服务（agentkit serve） + guiPort: 8002 # Web GUI（agentkit gui） + +# --------------------------------------------------------------------------- +# Ingress（TLS 终止） +# --------------------------------------------------------------------------- +ingress: + enabled: true + className: nginx + annotations: {} + host: agentkit.example.com + tls: + enabled: true + # 引用 secrets.tlsServerCert 对应的 TLS Secret + secretName: agentkit-tls + +# --------------------------------------------------------------------------- +# 资源 +# --------------------------------------------------------------------------- +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1000m" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Pod 安全上下文（非 root 用户，与 Dockerfile appuser 对齐） +# --------------------------------------------------------------------------- +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + readOnlyRootFilesystem: false # agentkit 需写临时文件（缓存/日志） + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + +# --------------------------------------------------------------------------- +# 探针 +# --------------------------------------------------------------------------- +probes: + liveness: + httpGet: + path: /api/v1/health + port: api + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readiness: + httpGet: + path: /api/v1/health + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# --------------------------------------------------------------------------- +# OTel 可观测性 +# --------------------------------------------------------------------------- +otel: + enabled: true + endpoint: http://otel-collector:4317 # OTLP gRPC + serviceName: fischer-agentkit + # exporter token 从 secrets.otelExporterToken 引用（OTel collector 启用鉴权时） + headers: {} + +# --------------------------------------------------------------------------- +# Redis / PostgreSQL 连接（密码从 secrets 引用） +# --------------------------------------------------------------------------- +redis: + url: redis://redis:6379/0 + # 实际密码通过 env REDIS_PASSWORD 注入（来自 secrets.redisPgPasswords） + +postgres: + # DATABASE_URL（不含密码，密码通过 ${POSTGRES_PASSWORD} 占位由 env 注入） + url: postgresql+asyncpg://agentkit@postgres:5432/agentkit + +# --------------------------------------------------------------------------- +# 认证 / IDP（引用 secrets） +# --------------------------------------------------------------------------- +auth: + scimToken: "" # 从 secrets.apiKey 引用（SCIM bearer） + idps: [] # IDP 配置列表（非密部分），密钥引用 secrets.idpSigningCerts + +# --------------------------------------------------------------------------- +# 密钥清单 — 10 个 secret slot（KTD-4 / R2a） +# --------------------------------------------------------------------------- +# backend 选择： +# sealed-secrets — 使用 Bitnami Sealed Secrets（模板 templates/sealed-secret.yaml） +# external-secrets — 使用 External Secrets Operator（模板 templates/external-secret.yaml） +# 客户按环境二选一；模板按 backend 条件渲染。禁止明文 Kubernetes Secret + Git 提交。 +secrets: + backend: sealed-secrets + + # External Secrets Operator 配置（仅当 backend: external-secrets 时生效） + externalSecret: + # SecretStore 名称（运维预创建，指向 Vault / AWS SM / GCP SM / Azure KV） + secretStoreName: agentkit-vault-store + # 后端类型：vault | aws | gcp | azure + backend: vault + + # --- 10 个 secret slot --- + # 每个 slot：name=Kubernetes Secret key；description=用途；rotation=轮换周期 + + # 1) JWT 签名密钥（HS256）— access(15m) + refresh(7d) token 签名 + jwtSigningKey: + key: jwt-signing-key + description: "JWT access/refresh token HS256 签名密钥" + rotation: 90d + + # 2) API Key — 服务间 / SCIM / 客户端鉴权（恒定时间比较） + apiKey: + key: api-key + description: "服务间调用 / SCIM bearer / agentkit pair 生成的 API Key" + rotation: 180d + + # 3) DB 凭据 — 应用层 DATABASE_URL（含用户名，密码占位由 env 注入） + dbCredentials: + key: database-url + description: "应用层 DATABASE_URL DSN（postgresql+asyncpg://agentkit:***@postgres:5432/agentkit）" + rotation: 90d + + # 4) IDP 签名证书 — OIDC/SAML IdP 签名证书 PEM（多 IDP 信任边界） + idpSigningCerts: + key: idp-signing-certs + description: "OIDC/SAML IdP 签名证书 PEM（JSON map: {idp_name: pem}，支持多 IDP 热轮换）" + rotation: 365d + + # 5) 第三方 API Key（LLM providers）— DashScope / DeepSeek / OpenAI / Anthropic 等 + thirdPartyApiKeys: + key: third-party-api-keys + description: "LLM provider API Key（JSON map: {provider: key}，DashScope/DeepSeek/OpenAI/Anthropic）" + rotation: 90d + + # 6) TLS 服务器证书私钥 — Ingress TLS 终止用的 cert + key + tlsServerCert: + key: tls-server-cert + description: "Ingress TLS 服务器证书 + 私钥（PEM，tls.crt + tls.key）" + rotation: 365d + + # 7) mTLS 客户端证书私钥 — 与下游服务（KMS/HSM、内部 API）双向 TLS + mtlsClientCert: + key: mtls-client-cert + description: "mTLS 客户端证书 + 私钥（PEM，client.crt + client.key + ca.crt）" + rotation: 180d + + # 8) KMS/HSM PKCS#11 PIN — 硬件安全模块访问 PIN + kmsHsmPin: + key: kms-hsm-pin + description: "KMS/HSM PKCS#11 访问 PIN（用于密钥托管 / 签名 / 解密）" + rotation: 365d + + # 9) OTel exporter token — OTLP collector 鉴权 token + otelExporterToken: + key: otel-exporter-token + description: "OTLP exporter 鉴权 token（collector 启用 bearer auth 时）" + rotation: 90d + + # 10) Redis-PG 密码 — Redis requirepass + PostgreSQL 服务密码（基础设施层） + redisPgPasswords: + key: redis-pg-passwords + description: "Redis requirepass + PostgreSQL POSTGRES_PASSWORD（JSON map: {redis, postgres}）" + rotation: 90d + +# --------------------------------------------------------------------------- +# workload identity — KMS/HSM projected service account token +# --------------------------------------------------------------------------- +workloadIdentity: + enabled: false # 启用后 serviceaccount.yaml 注入 projected SA token + audience: sts.example.com # 受众（云厂商 STS / 内部 KMS） + expirationSeconds: 3600 + +# --------------------------------------------------------------------------- +# agentkit.yaml 非密配置（注入 ConfigMap） +# --------------------------------------------------------------------------- +config: + server: + host: 0.0.0.0 + port: 8001 + workers: 1 + rate_limit: 60 + session: + backend: redis + bus: + backend: redis + task_store: + backend: redis + skills: + auto_discover: true + paths: ["./configs/skills"] + experts: + paths: ["./configs/experts"] + logging: + level: INFO + format: text + # 注意：llm / auth / telemetry 等含密钥的段由 env 注入，不进 ConfigMap diff --git a/deploy/offline/Makefile b/deploy/offline/Makefile new file mode 100644 index 0000000..67bb8fc --- /dev/null +++ b/deploy/offline/Makefile @@ -0,0 +1,37 @@ +# Fischer AgentKit — 离线安装包构建编排 +# 目标：在联网环境打包镜像 + Chart，拷贝到离线集群安装。 +# +# 用法： +# make all # 构建镜像 + 打包 chart（联网环境） +# make install # 离线集群安装（需先拷贝 dist/ 过去） +# make clean # 清理 dist/ + +SHELL := /bin/bash +.DEFAULT_GOAL := all + +# 可覆盖变量 +IMAGE_REPO ?= fischer-agentkit +IMAGE_TAG ?= 0.1.0 +CHART_DIR ?= ../helm/agentkit +DIST_DIR ?= dist +NAMESPACE ?= agentkit +RELEASE ?= agentkit + +.PHONY: all build-images package-chart install clean + +all: build-images package-chart + +# 构建镜像并保存为 tarball +build-images: + @bash scripts/build-images.sh $(IMAGE_REPO) $(IMAGE_TAG) $(DIST_DIR) + +# 打包 Helm chart 为 tgz +package-chart: + @bash scripts/package-chart.sh $(CHART_DIR) $(DIST_DIR) + +# 离线安装：加载镜像 + helm install +install: + @bash scripts/install.sh $(DIST_DIR) $(NAMESPACE) $(RELEASE) + +clean: + rm -rf $(DIST_DIR) diff --git a/deploy/offline/scripts/build-images.sh b/deploy/offline/scripts/build-images.sh new file mode 100755 index 0000000..62776c9 --- /dev/null +++ b/deploy/offline/scripts/build-images.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# build-images.sh — 构建 AgentKit Docker 镜像并保存为 tarball（离线安装用） +# 用法：bash build-images.sh [IMAGE_REPO] [IMAGE_TAG] [DIST_DIR] +set -euo pipefail + +IMAGE_REPO="${1:-fischer-agentkit}" +IMAGE_TAG="${2:-0.1.0}" +DIST_DIR="${3:-dist}" +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +mkdir -p "$DIST_DIR" +cd "$REPO_ROOT" + +IMAGE_REF="${IMAGE_REPO}:${IMAGE_TAG}" +echo "==> 构建 Docker 镜像：${IMAGE_REF}" +docker build -t "${IMAGE_REF}" -f Dockerfile . + +TARBALL="${DIST_DIR}/agentkit-image-${IMAGE_TAG}.tar.gz" +echo "==> 保存镜像到 tarball：${TARBALL}" +# ponytail: gzip 压缩镜像，离线传输体积更小；docker save 默认未压缩。 +docker save "${IMAGE_REF}" | gzip > "${TARBALL}" + +echo "==> 完成。产物：${TARBALL}" +echo " 大小：$(du -h "${TARBALL}" | cut -f1)" diff --git a/deploy/offline/scripts/install.sh b/deploy/offline/scripts/install.sh new file mode 100755 index 0000000..c99ebd2 --- /dev/null +++ b/deploy/offline/scripts/install.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# install.sh — 离线安装 AgentKit（加载镜像 + helm install） +# 用法：bash install.sh [DIST_DIR] [NAMESPACE] [RELEASE_NAME] +# 前置：已将 dist/ 拷贝到离线集群节点，节点已安装 docker + helm + kubectl。 +set -euo pipefail + +DIST_DIR="${1:-dist}" +NAMESPACE="${2:-agentkit}" +RELEASE="${3:-agentkit}" +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +cd "${SCRIPT_DIR}" + +echo "==> 检查依赖" +for cmd in docker helm kubectl; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: ${cmd} 未安装" >&2 + exit 1 + fi +done + +if [ ! -d "${DIST_DIR}" ]; then + echo "ERROR: 产物目录不存在：${DIST_DIR}" >&2 + echo " 请先在联网环境执行 make all，并将 dist/ 拷贝到本节点。" >&2 + exit 1 +fi + +# 1) 加载镜像 tarball +IMAGE_TARBALL="$(ls "${DIST_DIR}"/agentkit-image-*.tar.gz 2>/dev/null | head -1 || true)" +if [ -z "${IMAGE_TARBALL}" ]; then + echo "ERROR: 镜像 tarball 未找到（${DIST_DIR}/agentkit-image-*.tar.gz）" >&2 + exit 1 +fi +echo "==> 加载镜像：${IMAGE_TARBALL}" +gunzip -c "${IMAGE_TARBALL}" | docker load +# 列出刚加载的镜像（便于校验 tag） +LOADED_IMAGE="$(docker images --format '{{.Repository}}:{{.Tag}}' | grep agentkit | head -1)" +echo " 已加载：${LOADED_IMAGE}" + +# 2) helm install +CHART_TGZ="$(ls "${DIST_DIR}"/agentkit-*.tgz 2>/dev/null | grep -v agentkit-image | head -1 || true)" +if [ -z "${CHART_TGZ}" ]; then + echo "ERROR: chart tgz 未找到（${DIST_DIR}/agentkit-*.tgz）" >&2 + exit 1 +fi +echo "==> 创建 namespace：${NAMESPACE}" +kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 || \ + kubectl create namespace "${NAMESPACE}" + +echo "==> helm install：${RELEASE} (namespace=${NAMESPACE})" +helm upgrade --install "${RELEASE}" "${CHART_TGZ}" \ + --namespace "${NAMESPACE}" \ + --set image.repository="$(echo "${LOADED_IMAGE}" | cut -d: -f1)" \ + --set image.tag="$(echo "${LOADED_IMAGE}" | cut -d: -f2)" \ + --set image.pullPolicy=Never # 离线本地镜像，Never 强制使用已加载的本地镜像 + +echo "==> 等待 rollout" +kubectl rollout status deployment/"${RELEASE}-agentkit" -n "${NAMESPACE}" --timeout=300s || \ + echo "WARN: rollout 未在 300s 内完成，请检查 pod 状态：kubectl get pods -n ${NAMESPACE}" + +echo "==> 完成。验证：" +echo " kubectl get pods -n ${NAMESPACE}" +echo " kubectl get ingress -n ${NAMESPACE}" +echo " helm status ${RELEASE} -n ${NAMESPACE}" diff --git a/deploy/offline/scripts/package-chart.sh b/deploy/offline/scripts/package-chart.sh new file mode 100755 index 0000000..839f86d --- /dev/null +++ b/deploy/offline/scripts/package-chart.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# package-chart.sh — 打包 Helm chart 为 tgz（离线安装用） +# 用法：bash package-chart.sh [CHART_DIR] [DIST_DIR] +set -euo pipefail + +CHART_DIR="${1:-../helm/agentkit}" +DIST_DIR="${2:-dist}" +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +# 解析 CHART_DIR 相对路径（相对于 offline/ 目录） +case "$CHART_DIR" in + /*) ABS_CHART_DIR="$CHART_DIR" ;; + *) ABS_CHART_DIR="${SCRIPT_DIR}/$CHART_DIR" ;; +esac + +if [ ! -f "${ABS_CHART_DIR}/Chart.yaml" ]; then + echo "ERROR: Chart.yaml 未找到：${ABS_CHART_DIR}" >&2 + exit 1 +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "ERROR: helm 未安装。请先安装 Helm 3.x（https://helm.sh/docs/intro/install/）" >&2 + exit 1 +fi + +mkdir -p "${SCRIPT_DIR}/${DIST_DIR}" + +echo "==> 打包 Helm chart：${ABS_CHART_DIR}" +helm dependency build "${ABS_CHART_DIR}" 2>/dev/null || true # ponytail: 无依赖时静默跳过 +helm package "${ABS_CHART_DIR}" -d "${SCRIPT_DIR}/${DIST_DIR}" + +echo "==> 完成。产物目录：${SCRIPT_DIR}/${DIST_DIR}" +ls -lh "${SCRIPT_DIR}/${DIST_DIR}"/agentkit-*.tgz 2>/dev/null || true diff --git a/docs/deployment/k8s-install.md b/docs/deployment/k8s-install.md new file mode 100644 index 0000000..7f3e070 --- /dev/null +++ b/docs/deployment/k8s-install.md @@ -0,0 +1,244 @@ +# Fischer AgentKit — K8s 部署指南 + +本文档指导在原生 Kubernetes 1.28+ 集群上通过 Helm Chart 部署 Fischer AgentKit。 +覆盖在线与离线两种安装场景，以及 Sealed Secrets / External Secrets 两种密钥后端。 + +## 1. 前置条件 + +| 组件 | 版本 | 说明 | +|------|------|------| +| Kubernetes | >= 1.28 | 原生 K8s（信创 K8s / OpenShift P1 支持） | +| Helm | 3.x | Chart 部署工具 | +| kubectl | 与集群对齐 | 集群操作 | +| docker / containerd | 任意 | 镜像构建与加载 | + +集群需已部署以下 Operator 之一（二选一，对应 `secrets.backend`）： + +- **Sealed Secrets**（`secrets.backend: sealed-secrets`） + - 安装：`helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system` + - 工具：`kubeseal`（用于生成 encryptedData） +- **External Secrets Operator**（`secrets.backend: external-secrets`） + - 安装：`helm install external-secrets external-secrets/external-secrets -n kube-system` + - 后端：Vault / AWS Secrets Manager / GCP Secret Manager / Azure Key Vault（运维预创建 SecretStore） + +依赖中间件（Redis + PostgreSQL）需在命名空间内可达： +- Redis：`redis://redis:6379/0`（可指向集群内 Redis 或外部托管） +- PostgreSQL（含 pgvector 扩展）：`postgresql+asyncpg://agentkit@postgres:5432/agentkit` + +> ponytail: Chart 默认不部署 Redis/PostgreSQL（避免与客户已有基础设施重复）。 +> 升级路径：subchart 方式集成 bitnami/redis + bitnami/postgresql（P1）。 + +## 2. 在线安装 + +### 2.1 准备密钥 + +按 `docs/deployment/secret-inventory.md` 准备 10 个 secret slot 的明文值， +然后根据所选 backend 生成密钥资源。 + +**Sealed Secrets 方式**（推荐单集群 / 离线场景）： + +```bash +# 1) 写明文到临时 Secret（注意：永远不要 git commit 明文 Secret） +kubectl create secret generic agentkit-secrets --dry-run=client -o yaml \ + --from-literal=jwt-signing-key='<32字节随机串>' \ + --from-literal=api-key='' \ + --from-literal=database-url='postgresql+asyncpg://agentkit:@postgres:5432/agentkit' \ + --from-literal=idp-signing-certs='{"feishu":"-----BEGIN CERT..."}' \ + --from-literal=third-party-api-keys='{"dashscope":"sk-...","deepseek":"sk-..."}' \ + --from-literal=tls-server-cert='' \ + --from-literal=mtls-client-cert='' \ + --from-literal=kms-hsm-pin='' \ + --from-literal=otel-exporter-token='Bearer ' \ + --from-literal=redis-password='' \ + --from-literal=postgres-password='' \ + > /tmp/agentkit-secret-plaintext.yaml + +# 2) 用 kubeseal 加密（需 controller 已运行） +kubeseal --controller-name=sealed-secrets --controller-namespace=kube-system \ + --format yaml < /tmp/agentkit-secret-plaintext.yaml > agentkit-sealedsecret.yaml + +# 3) 明文临时文件必须销毁 +shred -u /tmp/agentkit-secret-plaintext.yaml + +# 4) 应用 SealedSecret +kubectl apply -f agentkit-sealedsecret.yaml -n agentkit +``` + +**External Secrets 方式**（推荐多集群 / 云托管场景）： + +```bash +# 1) 在 Vault / AWS SM / GCP SM 中预置 10 个 key（路径见 templates/external-secret.yaml remoteRef.key） +# 2) 创建 SecretStore（指向外部密钥后端） +kubectl apply -f - <<'EOF' +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: agentkit-vault-store + namespace: agentkit +spec: + provider: + vault: + server: https://vault.internal:8200 + path: secret + auth: + kubernetes: + mountPath: kubernetes + role: agentkit +EOF + +# 3) Chart 安装时 secrets.backend=external-secrets 自动生成 ExternalSecret +``` + +### 2.2 helm install + +```bash +# 创建 namespace +kubectl create namespace agentkit + +# 安装（默认 sealed-secrets backend） +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set ingress.host=agentkit.your-domain.com \ + --set image.repository=fischer-agentkit \ + --set image.tag=0.1.0 + +# 或使用 External Secrets backend +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set secrets.backend=external-secrets \ + --set secrets.externalSecret.secretStoreName=agentkit-vault-store \ + --set ingress.host=agentkit.your-domain.com + +# 或使用 override values 文件 +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + -f values-prod.yaml +``` + +### 2.3 启用 workload identity（KMS/HSM） + +```bash +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set workloadIdentity.enabled=true \ + --set workloadIdentity.audience=sts.your-cloud.com +``` + +## 3. 离线安装 + +离线场景（政企内网无公网）需先在联网环境打包，再拷贝到离线集群安装。 + +### 3.1 联网环境打包 + +```bash +cd deploy/offline +make all IMAGE_REPO=fischer-agentkit IMAGE_TAG=0.1.0 +# 产物：dist/agentkit-image-0.1.0.tar.gz + dist/agentkit-0.1.0.tgz +``` + +### 3.2 拷贝到离线集群 + +```bash +# 将整个 dist/ 目录拷贝到离线集群节点（scp / U 盘 / 内网传输） +scp -r dist/ user@offline-node:/opt/agentkit/dist +``` + +### 3.3 离线集群安装 + +```bash +ssh user@offline-node +cd /path/to/deploy/offline +make install DIST_DIR=dist NAMESPACE=agentkit RELEASE=agentkit +# install.sh 会：docker load 镜像 + helm install（image.pullPolicy=Never） +``` + +## 4. 安装后验证 + +```bash +# 1) Pod 状态 +kubectl get pods -n agentkit +# 期望：agentkit-xxx 1/1 Running + +# 2) Service / Ingress +kubectl get svc,ingress -n agentkit + +# 3) 健康检查 +kubectl exec -n agentkit deploy/agentkit-agentkit -- \ + curl -s http://localhost:8001/api/v1/health +# 期望：{"status":"ok"} + +# 4) Secret 已注入 +kubectl exec -n agentkit deploy/agentkit-agentkit -- env | grep -E 'JWT|API_KEY|DATABASE' | sed 's/=.*/=/' +# 期望：所有 env 都有值（不为空） + +# 5) helm 状态 +helm status agentkit -n agentkit +``` + +## 5. 故障排查 + +### 5.1 Pod CrashLoopBackOff + +```bash +# 查看日志 +kubectl logs -n agentkit deploy/agentkit-agentkit --tail=50 + +# 常见原因： +# - Secret 未注入：检查 SealedSecret 是否已解封（kubectl get sealedsecret -n agentkit） +# - DB 连接失败：检查 postgres 服务可达性 + DATABASE_URL 正确性 +# - Redis 连接失败：检查 redis 服务 + REDIS_PASSWORD +``` + +### 5.2 SealedSecret 未解封 + +```bash +kubectl describe sealedsecret agentkit-secrets -n agentkit +# 若 status.conditions 显示 error，原因通常是： +# - sealed-secrets-controller 未运行（kubectl get pods -n kube-system | grep sealed-secrets） +# - 加密时用的 controller 与当前 controller 私钥不匹配（需重新 kubeseal） +``` + +### 5.3 ExternalSecret 同步失败 + +```bash +kubectl get externalsecret agentkit-secrets -n agentkit -o yaml +# 关注 status.conditions，常见： +# - SecretStore 不存在或不可达 +# - Vault / SM 权限不足 +# - remoteRef.key 在外部后端不存在 +``` + +### 5.4 Ingress 不通 + +```bash +# 检查 ingress controller +kubectl get pods -n ingress-nginx +# 检查 TLS secret +kubectl get secret agentkit-tls -n agentkit +# 检查 DNS 解析到 ingress controller LB +``` + +## 6. 卸载 + +```bash +helm uninstall agentkit -n agentkit +kubectl delete namespace agentkit +# 注意：SealedSecret 解封后的明文 Secret 需手动删除 +kubectl delete sealedsecret agentkit-secrets -n agentkit 2>/dev/null || true +``` + +## 7. 升级 + +```bash +# 镜像升级 +helm upgrade agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set image.tag=0.2.0 + +# 配置变更 +helm upgrade agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + -f values-prod.yaml +``` + +密钥轮换流程见 `docs/deployment/key-rotation-runbook.md`。 diff --git a/docs/deployment/key-rotation-runbook.md b/docs/deployment/key-rotation-runbook.md new file mode 100644 index 0000000..7995a3b --- /dev/null +++ b/docs/deployment/key-rotation-runbook.md @@ -0,0 +1,297 @@ +# Fischer AgentKit — 密钥轮换 Runbook + +本文档给出全部 10 个 secret slot（KTD-4 / R2a）的轮换操作流程。 +所有轮换遵循「双密钥并行 → 切换 → 旧密钥退役」原则，避免服务中断。 + +## 通用轮换原则 + +1. **零停机**：新密钥生效后再淘汰旧密钥，避免单点切换失败。 +2. **审计先行**：轮换前记录当前密钥指纹（hash8），轮换后校验变更。 +3. **回滚预案**：保留旧密钥值至少一个轮换周期，便于快速回滚。 +4. **权限最小化**：轮换操作仅限归属角色，操作全程审计日志。 +5. **后端无关**：以下步骤适用于 Sealed Secrets 与 External Secrets 两种后端， + 差异处单独标注。 + +--- + +## Slot 1：JWT 签名密钥（90 天） + +**影响**：所有签发的 JWT access/refresh token 失效，用户需重新登录。 + +**步骤**： + +```bash +# 1) 生成新密钥（32 字节随机串，base64） +NEW_KEY=$(openssl rand -base64 32) + +# 2) 更新密钥后端 +# Sealed Secrets： +kubectl create secret generic agentkit-secrets-new --dry-run=client -o yaml \ + --from-literal=jwt-signing-key="${NEW_KEY}" | \ + kubeseal --controller-name=sealed-secrets --controller-namespace=kube-system \ + --format yaml --merge-into agentkit-sealedsecret.yaml +# External Secrets：在 Vault/SM 中更新 agentkit/jwt-signing-key + +# 3) 应用并等待同步 +kubectl apply -f agentkit-sealedsecret.yaml -n agentkit +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 4) 验证 +kubectl exec -n agentkit deploy/agentkit-agentkit -- \ + python -c "import os; print('jwt key len:', len(os.environ['JWT_SIGNING_KEY']))" + +# 5) 用户重新登录（旧 token 失效） +``` + +**回滚**：保留旧密钥值，若新密钥异常，恢复 SealedSecret/ExternalSecret 并 restart。 + +--- + +## Slot 2：API Key（180 天） + +**影响**：使用旧 API Key 的客户端调用失败。 + +**步骤**： + +```bash +# 1) 生成新 API Key +NEW_KEY=$(openssl rand -hex 32) + +# 2) 更新密钥后端（同 Slot 1 步骤 2，key=api-key） + +# 3) 通知所有使用 API Key 的客户端更新（agentkit pair 重新生成） + +# 4) 应用并 restart +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 5) 验证：用新 key 调用 /api/v1/auth/me 或 SCIM 端点 +``` + +**回滚**：恢复旧 key 值。 + +--- + +## Slot 3：DB 凭据 / DATABASE_URL（90 天） + +**影响**：数据库连接断开，需与 DBA 协同。 + +**步骤**： + +```bash +# 1) DBA 在 PostgreSQL 创建新密码（或新用户） +psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" + +# 2) 更新 DATABASE_URL（含新密码） +NEW_URL="postgresql+asyncpg://agentkit:new-pg-pwd@postgres:5432/agentkit" + +# 3) 更新密钥后端（key=database-url） + +# 4) restart pod（连接池重建） +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 5) 验证：触发一次 DB 操作（如 GET /api/v1/memory） + +# 6) DBA 回收旧密码（确认新连接正常后） +``` + +**注意**：PostgreSQL `ALTER USER` 立即生效，旧连接保持直到连接池重建。 +建议在低峰期操作。 + +**回滚**：DBA 恢复旧密码 + 恢复 DATABASE_URL。 + +--- + +## Slot 4：IDP 签名证书（365 天） + +**影响**：OIDC/SAML 登录校验失败。多 IDP 时仅影响轮换的 IdP。 + +**步骤**： + +```bash +# 1) 从 IdP 管理后台导出新签名证书 PEM + +# 2) 更新 JSON map（保留其他 IdP 证书不变） +# 例如轮换 feishu 证书： +NEW_CERTS='{"feishu":"-----BEGIN CERTIFICATE-----\n新证书\n-----END CERTIFICATE-----","azure_ad":"<原证书>"}' + +# 3) 更新密钥后端（key=idp-signing-certs） + +# 4) restart pod（证书热轮换不重启，但 Helm 部署需 restart 加载新 Secret） +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 5) 验证：通过该 IdP 发起一次 SSO 登录 +``` + +**回滚**：恢复旧证书 JSON map。 + +--- + +## Slot 5：第三方 API Key / LLM providers（90 天） + +**影响**：对应 provider 的 LLM 调用失败。JSON map 结构，单 provider 轮换不影响其他。 + +**步骤**： + +```bash +# 1) 在 provider 控制台生成新 key（DashScope / DeepSeek / OpenAI / Anthropic） + +# 2) 更新 JSON map（保留其他 provider 不变） +NEW_KEYS='{"dashscope":"sk-new1","deepseek":"sk-new2","openai":"sk-old"}' + +# 3) 更新密钥后端（key=third-party-api-keys） + +# 4) restart pod +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 5) 验证：调用对应 provider 的模型（如 agentkit chat 发一条测试消息） + +# 6) 在 provider 控制台撤销旧 key +``` + +**回滚**：恢复旧 key（需 provider 控制台未撤销旧 key，建议保留旧 key 一个周期）。 + +--- + +## Slot 6：TLS 服务器证书（365 天） + +**影响**：Ingress TLS 终止证书过期，浏览器告警 / mTLS 客户端握手失败。 + +**步骤**： + +```bash +# 1) 签发新证书（cert-manager 自动续期，或手动从 CA 申请） +# cert-manager 模式：证书到期前自动续期，无需手动操作。 + +# 2) 若手动：更新 kubernetes.io/tls 类型 Secret +kubectl create secret tls agentkit-tls \ + --cert=new.crt --key=new.key --dry-run=client -o yaml | \ + kubectl apply -f - -n agentkit + +# 3) Ingress controller 自动感知新证书（无需 restart） + +# 4) 验证 +echo | openssl s_client -connect agentkit.your-domain.com:443 -servername agentkit.your-domain.com 2>/dev/null | openssl x509 -noout -dates +``` + +**注意**：本 slot 不在 SealedSecret/ExternalSecret 的 encryptedData 内统一管理 +（TLS Secret 类型为 `kubernetes.io/tls`，与普通 Secret 不同）。 +SealedSecret 模板中登记仅用于清单追踪，实际由 cert-manager 或运维单独管理。 + +**回滚**：恢复旧证书 Secret。 + +--- + +## Slot 7：mTLS 客户端证书（180 天） + +**影响**：与下游服务（KMS/HSM、内部 API）mTLS 握手失败。 + +**步骤**： + +```bash +# 1) 从内部 PKI 签发新客户端证书 + 私钥 + CA 证书 + +# 2) 更新密钥后端（key=mtls-client-cert，PEM 文本） +NEW_PEM="$(cat client.crt client.key ca.crt)" + +# 3) 更新密钥后端 + +# 4) restart pod +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 5) 验证：触发一次 mTLS 调用（如 KMS 签名操作） +``` + +**回滚**：恢复旧 PEM。 + +--- + +## Slot 8：KMS/HSM PKCS#11 PIN（365 天） + +**影响**：HSM 访问失败，密钥托管 / 签名 / 解密不可用。 + +**步骤**： + +```bash +# 1) HSM 管理员通过 HSM 管理工具轮换 PIN（具体命令依 HSM 型号） + +# 2) 更新密钥后端（key=kms-hsm-pin） +NEW_PIN='' + +# 3) restart pod +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 4) 验证：触发一次 HSM 操作（如 PKCS#11 签名） +``` + +**注意**：HSM PIN 轮换需物理 / 管理员协同，建议提前预约维护窗口。 + +**回滚**：恢复旧 PIN（HSM 管理工具支持回滚 PIN）。 + +--- + +## Slot 9：OTel exporter token（90 天） + +**影响**：遥测数据上报失败（trace/metrics 丢失）。服务本身不受影响。 + +**步骤**： + +```bash +# 1) 在 OTel collector / 可观测性平台生成新 token + +# 2) 更新密钥后端（key=otel-exporter-token，格式 'Authorization: Bearer '） +NEW_TOKEN='Bearer new-otel-token' + +# 3) restart pod +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 4) 验证：在 collector 端确认收到新 trace/metrics +``` + +**回滚**：恢复旧 token。 + +--- + +## Slot 10：Redis-PG 密码（90 天） + +**影响**：Redis / PostgreSQL 连接断开。需与基础设施层协同。 + +**步骤**： + +```bash +# 1) Redis 轮换密码 +redis-cli -a '' CONFIG SET requirepass '' + +# 2) PostgreSQL 轮换密码（DBA） +psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" + +# 3) 更新密钥后端（两个 key：redis-password + postgres-password） + +# 4) 更新 Redis 服务配置（若 Redis 由 docker-compose / StatefulSet 管理， +# 需同步更新 requirepass 并 restart Redis） + +# 5) restart agentkit pod +kubectl rollout restart deployment/agentkit-agentkit -n agentkit + +# 6) 验证 +kubectl exec -n agentkit deploy/agentkit-agentkit -- \ + python -c "import redis,os; redis.from_url(os.environ['REDIS_URL'],password=os.environ['REDIS_PASSWORD']).ping()" +``` + +**注意**：Redis `CONFIG SET requirepass` 立即生效，旧连接保持到断开。 +建议低峰期操作，且 Redis 持久化配置（redis.conf）需同步更新，避免重启后失效。 + +**回滚**：恢复 Redis / PG 旧密码 + 恢复 Secret。 + +--- + +## 紧急轮换（泄露响应） + +发生密钥泄露时，立即按以下流程处置： + +1. **隔离**：撤销泄露密钥（provider 控制台 / HSM / CA） +2. **轮换**：按上述对应 slot 步骤紧急轮换 +3. **审计**：检索泄露密钥的使用日志，确认无异常调用 +4. **通报**：通知安全团队 + 受影响用户 +5. **复盘**：记录泄露原因，加固访问控制 + +紧急轮换不遵循「双密钥并行」原则，直接切换到新密钥并废弃旧密钥。 diff --git a/docs/deployment/secret-inventory.md b/docs/deployment/secret-inventory.md new file mode 100644 index 0000000..a766ce6 --- /dev/null +++ b/docs/deployment/secret-inventory.md @@ -0,0 +1,66 @@ +# Fischer AgentKit — 密钥清单（Secret Inventory） + +本文档列出 Helm Chart 涉及的全部 10 个 secret slot（KTD-4 / R2a）， +每个 slot 含：名称、用途、来源、轮换周期、归属角色、对应环境变量。 + +> 密钥后端二选一：Sealed Secrets（`secrets.backend: sealed-secrets`）或 +> External Secrets Operator（`secrets.backend: external-secrets`）。 +> 禁止明文 Kubernetes Secret + Git 提交。 + +## 密钥清单总表 + +| # | Slot 名称 | Secret Key | 环境变量 | 用途 | 来源 | 轮换周期 | 归属 | +|---|-----------|-----------|----------|------|------|----------|------| +| 1 | JWT 签名密钥 | `jwt-signing-key` | `JWT_SIGNING_KEY` | JWT access(15m)/refresh(7d) token HS256 签名 | 运维生成（32+ 字节随机串） | 90 天 | 安全团队 | +| 2 | API Key | `api-key` | `API_KEY` | 服务间调用 / SCIM bearer / `agentkit pair` 鉴权 | 运维生成 | 180 天 | 平台运维 | +| 3 | DB 凭据 | `database-url` | `DATABASE_URL` | 应用层 PG 连接 DSN（含用户名+密码） | DBA 提供 | 90 天 | DBA | +| 4 | IDP 签名证书 | `idp-signing-certs` | `IDP_SIGNING_CERTS` | OIDC/SAML IdP 签名证书 PEM（JSON map，多 IDP 热轮换） | 各 IDP 管理员提供 | 365 天 | 身份团队 | +| 5 | 第三方 API Key | `third-party-api-keys` | `THIRD_PARTY_API_KEYS` | LLM provider API Key（JSON map，DashScope/DeepSeek/OpenAI/Anthropic） | 各 provider 控制台 | 90 天 | LLM 运维 | +| 6 | TLS 服务器证书 | `tls-server-cert` | （Ingress TLS Secret） | Ingress TLS 终止证书 + 私钥 | 证书签发机构 / cert-manager | 365 天 | 平台运维 | +| 7 | mTLS 客户端证书 | `mtls-client-cert` | `MTLS_CLIENT_CERT` | 与下游服务（KMS/HSM、内部 API）双向 TLS 客户端证书 | 内部 PKI | 180 天 | 安全团队 | +| 8 | KMS/HSM PKCS#11 PIN | `kms-hsm-pin` | `KMS_HSM_PIN` | 硬件安全模块 PKCS#11 访问 PIN | HSM 管理员 | 365 天 | HSM 运维 | +| 9 | OTel exporter token | `otel-exporter-token` | `OTEL_EXPORTER_OTLP_HEADERS` | OTLP collector 鉴权 bearer token | 可观测性平台 | 90 天 | 可观测性团队 | +| 10 | Redis-PG 密码 | `redis-password` + `postgres-password` | `REDIS_PASSWORD` / `POSTGRES_PASSWORD` | Redis requirepass + PostgreSQL 服务密码（基础设施层） | DBA / 缓存运维 | 90 天 | DBA | + +## 风险等级 + +| Slot | 泄露影响 | 风险等级 | +|------|----------|----------| +| 1 JWT 签名密钥 | 任意伪造身份 token | 高 | +| 2 API Key | 服务间冒充 / SCIM 越权 | 高 | +| 3 DB 凭据 | 数据库全量读写 | 严重 | +| 4 IDP 签名证书 | SSO 信任链断裂（不能直接伪造，需配合 IdP 私钥） | 中 | +| 5 第三方 API Key | LLM 账单盗刷 / 数据外泄 | 高 | +| 6 TLS 服务器证书 | TLS 中间人 / 域名冒充 | 高 | +| 7 mTLS 客户端证书 | 内部 API 越权访问 | 高 | +| 8 KMS/HSM PIN | 密钥托管被解锁 / 加解密被滥用 | 严重 | +| 9 OTel token | 遥测数据投毒 / 窃取 | 中 | +| 10 Redis-PG 密码 | 缓存/数据库越权 | 严重 | + +## 存储位置 + +所有 secret slot 渲染到统一 Kubernetes Secret（名 `-secrets`）， +由 SealedSecret 或 ExternalSecret 控制器自动创建 / 同步： + +``` +-secrets (Secret) + ├── jwt-signing-key + ├── api-key + ├── database-url + ├── idp-signing-certs + ├── third-party-api-keys + ├── tls-server-cert + ├── mtls-client-cert + ├── kms-hsm-pin + ├── otel-exporter-token + ├── redis-password + └── postgres-password +``` + +deployment.yaml 通过 `env.valueFrom.secretKeyRef` 引用上述全部 key。 +TLS 服务器证书（slot 6）额外被 Ingress 的 `tls.secretName` 引用 +（需由 cert-manager 或运维单独创建为 `kubernetes.io/tls` 类型 Secret）。 + +## 轮换流程 + +详见 `docs/deployment/key-rotation-runbook.md`。 From de02ac8a9264d26d526b75f2c1b82ae9d60ad318 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 07:32:30 +0800 Subject: [PATCH 06/11] =?UTF-8?q?docs(compliance):=20U6=20=E2=80=94=20?= =?UTF-8?q?=E7=AD=89=E4=BF=9D=E4=B8=89=E7=BA=A7=E6=96=87=E6=A1=A3=E5=87=86?= =?UTF-8?q?=E5=A4=87=EF=BC=886=20=E7=BB=B4=E5=BA=A6=20+=20=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E7=82=B9=E6=B8=85=E5=8D=95=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U6 启动等保三级认证文档编写，按 GB/T 22239-2019 6 维度组织： 00-overview — 总览：认证目标、范围、AgentKit 架构摘要、各维度状态。 01-physical — 物理安全：客户机房部署假设。 02-network — 网络安全：Ingress TLS + NetworkPolicy + OTel mTLS。 03-host — 主机安全：非 root 容器 + distroless + 漏洞扫描。 04-application — 应用安全：SSO 多 IDP + RBAC 3 级 + 6 层终端安全 + OTel 审计。 05-data — 数据安全：PII 脱敏 + 国密 P1 参考 + PG 备份。 06-management — 管理安全：密钥轮换 runbook + break-glass 告警。 control-points.yaml — 控制点清单 + AgentKit 实现映射（已实现/P0/P1/待评估）。 --- docs/compliance/dengbao-level3/00-overview.md | 143 +++++ docs/compliance/dengbao-level3/01-physical.md | 99 ++++ docs/compliance/dengbao-level3/02-network.md | 197 +++++++ docs/compliance/dengbao-level3/03-host.md | 197 +++++++ .../dengbao-level3/04-application.md | 257 +++++++++ docs/compliance/dengbao-level3/05-data.md | 217 ++++++++ .../dengbao-level3/06-management.md | 215 ++++++++ .../dengbao-level3/control-points.yaml | 491 ++++++++++++++++++ 8 files changed, 1816 insertions(+) create mode 100644 docs/compliance/dengbao-level3/00-overview.md create mode 100644 docs/compliance/dengbao-level3/01-physical.md create mode 100644 docs/compliance/dengbao-level3/02-network.md create mode 100644 docs/compliance/dengbao-level3/03-host.md create mode 100644 docs/compliance/dengbao-level3/04-application.md create mode 100644 docs/compliance/dengbao-level3/05-data.md create mode 100644 docs/compliance/dengbao-level3/06-management.md create mode 100644 docs/compliance/dengbao-level3/control-points.yaml diff --git a/docs/compliance/dengbao-level3/00-overview.md b/docs/compliance/dengbao-level3/00-overview.md new file mode 100644 index 0000000..484430a --- /dev/null +++ b/docs/compliance/dengbao-level3/00-overview.md @@ -0,0 +1,143 @@ +# 等保三级认证文档 — 总览 + +## 1. 认证目标与范围 + +### 1.1 认证目标 + +本套文档用于 Fischer AgentKit（以下简称 "AgentKit"）通过 GB/T 22239-2019 +《信息安全技术 网络安全等级保护基本要求》**第三级**（等保三级）测评准备。 + +目标：在政企 POC 采购关闭前完成认证 readiness，使 AgentKit 作为应用系统 +具备部署到等保三级信息系统的合规条件。 + +### 1.2 认证范围 + +AgentKit 作为一个**应用系统**纳入等保三级测评，覆盖： + +- 应用软件本身（FastAPI 后端 + Vue 前端 + Tauri 桌面客户端） +- 部署形态：Kubernetes（客户内网）+ Redis + PostgreSQL（含 pgvector） +- 数据处理：用户输入、LLM 调用、记忆存储、终端命令执行 + +**不在范围内**（由客户/平台侧负责）： + +- 物理机房环境（客户机房） +- K8s 宿主机操作系统内核加固 +- 网络骨干设施（防火墙、入侵检测系统、堡垒机） +- PKI 证书签发机构（CA） +- HSM/KMS 硬件设备 + +AgentKit 文档中明确标注「依赖客户侧」与「AgentKit 实现」的边界，避免越界承诺。 + +### 1.3 文档结构 + +本套文档按 GB/T 22239-2019 三级要求 6 个维度组织： + +| 文件 | 维度 | 主要内容 | +|------|------|----------| +| `00-overview.md` | — | 总览（本文档） | +| `01-physical.md` | 物理安全 | 部署假设（客户机房）、物理访问、环境监控、电力 | +| `02-network.md` | 网络安全 | 网络架构、边界防护（Ingress TLS）、访问控制（NetworkPolicy）、入侵防范、安全审计（OTel） | +| `03-host.md` | 主机安全 | 容器镜像安全、K8s node 基线、身份鉴别（SA token）、访问控制（RBAC）、安全审计、入侵防范 | +| `04-application.md` | 应用安全 | 身份鉴别（SSO 多 IDP）、访问控制（RBAC 3 级 + 权限位）、安全审计（OTel + audit.py）、通信完整性（TLS + JWT）、软件容错、资源控制 | +| `05-data.md` | 数据安全 | 数据完整性（PG + pgvector）、数据保密性（PII 脱敏 + 国密 P1）、备份恢复、剩余信息保护（session revoke） | +| `06-management.md` | 管理安全 | 密钥轮换 runbook、break-glass 告警、变更管理、应急响应、安全评估 | +| `control-points.yaml` | — | 控制点清单 + AgentKit 实现映射（已实现 / P0 / P1 / 待评估） | + +## 2. AgentKit 架构摘要 + +### 2.1 技术栈 + +- **后端**：Python 3.11+、FastAPI、Uvicorn、Pydantic v2、SQLAlchemy 2（async） +- **前端**：Vue 3、TypeScript、Vite 5、Ant Design Vue 4、Pinia +- **桌面端**：Tauri 2.x（Rust 外壳 + Python sidecar） +- **基础设施**：Redis（总线/缓存/状态）、PostgreSQL + pgvector（情景记忆） +- **可观测性**：OpenTelemetry（OTLP gRPC 导出，U1 已强制依赖） +- **部署**：Helm Chart（K8s 内网部署，U5） + +### 2.2 部署形态 + +``` +客户内网 K8s 集群 + ├── Ingress (nginx, TLS 终止) ← 唯一对外入口 + │ └── TLS 证书（cert-manager 或手动） + ├── AgentKit Pod (非 root, UID 1001) + │ ├── FastAPI serve (:8001) — API + │ └── FastAPI gui (:8002) — Web GUI + ├── Redis (内部, requirepass) + ├── PostgreSQL + pgvector (内部, 密码) + └── OTel Collector (内部, bearer token) + ↑ OTLP gRPC (mTLS: 待评估) +``` + +密钥通过 Sealed Secrets 或 External Secrets Operator 注入（禁止明文 +Kubernetes Secret + Git 提交，KTD-4 / R2a）。共 10 个 secret slot， +每个含轮换周期（90~365 天）。 + +### 2.3 关键安全子系统映射 + +| 子系统 | 模块 | 等保维度 | +|--------|------|----------| +| OTel 可观测性 | `src/agentkit/telemetry/` | 网络/主机/应用审计 | +| PII 脱敏 | `src/agentkit/llm/pii_filter.py` | 数据保密性 | +| SSO 多 IDP | `src/agentkit/server/auth/providers/`、`idp_registry.py` | 应用身份鉴别 | +| SCIM 2.0 | `src/agentkit/server/auth/scim/router.py` | 应用账户生命周期 | +| RBAC 审计 | `src/agentkit/server/auth/audit.py` | 应用安全审计 | +| 终端安全 6 层 | `src/agentkit/server/auth/terminal_security.py` | 应用访问控制 | +| RBAC 权限 | `src/agentkit/server/auth/permissions.py` | 应用访问控制 | +| JWT 会话 | `src/agentkit/server/auth/jwt_utils.py`、`denylist.py` | 应用身份鉴别 + 剩余信息保护 | +| 密码哈希 | `src/agentkit/server/auth/password.py` | 应用身份鉴别 | +| Helm 部署 | `deploy/helm/agentkit/` | 网络/主机/管理 | +| 密钥轮换 | `docs/deployment/key-rotation-runbook.md` | 管理安全 | + +## 3. 各维度实现状态概览 + +下表汇总各维度控制点的实现状态。状态定义： + +- **已实现**：代码/配置已落地，有对应文件可查 +- **P0**：认证 readiness 必需，本次 POC 采购关闭前需补齐 +- **P1**：下一阶段补齐，文档中标注为参考 +- **待评估**：依赖客户侧或需进一步评估方案 + +| 维度 | 控制点数 | 已实现 | P0 | P1 | 待评估 | +|------|----------|--------|----|----|--------| +| 物理安全 | 6 | 0 | 0 | 0 | 6 | +| 网络安全 | 8 | 4 | 2 | 1 | 1 | +| 主机安全 | 7 | 4 | 2 | 0 | 1 | +| 应用安全 | 8 | 8 | 0 | 0 | 0 | +| 数据安全 | 8 | 6 | 1 | 1 | 0 | +| 管理安全 | 7 | 4 | 1 | 1 | 1 | +| **合计** | **44** | **26** | **6** | **3** | **9** | + +> 应用安全是 AgentKit 自身责任范围，已全部实现。物理安全完全依赖客户机房， +> 故全部为「待评估」。网络/主机/数据/管理维度的 P0 项（共 6 个）是认证 +> readiness 的关键补齐项，详见 `control-points.yaml`。 +> +> **P0 关键补齐项清单**： +> - NET-03 NetworkPolicy 模板 +> - NET-07 审计日志保留 180 天（KTD-9） +> - HST-03 容器日志标准化采集 +> - HST-07 镜像漏洞扫描（Trivy/CI 集成） +> - DAT-03 PostgreSQL 备份恢复 runbook +> - MGT-02 CI/CD 规范文档化 + +## 4. 与既有交付的引用关系 + +本套文档不重复实现细节，只映射到既有交付物： + +| 既有交付 | 等保映射 | 引用文档 | +|----------|----------|----------| +| U1 OTel（telemetry/） | 应用/网络审计 | `02-network.md` §5、`04-application.md` §4 | +| U2 PII 脱敏（pii_filter.py） | 数据保密性 | `05-data.md` §2 | +| U3 DR-fixes（bitable） | 应用容错 | `04-application.md` §5 | +| U4 SSO/SCIM/Audit | 应用身份鉴别/审计 | `04-application.md` §1/§4、`06-management.md` §2 | +| U5 K8s Helm Chart | 网络/主机/管理 | `02-network.md`、`03-host.md`、`06-management.md` | +| R7a 终端安全 6 层 | 应用访问控制 | `04-application.md` §2 | + +## 5. 文档使用说明 + +1. 测评机构按 `control-points.yaml` 逐条核对，每条含 `id` / `dimension` / + `title` / `requirement` / `implementation` / `status` / `references`。 +2. `references` 字段给出 AgentKit 代码文件绝对路径，使用 `file:///` 链接格式。 +3. 状态为 P0 的控制点需在认证前补齐，对应文件中标有「P0 补齐项」段落。 +4. 状态为 P1 的控制点在文档中标注为「P1 参考」，不阻碍三级认证但需写入整改计划。 +5. 状态为「待评估」的控制点依赖客户侧环境，需在客户机房测评时单独核对。 diff --git a/docs/compliance/dengbao-level3/01-physical.md b/docs/compliance/dengbao-level3/01-physical.md new file mode 100644 index 0000000..0611843 --- /dev/null +++ b/docs/compliance/dengbao-level3/01-physical.md @@ -0,0 +1,99 @@ +# 01 — 物理安全 + +> GB/T 22239-2019 第三级「物理和环境安全」控制点映射。 + +## 1. 维度说明 + +AgentKit 作为应用系统以容器形态部署在客户内网 Kubernetes 集群中， +**不拥有也不运维物理机房**。物理安全责任由客户机房运营方承担。 + +本维度文档的目的是： + +1. 明确 AgentKit 部署对物理环境的**假设**（前置条件）。 +2. 列出 AgentKit 自身在物理安全维度的**实现空缺**（全部为「待评估」）。 +3. 在客户机房测评时，由客户方提供物理安全控制点证据， + AgentKit 侧仅提供「部署形态证明」（Helm Chart / Pod 清单）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| PHY-01 | 物理访问控制：机房出入口配置电子门禁，记录进入人员和时间 | +| PHY-02 | 防盗窃和防破坏：主要设备设置防盗标识，机房设置监控系统 | +| PHY-03 | 防雷击：机房设置防雷保安器，接地电阻符合要求 | +| PHY-04 | 防火：机房设置火灾自动消防系统，检测报警器 | +| PHY-05 | 防水和防潮：机房安装水敏检测装置，防止水管安装 | +| PHY-06 | 防静电：机房采用防静电地板，设置静电消除器 | + +## 3. AgentKit 实现映射 + +### 3.1 部署假设（前置条件） + +AgentKit 容器化部署对物理环境的假设： + +1. 客户机房满足 GB/T 22239-2019 三级物理安全全部要求。 +2. K8s 宿主机位于受控机房内，物理访问由客户运营方管理。 +3. 电力供应由机房提供（含 UPS / 后备发电机）。 +4. 网络骨干（防火墙、IDS、堡垒机）由客户网络团队运维。 + +### 3.2 控制点状态 + +全部 6 个物理安全控制点状态均为 **待评估** —— 依赖客户机房测评时由客户方 +提供证据。AgentKit 侧仅提供以下证明材料： + +- 部署形态证明：[deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + （`podSecurityContext.runAsNonRoot: true`，`runAsUser: 1000`） +- Pod 资源清单：[deploy/helm/agentkit/templates/deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) +- 容器镜像基线：[Dockerfile](file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile) + （多阶段构建 + `USER appuser` 非 root 运行） + +### 3.3 控制点详情 + +#### PHY-01 物理访问控制 — 待评估 + +- **要求**：机房出入口配置电子门禁，记录进入人员和时间。 +- **AgentKit 实现**：无物理访问控制责任。AgentKit 软件资产以镜像形式 + 存储在客户镜像仓库，物理访问由客户机房运营方控制。 +- **客户侧需提供**：机房门禁系统记录、人员进出日志、访问审批流程。 + +#### PHY-02 防盗窃和防破坏 — 待评估 + +- **要求**：主要设备设置防盗标识，机房设置监控系统。 +- **AgentKit 实现**：无物理设备责任。K8s worker node 由客户运维。 +- **客户侧需提供**：机房监控系统、设备资产清单。 + +#### PHY-03 防雷击 — 待评估 + +- **要求**：机房设置防雷保安器，接地电阻符合要求。 +- **AgentKit 实现**：无防雷责任。 +- **客户侧需提供**：防雷检测报告、接地电阻测试记录。 + +#### PHY-04 防火 — 待评估 + +- **要求**：机房设置火灾自动消防系统，检测报警器。 +- **AgentKit 实现**：无消防责任。 +- **客户侧需提供**：消防系统验收报告、定期检测记录。 + +#### PHY-05 防水和防潮 — 待评估 + +- **要求**：机房安装水敏检测装置，防止水管安装。 +- **AgentKit 实现**：无防水责任。 +- **客户侧需提供**：水敏检测装置部署记录、机房湿度监控。 + +#### PHY-06 防静电 — 待评估 + +- **要求**：机房采用防静电地板，设置静电消除器。 +- **AgentKit 实现**：无防静电责任。 +- **客户侧需提供**：防静电地板验收报告、静电消除器检测记录。 + +## 4. 测评建议 + +物理安全维度在 AgentKit 侧**无代码/配置可核对**，全部依赖客户机房测评。 +建议在测评前与客户安全团队确认： + +1. 客户机房是否已通过等保三级（或更高）测评。 +2. 客户机房测评报告是否覆盖 AgentKit 部署区域。 +3. 若客户机房未单独测评，需将 AgentKit 部署区域纳入本次测评范围。 + +AgentKit 侧提供的证明材料仅证明「软件部署形态合规」（非 root 容器、 +资源受限、Helm Chart 标准化部署），不替代物理环境测评。 diff --git a/docs/compliance/dengbao-level3/02-network.md b/docs/compliance/dengbao-level3/02-network.md new file mode 100644 index 0000000..18cdc6d --- /dev/null +++ b/docs/compliance/dengbao-level3/02-network.md @@ -0,0 +1,197 @@ +# 02 — 网络安全 + +> GB/T 22239-2019 第三级「网络和通信安全」控制点映射。 + +## 1. 维度说明 + +AgentKit 部署在客户内网 K8s 集群，网络安全的边界由 Ingress 控制， +内部东西向流量由 K8s 网络模型与（待补齐的）NetworkPolicy 控制。 +AgentKit 自身实现： + +- **已实现**：Ingress TLS 终止、OTel 安全审计（日志/trace）、 + Redis/PG 密码鉴权（基础设施层）、API 限流。 +- **P0 补齐项**：NetworkPolicy 模板（东西向流量隔离）、 + 审计日志保留策略（180 天，KTD-9）。 +- **P1 参考**：OTel exporter mTLS、网络入侵检测接入。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| NET-01 | 网络架构：划分网络区域，避免单点故障 | +| NET-02 | 边界防护：边界实施访问控制和安全过滤 | +| NET-03 | 访问控制：网络层访问控制策略，最小权限 | +| NET-04 | 入侵防范：网络入侵检测，恶意流量检测 | +| NET-05 | 安全审计：网络审计日志，覆盖关键网络活动 | +| NET-06 | 恶意代码防范：网络边界恶意代码检测 | +| NET-07 | 安全审计日志保留：日志保留期满足合规要求 | +| NET-08 | 资源控制：会话与带宽限制，防止资源耗尽 | + +## 3. AgentKit 实现映射 + +### 3.1 网络架构（NET-01）— 已实现 + +AgentKit K8s 部署采用单 Ingress 入口 + 内部 Service 架构： + +``` +外网/办公网 ── HTTPS ──> Ingress (nginx, TLS 终止) + │ + ▼ + AgentKit Service (ClusterIP) + │ + ▼ + AgentKit Pod (:8001 API, :8002 GUI) + │ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + Redis PostgreSQL OTel Collector + (ClusterIP) (ClusterIP) (ClusterIP) +``` + +- **Ingress 是唯一对外入口**：[deploy/helm/agentkit/templates/ingress.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml) +- **Service 类型为 ClusterIP**（不直接对外暴露）： + [deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + （`service.type: ClusterIP`） +- **API 与 GUI 走分离路径**：`/api` → API Service，`/` → GUI Service + +### 3.2 边界防护（NET-02）— 已实现 + +Ingress 启用 TLS 终止，所有外网流量强制 HTTPS： + +```yaml +ingress: + enabled: true + className: nginx + tls: + enabled: true + secretName: agentkit-tls # 引用 secrets.tlsServerCert +``` + +- TLS 证书通过 cert-manager 自动续期或手动签发（365 天轮换）。 +- HTTP→HTTPS 跳转由 nginx ingress controller 默认配置提供 + （客户侧 ingress controller 责任）。 +- 证书私钥通过 Sealed Secret / External Secret 注入（KTD-4 / R2a）。 + +**参考文件**： +[deploy/helm/agentkit/templates/ingress.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml) +[deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml)（`ingress` 段、`secrets.tlsServerCert` slot） + +### 3.3 访问控制 / NetworkPolicy（NET-03）— P0 补齐项 + +**当前状态：P0**。Helm Chart 暂无 NetworkPolicy 模板， +Pod 间东西向流量默认全通（K8s 默认网络模型）。 + +**P0 补齐计划**： + +1. 新增 `deploy/helm/agentkit/templates/networkpolicy.yaml`。 +2. 默认 deny-all，按白名单放行： + - Ingress → AgentKit Pod（8001/8002） + - AgentKit Pod → Redis（6379） + - AgentKit Pod → PostgreSQL（5432） + - AgentKit Pod → OTel Collector（4317） +3. 命名空间隔离：AgentKit 独立 namespace，跨 namespace 流量显式声明。 + +**当前缓解**：Service 全部 ClusterIP（不对外），Redis/PG 由客户 K8s 集群 +内部网络策略保护（客户侧责任）。 + +### 3.4 入侵防范（NET-04）— P1 参考 + +**当前状态：P1**。AgentKit 自身不部署网络 IDS， +依赖客户侧网络骨干的 IDS / IPS。 + +- API 层面：[RateLimitMiddleware](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 实现单进程滑动窗口限流，Webhook 入站端点单独限流 + （[src/agentkit/server/routes/channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- 异常登录：JWT 验证失败、SCIM token 校验失败返回 401 + （[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) + 使用 `hmac.compare_digest` 恒定时间比较）。 + +**P1 补齐计划**：接入客户 SIEM，将 AgentKit OTel 日志投递到客户 +入侵检测平台，由 SIEM 做 abnormal pattern 检测。 + +### 3.5 安全审计（NET-05）— 已实现 + +AgentKit 通过 OpenTelemetry（U1，强制依赖）实现网络活动审计： + +- **HTTP 请求 trace**：FastAPI Instrumentor 自动埋点，每个请求生成 span， + 含 HTTP method/path/status/duration。 + [src/agentkit/telemetry/setup.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py) + （`FastAPIInstrumentor`） +- **审计事件 span**：RBAC 角色变更、deprovisioning 写入 OTel span event + （[src/agentkit/server/auth/audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 +- **SCIM 推送失败告警**：span event `scim.push.failure` + （[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py)）。 +- **终端命令审计**：每条终端命令写入 `terminal_audit_logs` 表 + （[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 + +OTLP gRPC 导出到内部 collector +（[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`otel.endpoint: http://otel-collector:4317`）。 + +### 3.6 恶意代码防范（NET-06）— 待评估 + +**当前状态：待评估**。网络边界恶意代码检测由客户网络骨干的 +防病毒网关 / 沙箱负责。AgentKit 不在网络边界位置。 + +AgentKit 侧的缓解： + +- 容器镜像来自可信构建（CI/CD），不在运行时下载可执行文件。 +- 终端命令执行受 6 层白名单 + 危险检测约束（R7a）， + 限制任意命令执行能力（详见 `04-application.md` §2）。 + +### 3.7 安全审计日志保留（NET-07）— P0 补齐项 + +**当前状态：P0**。审计日志保留策略未在代码中强制实施： + +- `auth_audit_log` 表（SQLite）和 `terminal_audit_logs` 表无自动清理 / + 归档策略，依赖运维手动管理。 +- OTel collector 侧的日志保留由客户可观测性平台配置，AgentKit 不控制。 + +**P0 补齐计划**： + +1. 在 Helm values 增加 `audit.retentionDays: 180` 配置（KTD-9）。 +2. 实现审计日志归档 CronJob：超过 180 天的记录导出到对象存储（客户侧） + 并从主表清理。 +3. OTel collector 配置 180 天保留策略（与客户可观测性团队协同）。 + +### 3.8 资源控制（NET-08）— 已实现 + +AgentKit 在应用层与容器层均实现资源控制： + +- **容器层**：Pod resources requests/limits + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `resources` 段，CPU 250m~1000m / Memory 512Mi~2Gi）。 +- **应用层**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 实现 IP 维度滑动窗口限流，Webhook 入站端点独立限流 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- **会话超时**：终端 session 闲置 1800s 自动断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py) + `SESSION_IDLE_TIMEOUT = 1800`）。 +- **JWT 短时效**：access token 15min + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + `ACCESS_TOKEN_TTL`）。 + +## 4. OTel exporter mTLS（P1 参考） + +**当前状态：P1**。OTel exporter 通过 bearer token 鉴权 +（[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`secrets.otelExporterToken` slot），未启用 mTLS。 + +Helm Chart 已预留 `secrets.mtlsClientCert` slot（用于下游服务 mTLS）， +但 OTel exporter 侧的 mTLS 配置待补齐。 + +**P1 补齐计划**： + +1. 在 OTel SDK 初始化时配置 `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE` / + `OTEL_EXPORTER_OTLP_CLIENT_KEY` / `OTEL_EXPORTER_OTLP_CA_CERT` 环境变量。 +2. OTel collector 侧启用 mTLS 服务端证书验证。 +3. 复用 `secrets.mtlsClientCert` slot 注入客户端证书。 + +## 5. 测评建议 + +1. **NET-01 / NET-02 / NET-05 / NET-08**：直接展示 Helm Ingress + OTel + + 限流配置即可。 +2. **NET-03 / NET-07**：P0 补齐项，需在测评前完成 NetworkPolicy 模板和 + 审计日志保留 CronJob。 +3. **NET-04 / NET-06**：依赖客户侧网络骨干，由客户网络团队提供 IDS / + 恶意代码检测证据。 diff --git a/docs/compliance/dengbao-level3/03-host.md b/docs/compliance/dengbao-level3/03-host.md new file mode 100644 index 0000000..9fec7ea --- /dev/null +++ b/docs/compliance/dengbao-level3/03-host.md @@ -0,0 +1,197 @@ +# 03 — 主机安全 + +> GB/T 22239-2019 第三级「设备和计算安全」控制点映射。 +> 本维度以**容器与 Pod**为「主机」单元（AgentKit 不直接运维 K8s node）。 + +## 1. 维度说明 + +AgentKit 以容器形态运行在客户 K8s 集群。「主机安全」分为两层： + +- **容器/Pod 层**（AgentKit 责任）：镜像安全、运行时安全上下文、 + ServiceAccount、Pod 级 RBAC。 +- **K8s node 层**（客户责任）：宿主机操作系统内核加固、node 基线、 + 主机入侵检测（HIDS）。 + +AgentKit 自身实现： + +- **已实现**：非 root 容器、最小权限安全上下文（capabilities drop ALL）、 + ServiceAccount 隔离、容器级资源限制。 +- **P0 补齐项**：容器镜像漏洞扫描（CI 集成 Trivy/Grype）、 + Pod 安全审计日志采集标准化。 +- **P1 参考**：distroless 基础镜像。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| HST-01 | 身份鉴别：设备/系统身份标识与鉴别 | +| HST-02 | 访问控制：操作系统/账户权限最小化，远程管理受控 | +| HST-03 | 安全审计：设备审计日志，覆盖系统管理活动 | +| HST-04 | 入侵防范：主机入侵检测，系统加固 | +| HST-05 | 恶意代码防范：主机防病毒，恶意代码检测 | +| HST-06 | 资源控制：CPU/内存/磁盘限制，会话超时 | +| HST-07 | 镜像与补丁管理：系统补丁及时更新，镜像来源可信 | + +## 3. AgentKit 实现映射 + +### 3.1 容器镜像安全（HST-07）— 部分已实现 / P0 补齐 + +#### 已实现 + +- **多阶段构建**：builder + runner 两阶段，最终镜像不含构建工具链。 + [Dockerfile](file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile) +- **非 root 用户**：runner 阶段创建 `appuser`（UID 1001 / GID 1001）， + `USER appuser` 强制非 root 运行。 +- **基础镜像**：`python:3.11-slim`（精简版，比 full 镜像减少攻击面）。 +- **HEALTHCHECK**：内置健康检查 + （`/api/v1/health`，30s 间隔）。 +- **不写字节码**：`PYTHONDONTWRITEBYTECODE=1`，减少磁盘 .pyc 残留。 + +#### P0 补齐项：镜像漏洞扫描 + +**当前状态：P0**。CI 流水线未集成容器镜像漏洞扫描。 + +**P0 补齐计划**： + +1. CI 流水线增加 Trivy 扫描步骤： + ```yaml + - name: Trivy scan + run: trivy image --severity HIGH,CRITICAL --exit-code 1 fischer-agentkit:${{ github.sha }} + ``` +2. Helm Chart 增加 `image.digest` 字段，部署时使用不可变 digest 而非 tag。 +3. 镜像签名（cosign）— P1，可选。 + +### 3.2 K8s node 安全基线（HST-04 / HST-05）— 待评估 + +**当前状态：待评估**。K8s 宿主机由客户运维，AgentKit 不控制 node 操作系统。 + +客户侧需提供： + +- node 操作系统加固基线（CIS Benchmark for Kubernetes / 操作系统）。 +- 主机入侵检测（HIDS，如 Falco / OSSEC）。 +- 主机防病毒（如客户机房要求）。 + +AgentKit 侧的缓解： + +- Pod `securityContext` 强制非 root + drop ALL capabilities + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `podSecurityContext` / `securityContext` 段），即使 node 被攻破， + 容器逃逸到 root 的难度提升。 + +### 3.3 身份鉴别 / ServiceAccount（HST-01）— 已实现 + +AgentKit Pod 使用专用 ServiceAccount，不共享 default SA： + +- **ServiceAccount 创建**： + [deploy/helm/agentkit/templates/serviceaccount.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/serviceaccount.yaml) +- **values 配置**： + ```yaml + serviceAccount: + create: true + name: "" # 留空则用 fullname（专用 SA） + annotations: {} # 云厂商 workload identity 注解 + ``` +- **Workload Identity**（可选）：`workloadIdentity.enabled: true` 时 + 注入 projected SA token，用于访问云 KMS / STS + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `workloadIdentity` 段）。 + +Pod 内应用层身份鉴别见 `04-application.md` §1（SSO + JWT）。 + +### 3.4 访问控制 / Pod 级 RBAC（HST-02）— 已实现 + +- **capabilities drop ALL**： + ```yaml + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + ``` + ([values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) `securityContext` 段) +- **runAsNonRoot 强制**：`podSecurityContext.runAsNonRoot: true`， + K8s admission 拒绝非 root Pod。 +- **readOnlyRootFilesystem**：当前为 `false`（agentkit 需写临时文件用于缓存/日志）， + 通过 `emptyDir` 挂载 `/tmp` 实现写入隔离 + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `volumes.tmp`）。 +- **privileged: false**（默认，未启用特权模式）。 + +应用层 RBAC（member/operator/admin + 9 权限位）见 +`04-application.md` §2。 + +### 3.5 安全审计 / 容器日志（HST-03）— 已实现 + P0 补齐 + +#### 已实现 + +- **stdout/stderr 标准输出**：AgentKit 日志全部输出到 stdout， + 由 K8s 日志采集（如 Loki / Fluent Bit）收集。 +- **OTel trace**：每个 Pod 的请求 trace 通过 OTLP 导出到 collector + （U1，强制依赖）。 +- **审计事件**：RBAC 变更、deprovisioning、SCIM 操作写入 `auth_audit_log` 表 + + OTel span event（[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 +- **终端命令审计**：每条终端命令 + 决策（executed/confirmed/blocked/denied） + 写入 `terminal_audit_logs` 表 + （[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 + +#### P0 补齐项：审计日志标准化采集 + +**当前状态：P0**。容器日志采集到客户日志平台的链路未标准化。 + +**P0 补齐计划**： + +1. Helm Chart 增加 `logging.fluentBitSidecar` 选项（可选 sidecar 采集）。 +2. 审计日志 JSON 化（structured logging），便于 SIEM 解析。 +3. 与 `02-network.md` §3.7 的 180 天保留策略协同落地。 + +### 3.6 入侵防范（HST-04）— 已实现（容器层） + +容器层入侵防范已实现： + +- **非 root + drop ALL capabilities**：限制容器内可执行的系统调用。 +- **allowPrivilegeEscalation: false**：阻止子进程提权。 +- **资源 limits**：CPU 1000m / Memory 2Gi 上限，防止容器失控耗尽 node 资源 + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `resources.limits`）。 +- **Liveness/Readiness 探针**：故障 Pod 自动重启 + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `livenessProbe` / `readinessProbe`）。 + +node 层入侵检测（HIDS）见 §3.2，依赖客户侧。 + +### 3.7 资源控制（HST-06）— 已实现 + +- **Pod resources**： + ```yaml + resources: + requests: { cpu: 250m, memory: 512Mi } + limits: { cpu: 1000m, memory: 2Gi } + ``` +- **应用层限流**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + IP 维度滑动窗口。 +- **会话超时**：终端 session 1800s 闲置断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py)）。 +- **JWT 短时效**：access 15min / refresh 7d + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py)）。 + +## 4. distroless 基础镜像（P1 参考） + +**当前状态：P1**。当前基础镜像为 `python:3.11-slim`，含 shell + 包管理器， +攻击面大于 distroless。 + +**P1 升级路径**： + +1. 切换到 `gcr.io/distroless/python3-debian12` 作为 runner 基础镜像。 +2. 需调整：distroless 无 shell，HEALTHCHECK 的 `python -c` 命令需改为 + HTTP 探针（K8s livenessProbe httpGet，已支持）。 +3. 多阶段构建的 builder 阶段保持 `python:3.11-slim`（构建工具链需要）。 + +distroless 不阻碍等保三级认证，但作为安全增强项写入整改计划。 + +## 5. 测评建议 + +1. **HST-01 / HST-02 / HST-06**：直接展示 ServiceAccount + securityContext + + resources 配置即可。 +2. **HST-03**：已实现部分展示 audit.py + terminal_security.py， + P0 补齐日志标准化采集。 +3. **HST-07**：P0 补齐镜像漏洞扫描后展示 Trivy 扫描报告。 +4. **HST-04 / HST-05**：容器层已实现，node 层由客户提供 HIDS 证据。 diff --git a/docs/compliance/dengbao-level3/04-application.md b/docs/compliance/dengbao-level3/04-application.md new file mode 100644 index 0000000..2b05041 --- /dev/null +++ b/docs/compliance/dengbao-level3/04-application.md @@ -0,0 +1,257 @@ +# 04 — 应用安全 + +> GB/T 22239-2019 第三级「应用和数据安全 — 应用部分」控制点映射。 + +## 1. 维度说明 + +应用安全是 AgentKit **自身核心责任范围**，全部 8 个控制点已实现。 +本维度文档引用 U4 SSO 多 IDP / SCIM、U1 OTel 审计、R7a 终端安全 6 层、 +RBAC 3 级权限、JWT 会话等既有实现，不重复代码细节。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| APP-01 | 身份鉴别：用户身份标识与鉴别，口令复杂度，登录失败处理 | +| APP-02 | 访问控制：基于角色的访问控制，最小权限，默认拒绝 | +| APP-03 | 安全审计：应用审计日志，覆盖用户操作关键事件 | +| APP-04 | 通信完整性：传输完整性保护，防篡改 | +| APP-05 | 软件容错：错误处理，故障隔离，降级机制 | +| APP-06 | 资源控制：会话并发控制，超时自动断开 | +| APP-07 | 应用账户生命周期：开户/变更/销户审计，SCIM 自动化 | +| APP-08 | 代码安全：输入验证，输出编码，依赖漏洞管理 | + +## 3. AgentKit 实现映射 + +### 3.1 身份鉴别（APP-01）— 已实现 + +#### 多 IDP 单点登录（U4 SSO） + +AgentKit 支持 OIDC + SAML 双协议并存的多 IDP 信任边界： + +- **OIDC 适配器**（authlib redirect flow + JIT 用户创建）： + [src/agentkit/server/auth/providers/oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +- **SAML 2.0 适配器**（python3-saml / OneLogin ACS）： + [src/agentkit/server/auth/providers/saml.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/saml.py) +- **多 IDP 注册表**（热证书轮换 + 单 IDP 故障隔离，R1a 按 sub 关联 + 禁止邮箱合并）： + [src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + +支持的 IDP 类型：飞书 / Azure AD / 阿里云 IDaaS 等任意 OIDC/SAML 合规 IDP。 + +#### 本地密码认证 + +- **bcrypt 哈希**：rounds=12（OWASP 推荐值） + [src/agentkit/server/auth/password.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py) +- **JWT 会话**：HS256 签名，access 15min + refresh 7d（30d 记住我） + [src/agentkit/server/auth/jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + - V2 claims 含 `sid`（session id）+ `jti`（per-token unique id）， + 支持服务端 session 撤销。 +- **Refresh token 轮换 + reuse 检测**：旧 token 进入 denylist 30s 窗口， + reuse 触发撤销该用户全部 session + [src/agentkit/server/auth/denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py) +- **登录失败处理**：JWT 验证失败返回 401，refresh token reuse 触发 + 全 session 撤销（强制重新登录）。 +- **OAuth state CSRF 防护**：OIDC state 缓存 TTL 5min，一次性消费 + （[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) `_StateCache`）。 + +### 3.2 访问控制 / RBAC 3 级（APP-02）— 已实现 + +#### 3 级 RBAC + 9 权限位 + +[src/agentkit/server/auth/permissions.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/permissions.py) + +| 角色 | 权限 | +|------|------| +| `member` | CHAT / KB_QUERY / WORKFLOW_EXECUTE | +| `operator` | member 全部 + KB_WRITE + TERMINAL_LOCAL_USE + TERMINAL_WHITELIST_MANAGE | +| `admin` | operator 全部 + TERMINAL_SERVER_USE + USER_MANAGE + SYSTEM_CONFIG | + +权限位作为稳定字符串标识符用于 JWT payload + 前端 store + 审计日志， +不重命名既有值（向前兼容）。 + +#### 终端安全 6 层白名单（R7a） + +[src/agentkit/server/auth/terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py) + +评估优先级（最高→最低）： + +1. **Blocklist**（admin 管理，永远拒绝） +2. **Built-in safe whitelist**（`_SAFE_COMMAND_PREFIXES`） +3. **Global whitelist**（admin 管理，DB 持久化） +4. **User whitelist**（per-user，DB 持久化 + 内存缓存） +5. **Session whitelist**（per-session，仅内存） +6. **Danger detection**（`_is_dangerous`，需确认） + +**关键安全特性**：shell 操作符（`&&`、`;`、`|`、`$()`、backticks、`>`、`<`、 +换行）**永远绕过所有 whitelist 层**，强制走 danger detection — +即使 `git push` 在白名单中，`git push && rm -rf /` 仍需确认。 + +#### 终端双层授权 + +终端端点要求 RBAC 角色 + 个人授权标志双重校验： + +- `is_terminal_authorized` — 本地终端（client sidecar） +- `is_server_terminal_authorized` — 服务端终端（server PTY） + +允许 admin 按用户授权终端而不改变其角色。 + +#### SCIM token 恒定时间比较 + +SCIM bearer token 使用 `hmac.compare_digest` 恒定时间比较防时序攻击： +[src/agentkit/server/auth/scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +（`_verify_scim_token`）。 + +### 3.3 安全审计（APP-03）— 已实现 + +#### OTel 审计通道（U1 强制依赖） + +[src/agentkit/server/auth/audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) + +审计事件类型： + +- `role_change` — RBAC 角色变更（actor/target/role_before/role_after） +- `deprovision` — 手动 deprovisioning（含 `break_glass` 标记） + +每条记录含 actor/target/reason/source/timestamp，写入 `auth_audit_log` 表 ++ 发 OTel span event（OTel 不可用时降级为 SQLite + 日志）。 + +审计来源（`source` 字段）： + +- `manual` — 手动操作 +- `scim` — SCIM 自动化 +- `cron_reconciliation` — 定时对账 +- `break_glass` — 应急通道（高权限账户） + +#### SCIM 推送失败告警 + +SCIM push 失败实时告警：日志 error + OTel span event `scim.push.failure` +（[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +`_alert_scim_failure`）。 + +#### 终端命令审计 + +每条终端命令 + 决策结果（executed/confirmed/rejected/blocked/denied） ++ cwd + exit_code 写入 `terminal_audit_logs` 表 +（[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py) +`write_audit_log`）。 + +#### OTel 指标 + +U1 OTel 暴露以下审计相关指标 +（[src/agentkit/telemetry/metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)）： + +- `agent.request.total` / `agent.execution.duration` +- `gen_ai.usage.tokens` / `tool.call.duration` +- `prompt_cache.hit` / `prompt_cache.miss` +- `pii_filter.coverage` / `pii_filter.redacted`（U2 PII 脱敏覆盖率） + +### 3.4 通信完整性（APP-04）— 已实现 + +- **传输层 TLS**：Ingress TLS 终止（详见 `02-network.md` §3.2）， + 客户端→Ingress 全程 HTTPS。 +- **JWT 签名完整性**：HS256 签名，任何 payload 字段篡改导致签名校验失败 + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + `verify_token`）。 +- **mTLS 客户端证书**（下游服务）：Helm Chart 预留 `secrets.mtlsClientCert` + slot，用于与 KMS/HSM、内部 API 双向 TLS（详见 `06-management.md`）。 +- **Webhook 签名校验**：入站 Webhook fail-closed，签名校验失败返回 401 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 + +### 3.5 软件容错（APP-05）— 已实现 + +- **LLM 网关 fallback**：多 provider fallback 链 + （[src/agentkit/llm/gateway.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/gateway.py)）。 +- **OTel 运行时容错**：OTel exporter 连接失败时降级为 NoOpTracer， + 应用启动不崩溃（[tracer.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/tracer.py) + `init_telemetry`）。 +- **PII 脱敏 fail-closed**：脱敏异常时拒绝发送至 LLM，不降级到原文 + （[pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py) + `fail_closed=True` 默认）。 +- **专家团队失败回退**：所有阶段失败时回退到单 agent 模式 + （详见 AGENTS.md「专家团队模式」）。 +- **DR-fixes（U3）**：bitable 字段校验（DR-1）、LastViewDeletionError（DR-4）、 + 工具调用容错（DR-5）。 + - [src/agentkit/bitable/repository.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/repository.py) + - [src/agentkit/bitable/service.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/service.py) + - [src/agentkit/tools/bitable_tool.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/tools/bitable_tool.py) + +### 3.6 资源控制（APP-06）— 已实现 + +- **IP 限流**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 滑动窗口（`max_requests` / `window_seconds` 可配置）。 +- **Webhook 独立限流**：[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py) + 入站端点单独限流 + nonce dedup。 +- **会话超时**： + - 终端 session 闲置 1800s 自动断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py)）。 + - JWT access token 15min 过期强制重新认证。 +- **专家名称正则校验**：`_EXPERT_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")` + 防止注入（项目规则）。 +- **HandoffTransport 有界队列**：`maxsize=1024`，关闭使用 sentinel 模式 + 防止无界内存增长（项目规则）。 + +### 3.7 应用账户生命周期 / SCIM（APP-07）— 已实现 + +#### SCIM 2.0 自动化 + +[src/agentkit/server/auth/scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) + +端点： + +- `POST /scim/v2/Users` — 创建用户（随机密码，不可本地登录） +- `PATCH /scim/v2/Users/{id}` — 禁用（active=false）/ 更新字段 +- `GET /scim/v2/Users` — 列表 / 过滤（`userName eq` / `active eq`） + +Bearer token 认证（独立于 JWT，恒定时间比较）。 + +#### Deprovisioning（账户销户） + +管理员通过 `/admin/users/{id}/deprovision` 强制销户： + +1. 禁用本地用户账户（`is_active = 0`） +2. 撤销该用户所有活跃 session +3. 记录审计（actor/target/reason/source/timestamp）+ OTel +4. `break_glass=True` 时记录 `source=break_glass`（应急通道标记） + +详见 `06-management.md` §2 break-glass 告警流程。 + +#### JIT 用户创建（R1a） + +OIDC/SAML 首次登录 JIT 创建本地用户 + `user_idp_links` 行， +按 `(idp_name, sub)` 严格关联，**禁止邮箱合并**已存在账户 +（[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +`_find_or_create_user`）。 + +### 3.8 代码安全（APP-08）— 已实现 + +- **输入验证**：所有 API 入口使用 Pydantic v2 模型校验 + （`model_config = ConfigDict(extra="forbid")` 拒绝额外字段）。 + - SCIM 模型：[src/agentkit/server/auth/scim/models.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/models.py) + - IDP 配置：[src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + （`OIDCConfig` / `SAMLConfig` 均 `extra="forbid"`） +- **禁止 `any` 类型**：项目规则强制使用 Pydantic 模型或 `Unknown` + （AGENTS.md）。 +- **专家名称正则校验**：`_EXPERT_NAME_RE` 防 SQL/命令注入。 +- **API Key 恒定时间比较**：`hmac.compare_digest`（项目规则）。 +- **Ruff lint + format**：`ruff check src/ && ruff format src/`（目标 py311） + 作为代码质量基线。 +- **依赖管理**：`pyproject.toml` 锁定精确版本，`pip install -e ".[dev]"` + 可复现环境。 +- **终端命令安全**：6 层白名单 + shell 操作符检测 + （详见 §3.2），防止命令注入链。 + +## 4. 测评建议 + +应用安全维度全部控制点已实现，测评时直接展示对应代码文件即可： + +1. **APP-01**：展示 oidc.py / saml.py / jwt_utils.py / password.py。 +2. **APP-02**：展示 permissions.py / terminal_security.py + RBAC 矩阵表。 +3. **APP-03**：展示 audit.py + 终端审计日志 + OTel 指标清单。 +4. **APP-04**：展示 ingress.yaml TLS + jwt_utils.py 签名校验 + channels.py + Webhook 签名。 +5. **APP-05**：展示 gateway.py fallback + pii_filter.py fail-closed + + DR-fixes 引用。 +6. **APP-06**：展示 middleware.py 限流 + terminal_server.py 超时。 +7. **APP-07**：展示 scim/router.py + deprovision 端点 + JIT 创建逻辑。 +8. **APP-08**：展示 Pydantic 模型校验 + Ruff 配置 + 终端命令安全。 diff --git a/docs/compliance/dengbao-level3/05-data.md b/docs/compliance/dengbao-level3/05-data.md new file mode 100644 index 0000000..ab2c3c5 --- /dev/null +++ b/docs/compliance/dengbao-level3/05-data.md @@ -0,0 +1,217 @@ +# 05 — 数据安全 + +> GB/T 22239-2019 第三级「应用和数据安全 — 数据部分」控制点映射。 + +## 1. 维度说明 + +AgentKit 处理的数据类型： + +- **用户输入**：聊天消息、终端命令、工作流配置 +- **LLM 调用**：prompt + completion（含潜在 PII） +- **持久化数据**：用户账户、会话、记忆（情景/语义/工作） +- **审计数据**：RBAC 变更、终端命令、SCIM 操作 + +AgentKit 自身实现： + +- **已实现**：PII 脱敏（U2 fail-closed + hash 审计）、 + session 撤销（剩余信息保护）、PG + pgvector 完整性、 + 审计数据 hash 化。 +- **P0 补齐项**：PostgreSQL 备份恢复 runbook。 +- **P1 参考**：国密加密（R11a，SM4/SM2 替代 AES/RSA）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| DAT-01 | 数据完整性：数据传输/存储完整性，校验机制 | +| DAT-02 | 数据保密性：敏感数据加密存储/传输，密钥管理 | +| DAT-03 | 数据备份与恢复：重要数据定期备份，恢复演练 | +| DAT-04 | 剩余信息保护：鉴别信息/文件/内存空间释放前清理 | +| DAT-05 | 个人信息保护：PII 采集/处理/传输最小化与脱敏 | +| DAT-06 | 数据采集与分类分级：按敏感度分级保护 | +| DAT-07 | 数据销毁：数据销毁审计，不可恢复 | +| DAT-08 | 国密算法合规（R11a，政企客户要求）：敏感数据加密使用国密算法 | + +## 3. AgentKit 实现映射 + +### 3.1 数据完整性（DAT-01）— 已实现 + +- **PostgreSQL + pgvector**：主数据库使用 PostgreSQL（事务 ACID 保证）， + pgvector 扩展存储向量记忆数据。 + - 连接串：[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `postgres.url: postgresql+asyncpg://agentkit@postgres:5432/agentkit` + - SQLAlchemy 2 async ORM 提供应用层事务管理。 +- **JWT 签名完整性**：HS256 签名防篡改（详见 `04-application.md` §3.4）。 +- **Webhook 签名校验**：fail-closed 防篡改 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- **Pydantic 模型校验**：所有 API 入口 `extra="forbid"` 拒绝未知字段， + 防止数据注入。 + +### 3.2 数据保密性 / PII 脱敏（DAT-02 / DAT-05）— 已实现（U2） + +#### PII 脱敏 hook + +[src/agentkit/llm/pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py) + +设计要点： + +- **正则版（默认）**：识别手机号 / 邮箱 / 身份证 / 银行卡， + 替换为 `[REDACTED_TYPE:hash8]`。 +- **ner_hook（可选）**：客户内网 NER 模型（LAC/HanLP）， + `module:func` 路径动态 import，优先于正则版。 +- **fail-closed**：任何异常时拒绝发送至 LLM（`raise PIIFilterError`）， + **不降级到原文发送**。 +- **hash 审计**：脱敏前后记录 sha256 前 8 位（仅 hash，不含原文）， + 用于审计追溯。 + +#### PII 类型与正则 + +| PII 类型 | 正则模式 | 替换格式 | +|----------|----------|----------| +| 身份证 | 18 位（含校验位 X） | `[REDACTED_ID_CARD:hash8]` | +| 手机号 | `1[3-9]\d{9}` | `[REDACTED_PHONE:hash8]` | +| 邮箱 | 标准邮箱正则 | `[REDACTED_EMAIL:hash8]` | +| 银行卡 | 16-19 位数字 | `[REDACTED_BANK_CARD:hash8]` | + +> 身份证正则优先于手机号正则匹配，避免身份证号内 11 位数字子串被 +> 手机号正则误匹配。 + +#### OTel 脱敏指标 + +- `pii_filter.coverage` — 脱敏扫描覆盖率（status: success/failed_closed/fallback_regex） +- `pii_filter.redacted` — 脱敏实体计数 + +（[metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)） + +#### fail-closed 策略 + +```python +try: + result = _redact(text, ner_hook=ner_hook) + ... +except Exception as e: + if fail_closed: + raise PIIFilterError(f"PII filter failed (fail-closed): {e}") from e + # fail-open（非默认）：降级到正则版重试 +``` + +生产环境默认 `fail_closed=True`，宁可拒绝服务也不泄露 PII。 + +### 3.3 国密加密（R11a）— P1 参考 + +**当前状态：P1**。AgentKit 当前使用： + +- **传输加密**：TLS（Ingress 终止），算法套件由 nginx ingress controller + 默认配置（含 AES-GCM 等国际算法）。 +- **密码哈希**：bcrypt（国际算法）。 +- **JWT 签名**：HS256（HMAC-SHA256，国际算法）。 +- **PII hash**：SHA-256（国际算法）。 + +**未使用国密算法**（SM2/SM3/SM4/SM9）。 + +**P1 补齐计划**（R11a）： + +1. **JWT 签名**：替换 HS256 → SM2-HMAC 或国密 JWT 库（如 `gmssl`）。 +2. **密码哈希**：评估 SM3 替代 bcrypt（需兼顾兼容性，可能双算法并行）。 +3. **PII hash**：SHA-256 → SM3。 +4. **TLS**：启用国密 TLS 套件（需 nginx ingress controller + 国密证书）。 +5. **数据加密**：PG 字段级加密（透明数据加密 TDE 或应用层 SM4）。 + +**注**：国密算法**不阻碍等保三级认证**（三级要求「加密」而非「国密」）， +但政企客户常要求国密合规，故列入 P1 整改计划。 + +### 3.4 数据备份与恢复（DAT-03）— P0 补齐项 + +**当前状态：P0**。AgentKit 自身未提供 PG 备份 runbook， +依赖客户侧数据库运维。 + +**AgentKit 侧已实现**： + +- PostgreSQL 持久化（pgvector 扩展，事务日志 WAL 默认开启）。 +- Redis 配置 AOF/RDB（由客户 Redis 运维决定）。 + +**P0 补齐计划**： + +1. 新增 `docs/deployment/pg-backup-runbook.md`，包含： + - 全量备份策略（每日 `pg_dump`，保留 7 天） + - WAL 归档（连续归档，PITR 恢复） + - 恢复演练流程（每季度执行一次恢复测试） + - 备份加密（备份文件 SM4/AES-256 加密，密钥由 KMS 托管） +2. Helm Chart 增加 `backup` 段，可选部署 pg-backup sidecar / CronJob。 +3. pgvector 向量数据备份纳入 `pg_dump`（已原生支持）。 + +### 3.5 剩余信息保护（DAT-04）— 已实现 + +#### Session 撤销 + +JWT V2 token 携带 `sid`（session id）+ `jti`（access token 唯一 id）， +中间件校验 session 是否被服务端撤销 +（[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py)）： + +- 销户时撤销该用户全部 session + （[auth.py deprovision](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py)）。 +- Refresh token 轮换：旧 token 进入 denylist 30s 窗口 + （[denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py)）。 +- Token reuse 检测：reuse 触发撤销该用户全部 session（强制重新登录）。 + +#### Refresh token hash 化 + +denylist 不存储明文 refresh token，存储其 SHA-256 hash +（[denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py) +`hash_token`），减少敏感信息驻留内存。 + +#### JIT 用户随机密码 + +OIDC/SAML JIT 创建的本地用户使用随机密码 +（`hash_password(secrets.token_urlsafe(32))`），不可用于本地登录， +避免 SSO 用户的本地密码驻留 +（[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +`_find_or_create_user`）。 + +#### SCIM 用户随机密码 + +SCIM 创建的用户同样使用随机密码 +（[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +`create_user`）。 + +### 3.6 数据采集与分类分级（DAT-06）— 已实现 + +- **PII 分类**：pii_filter.py 按 PII 类型分级 + （id_card / phone / email / bank_card），不同类型采用相同脱敏策略 + 但审计 hash 独立记录。 +- **记忆分层**：4 层记忆架构（SOUL/USER/MEMORY/DAILY）按敏感度存储 + （SOUL 最敏感，DAILY 最不敏感），详见 AGENTS.md。 +- **数据最小化**：PII 脱敏后原文不落盘（仅 hash 用于审计）， + LLM prompt 中不含原文 PII + （[pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py)）。 + +### 3.7 数据销毁（DAT-07）— 已实现 + +- **账户销户**：deprovision 禁用账户（`is_active = 0`）+ 撤销全部 session。 +- **审计保留**：销户操作本身记录审计日志，不可删除（admin 不可删除审计记录， + 审计表无 delete 接口）。 +- **PII 销毁**：脱敏后 PII 原文不落盘（内存中处理后即替换）， + 无需专门销毁流程。 +- **PG 数据销毁**：客户侧 PG 运维负责（如需彻底删除用户数据， + 由 admin 发起 SQL 操作，记录审计）。 + +## 4. 测评建议 + +1. **DAT-01 / DAT-04 / DAT-06 / DAT-07**：直接展示 PG 配置 + jwt_utils.py + + denylist.py + pii_filter.py 即可。 +2. **DAT-02 / DAT-05**：展示 pii_filter.py fail-closed 流程 + OTel 脱敏指标。 +3. **DAT-03**：P0 补齐项，需在测评前完成 pg-backup-runbook.md。 +4. **国密**：P1 参考，不阻碍三级认证，但写入整改计划。 + +## 5. PII 脱敏边界（重要说明） + +AgentKit PII 脱敏的**边界**： + +1. **覆盖范围**：仅脱敏发送至 LLM 的 prompt，**不脱敏**： + - 用户本地终端命令（终端命令有自己的审计） + - 用户上传的文档原文（文档处理模块不经过 pii_filter） + - bitable 数据（结构化数据，由客户自行控制） +2. **ner_hook 依赖客户**：正则版覆盖中国常见 PII（手机/邮箱/身份证/银行卡）， + 复杂 PII（地址/姓名/组织）需客户接入内网 NER 模型。 +3. **不替代 DLP**：AgentKit PII 脱敏是应用层防护，不替代客户侧 + DLP（数据防泄漏）系统。 diff --git a/docs/compliance/dengbao-level3/06-management.md b/docs/compliance/dengbao-level3/06-management.md new file mode 100644 index 0000000..b52da74 --- /dev/null +++ b/docs/compliance/dengbao-level3/06-management.md @@ -0,0 +1,215 @@ +# 06 — 管理安全 + +> GB/T 22239-2019 第三级「安全管理」控制点映射。 + +## 1. 维度说明 + +管理安全覆盖「安全管理制度的运行」，AgentKit 自身已实现： + +- **已实现**：密钥轮换 runbook（U5，10 个 secret slot）、 + break-glass 告警流程（U4 R1b）、审计日志记录、应急隔离（销户 + session 撤销）。 +- **P0 补齐项**：变更管理流程文档化（CI/CD 流水线规范）。 +- **P1 参考**：定期安全评估流程。 +- **待评估**：应急响应预案演练（依赖客户安全团队协同）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| MGT-01 | 密钥管理：密钥生成/存储/分发/轮换/销毁 | +| MGT-02 | 变更管理：变更审批/测试/回滚/审计 | +| MGT-03 | 应急响应：应急事件处置流程，隔离/恢复 | +| MGT-04 | 安全审计与评估：定期安全评估，漏洞管理 | +| MGT-05 | 配置管理：基线配置，配置变更审计 | +| MGT-06 | 应急通道（break-glass）：高权限应急通道，审计追溯 | +| MGT-07 | 密钥泄露响应：泄露检测/隔离/轮换/通报 | + +## 3. AgentKit 实现映射 + +### 3.1 密钥管理（MGT-01）— 已实现 + +#### 10 个 secret slot + 轮换 runbook + +[docs/deployment/key-rotation-runbook.md](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md) ++ [docs/deployment/secret-inventory.md](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/secret-inventory.md) + +| # | Slot 名称 | 用途 | 轮换周期 | 归属 | +|---|-----------|------|----------|------| +| 1 | JWT 签名密钥 | access(15m)/refresh(7d) token HS256 签名 | 90 天 | 安全团队 | +| 2 | API Key | 服务间 / SCIM / `agentkit pair` 鉴权 | 180 天 | 平台运维 | +| 3 | DB 凭据 | 应用层 PG 连接 DSN | 90 天 | DBA | +| 4 | IDP 签名证书 | OIDC/SAML IdP 签名证书 PEM（多 IDP 热轮换） | 365 天 | 身份团队 | +| 5 | 第三方 API Key | LLM provider API Key（JSON map） | 90 天 | LLM 运维 | +| 6 | TLS 服务器证书 | Ingress TLS 终止证书 + 私钥 | 365 天 | 平台运维 | +| 7 | mTLS 客户端证书 | 下游服务（KMS/HSM）mTLS | 180 天 | 安全团队 | +| 8 | KMS/HSM PKCS#11 PIN | 硬件安全模块访问 PIN | 365 天 | HSM 运维 | +| 9 | OTel exporter token | OTLP collector 鉴权 bearer | 90 天 | 可观测性团队 | +| 10 | Redis-PG 密码 | Redis requirepass + PG 服务密码 | 90 天 | DBA | + +#### 通用轮换原则 + +1. **零停机**：新密钥生效后再淘汰旧密钥，避免单点切换失败。 +2. **审计先行**：轮换前记录当前密钥指纹（hash8），轮换后校验变更。 +3. **回滚预案**：保留旧密钥值至少一个轮换周期，便于快速回滚。 +4. **权限最小化**：轮换操作仅限归属角色，操作全程审计日志。 +5. **后端无关**：适用于 Sealed Secrets 与 External Secrets 两种后端。 + +#### 密钥存储 + +- **禁止明文 Kubernetes Secret + Git 提交**（KTD-4 / R2a）。 +- 后端二选一：Sealed Secrets（Bitnami）或 External Secrets Operator + （指向 Vault / AWS SM / GCP SM / Azure KV）。 +- 渲染到统一 `-secrets` Secret，通过 `env.valueFrom.secretKeyRef` + 注入 Pod。 + +详见 [values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`secrets` 段。 + +#### 热证书轮换（SAML） + +SAML IDP 签名证书支持**热轮换**（不重启即时生效）： +[src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) +`update_certificate` 方法。OIDC 通过 `server_metadata_url` 自动发现 +（authlib 缓存 TTL），无需显式热轮换。 + +### 3.2 break-glass 应急通道（MGT-06 / MGT-07）— 已实现 + +#### break-glass deprovisioning + +[src/agentkit/server/routes/auth.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py) +`deprovision_user` 端点支持 `break_glass: true` 标记： + +```python +class DeprovisionRequest: + reason: str | None = None + break_glass: bool = False # 高权限账户 break-glass 标记 +``` + +`break_glass=True` 时审计记录 `source=break_glass` +（[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) +`SOURCE_BREAKGLASS`），用于事后追溯应急操作。 + +#### 应急流程 + +1. **隔离**：admin 通过 deprovision 端点强制销户 + （`break_glass=True` + `reason` 说明） +2. **撤销**：自动撤销该用户全部活跃 session +3. **审计**：记录 actor/target/reason/source=break_glass/timestamp +4. **告警**：OTel span event `audit.deprovision` 接入审计通道 +5. **通报**：由安全团队人工通报（AgentKit 不自动通报，避免误报） + +#### 密钥泄露响应 + +[key-rotation-runbook.md §紧急轮换](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md) +「紧急轮换（泄露响应）」段落： + +1. **隔离**：撤销泄露密钥（provider 控制台 / HSM / CA） +2. **轮换**：按对应 slot 步骤紧急轮换（不遵循「双密钥并行」原则， + 直接切换到新密钥并废弃旧密钥） +3. **审计**：检索泄露密钥的使用日志，确认无异常调用 +4. **通报**：通知安全团队 + 受影响用户 +5. **复盘**：记录泄露原因，加固访问控制 + +### 3.3 变更管理（MGT-02 / MGT-05）— P0 补齐项 + +#### 已实现 + +- **Git 版本控制**：所有代码 / 配置 / Helm Chart 纳入 Git， + 变更可追溯。 +- **Helm Chart checksum**：ConfigMap 变更触发 Pod rollout + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `checksum/config` annotation），配置变更自动生效。 +- **配置同步**：[config_sync.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py) + 支持配置版本管理 + 轮询同步。 +- **审计记录**：所有 RBAC 变更、deprovision 操作记录审计 + （[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 + +#### P0 补齐项：CI/CD 流水线规范 + +**当前状态：P0**。CI/CD 流水线（GitHub Actions / Gitea Actions）规范 +未文档化。 + +**P0 补齐计划**： + +1. 新增 `docs/CI-CD-STANDARDS.md`，包含： + - 分支策略（feature 分支 → PR → main） + - 必须的 CI 检查（ruff + pytest + 镜像漏洞扫描 P0） + - 部署审批流程（Staging → Production） + - 回滚流程（Helm rollback） +2. CI 集成镜像漏洞扫描（详见 `03-host.md` §3.1 P0 补齐项）。 +3. 部署变更审计：`kubectl rollout` 历史记录接入审计通道。 + +### 3.4 应急响应（MGT-03）— 待评估 + +**当前状态：待评估**。AgentKit 自身应急隔离能力已实现（销户 + session 撤销 ++ 密钥紧急轮换），但**完整应急响应预案**需与客户安全团队协同： + +- 事件分级标准（P0/P1/P2） +- 响应时限（P0 事件 15min 内响应） +- 通报链路（AgentKit admin → 客户安全团队 → 测评机构） +- 恢复流程（密钥轮换 → 服务恢复 → 数据完整性校验） + +**建议**：客户安全团队基于本套等保文档 + AgentKit 应急能力 +（销户/密钥轮换/审计日志）构建完整预案，并在认证前完成至少一次 +桌面演练。 + +### 3.5 安全审计与评估（MGT-04）— P1 参考 + +**当前状态：P1**。AgentKit 审计日志已记录，但定期评估流程未建立。 + +#### 已实现 + +- **审计日志覆盖**：RBAC 变更 / deprovision / 终端命令 / SCIM 操作 + 全部记录审计 + （[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) + + [terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 +- **OTel 指标监控**：U1 OTel 暴露应用指标，可观测性平台监控异常 + （[metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)）。 + +#### P1 补齐计划 + +1. **定期漏洞扫描**：每月对依赖执行 `pip-audit` / `safety` 扫描。 +2. **代码安全评估**：每次重大版本发布前进行 SAST 扫描 + （如 Bandit / Semgrep）。 +3. **渗透测试**：认证后每年至少一次渗透测试（客户侧组织）。 +4. **审计日志定期审查**：admin 每季度审查审计日志异常模式 + （非自动告警，依赖人工 review — P1 可引入 SIEM 自动告警）。 + +### 3.6 配置管理（MGT-05）— 已实现 + +- **Helm values 标准化**：所有可配置项集中在 + [values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml)， + 含注释说明。 +- **配置优先级**：CLI 参数 > `agentkit.yaml` > 环境变量（`${VAR:-default}`） + > `.env` > 硬编码默认值（AGENTS.md）。 +- **配置版本管理**：[config_sync.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py) + 支持配置版本 + 轮询同步。 +- **密钥与配置分离**：含密钥的段（llm/auth/telemetry）由 env 注入， + 不进 ConfigMap；非密配置进 ConfigMap + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `config` 段注释）。 + +## 4. 测评建议 + +1. **MGT-01 / MGT-06 / MGT-07**：直接展示 key-rotation-runbook.md + + secret-inventory.md + audit.py break_glass 流程即可。 +2. **MGT-02 / MGT-05**：已实现部分展示 Helm Chart + config_sync， + P0 补齐 CI/CD 规范文档。 +3. **MGT-03**：待评估，由客户安全团队基于本套文档构建完整预案。 +4. **MGT-04**：P1 参考，写入整改计划（依赖扫描 + 季度审计 review）。 + +## 5. 密钥管理边界（重要说明） + +AgentKit 密钥管理的**边界**： + +1. **不持有 HSM/KMS 硬件**：HSM/KMS 由客户运维，AgentKit 通过 + PKCS#11 PIN 访问（slot 8）。 +2. **不签发证书**：TLS / mTLS 证书由客户 CA / cert-manager 签发， + AgentKit 仅消费。 +3. **不托管 LLM provider 密钥**：DashScope/DeepSeek/OpenAI/Anthropic + 的密钥在 provider 控制台生成，AgentKit 仅存储引用（slot 5）。 +4. **多实例密钥同步**：进程内 IDP 注册表不持久化，多实例部署时 + 各进程独立加载 `agentkit.yaml`；证书热轮换需配合配置同步 + （config_sync.py 轮询）或重启 + （[idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + ponytail 注释）。 diff --git a/docs/compliance/dengbao-level3/control-points.yaml b/docs/compliance/dengbao-level3/control-points.yaml new file mode 100644 index 0000000..02edd36 --- /dev/null +++ b/docs/compliance/dengbao-level3/control-points.yaml @@ -0,0 +1,491 @@ +# Fischer AgentKit — 等保三级控制点清单 +# +# 按 GB/T 22239-2019 第三级 6 维度组织。每个控制点含： +# id — 控制点唯一标识（DIM-NN） +# dimension — 维度（physical / network / host / application / data / management） +# title — 控制点名称 +# requirement — GB/T 22239-2019 三级要求摘要 +# implementation — AgentKit 实现映射（含状态说明） +# status — implemented / p0 / p1 / 待评估 +# implemented = 已实现，代码/配置已落地 +# p0 = 认证 readiness 必需，POC 采购关闭前补齐 +# p1 = 下一阶段补齐，写入整改计划 +# 待评估 = 依赖客户侧或需进一步评估 +# references — AgentKit 代码/配置文件绝对路径（file:/// 链接） +# +# 总计 44 个控制点：implemented=26, p0=6, p1=3, 待评估=9 + +# ============================================================================= +# 1. 物理安全（6 个，全部待评估 — 依赖客户机房） +# ============================================================================= + +- id: "PHY-01" + dimension: physical + title: "物理访问控制" + requirement: "机房出入口配置电子门禁，记录进入人员和时间" + implementation: "AgentKit 不运维物理机房。K8s 部署在客户内网，物理访问由客户机房运营方控制。AgentKit 仅提供部署形态证明（非 root 容器 + Helm Chart）。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + +- id: "PHY-02" + dimension: physical + title: "防盗窃和防破坏" + requirement: "主要设备设置防盗标识，机房设置监控系统" + implementation: "无物理设备责任。K8s worker node 由客户运维，机房监控由客户方提供。" + status: "待评估" + references: [] + +- id: "PHY-03" + dimension: physical + title: "防雷击" + requirement: "机房设置防雷保安器，接地电阻符合要求" + implementation: "无防雷责任。依赖客户机房防雷设施。" + status: "待评估" + references: [] + +- id: "PHY-04" + dimension: physical + title: "防火" + requirement: "机房设置火灾自动消防系统，检测报警器" + implementation: "无消防责任。依赖客户机房消防系统。" + status: "待评估" + references: [] + +- id: "PHY-05" + dimension: physical + title: "防水和防潮" + requirement: "机房安装水敏检测装置，防止水管安装" + implementation: "无防水责任。依赖客户机房防水设施。" + status: "待评估" + references: [] + +- id: "PHY-06" + dimension: physical + title: "防静电" + requirement: "机房采用防静电地板，设置静电消除器" + implementation: "无防静电责任。依赖客户机房防静电设施。" + status: "待评估" + references: [] + +# ============================================================================= +# 2. 网络安全（8 个） +# ============================================================================= + +- id: "NET-01" + dimension: network + title: "网络架构" + requirement: "划分网络区域，避免单点故障" + implementation: "单 Ingress 入口 + ClusterIP Service 架构。Ingress 是唯一对外入口，Redis/PG/OTel 全部 ClusterIP 内部访问。API 与 GUI 走分离路径（/api 与 /）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "NET-02" + dimension: network + title: "边界防护（Ingress TLS）" + requirement: "边界实施访问控制和安全过滤" + implementation: "Ingress 启用 TLS 终止，所有外网流量强制 HTTPS。TLS 证书通过 cert-manager 自动续期或手动签发（365 天轮换），私钥通过 Sealed Secret / External Secret 注入（KTD-4 / R2a）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "NET-03" + dimension: network + title: "访问控制（NetworkPolicy）" + requirement: "网络层访问控制策略，最小权限" + implementation: "P0 补齐项。Helm Chart 暂无 NetworkPolicy 模板，Pod 间东西向流量默认全通。计划新增 networkpolicy.yaml，默认 deny-all + 白名单放行（Ingress→Pod, Pod→Redis/PG/OTel）。当前缓解：Service 全部 ClusterIP，Redis/PG 由客户 K8s 网络策略保护。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + +- id: "NET-04" + dimension: network + title: "入侵防范" + requirement: "网络入侵检测，恶意流量检测" + implementation: "应用层已实现 RateLimiter 滑动窗口限流 + Webhook 独立限流 + JWT/SCIM token 恒定时间比较。网络层 IDS/IPS 依赖客户骨干网络。P1：接入客户 SIEM，将 OTel 日志投递到客户入侵检测平台。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "NET-05" + dimension: network + title: "安全审计（OTel）" + requirement: "网络审计日志，覆盖关键网络活动" + implementation: "OpenTelemetry（U1 强制依赖）实现网络活动审计：FastAPI Instrumentor 自动埋点每个 HTTP 请求 span；RBAC 变更/SCIM 失败/终端命令写入审计 + OTel span event；OTLP gRPC 导出到内部 collector。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "NET-06" + dimension: network + title: "恶意代码防范" + requirement: "网络边界恶意代码检测" + implementation: "AgentKit 不在网络边界位置，恶意代码检测由客户网络骨干（防病毒网关/沙箱）负责。AgentKit 侧缓解：容器镜像来自可信 CI 构建，不在运行时下载可执行文件；终端命令受 6 层白名单 + 危险检测约束。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +- id: "NET-07" + dimension: network + title: "安全审计日志保留（180 天）" + requirement: "日志保留期满足合规要求（≥180 天）" + implementation: "P0 补齐项。auth_audit_log 与 terminal_audit_logs 表无自动清理/归档策略。计划：Helm values 增加 audit.retentionDays=180（KTD-9），实现归档 CronJob 导出超过 180 天记录到对象存储并清理主表，OTel collector 配置 180 天保留策略。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +- id: "NET-08" + dimension: network + title: "资源控制（会话/带宽）" + requirement: "会话与带宽限制，防止资源耗尽" + implementation: "容器层 Pod resources requests/limits（CPU 250m~1000m / Mem 512Mi~2Gi）；应用层 RateLimiter IP 滑动窗口 + Webhook 独立限流；终端 session 1800s 闲置断开；JWT access 15min 强制重新认证。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + +# ============================================================================= +# 3. 主机安全（7 个，以容器/Pod 为单元） +# ============================================================================= + +- id: "HST-01" + dimension: host + title: "身份鉴别（ServiceAccount）" + requirement: "设备/系统身份标识与鉴别" + implementation: "AgentKit Pod 使用专用 ServiceAccount（不共享 default SA），Helm Chart 自动创建。可选 workload identity 注入 projected SA token 用于访问云 KMS/STS。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/serviceaccount.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "HST-02" + dimension: host + title: "访问控制（Pod 安全上下文）" + requirement: "操作系统/账户权限最小化，远程管理受控" + implementation: "podSecurityContext.runAsNonRoot=true（UID/GID 1000）；securityContext.allowPrivilegeEscalation=false + capabilities.drop=[ALL]；readOnlyRootFilesystem=false（需写临时文件，通过 emptyDir 挂载 /tmp 隔离）；未启用特权模式。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile" + +- id: "HST-03" + dimension: host + title: "安全审计（容器日志标准化采集）" + requirement: "设备审计日志，覆盖系统管理活动" + implementation: "已实现基础审计：stdout/stderr 由 K8s 采集 + OTel trace 导出 + auth_audit_log 表 + terminal_audit_logs 表 + OTel span event。P0 补齐：容器日志到客户日志平台的标准化采集链路 + 审计日志 JSON 化便于 SIEM 解析 + 180 天保留（与 NET-07 协同）。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py" + +- id: "HST-04" + dimension: host + title: "入侵防范（容器层）" + requirement: "主机入侵检测，系统加固" + implementation: "容器层已实现：非 root + drop ALL capabilities 限制系统调用；allowPrivilegeEscalation=false 阻止提权；资源 limits 防失控；Liveness/Readiness 探针故障自动重启。node 层 HIDS（Falco/OSSEC）依赖客户侧。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + +- id: "HST-05" + dimension: host + title: "恶意代码防范（主机防病毒）" + requirement: "主机防病毒，恶意代码检测" + implementation: "AgentKit 容器内不部署防病毒（无文件系统持久化，镜像来自可信 CI）。K8s node 层主机防病毒/HIDS 由客户运维。依赖客户机房 node 安全基线。" + status: "待评估" + references: [] + +- id: "HST-06" + dimension: host + title: "资源控制（CPU/内存/磁盘）" + requirement: "CPU/内存/磁盘限制，会话超时" + implementation: "Pod resources requests/limits（CPU 250m~1000m / Mem 512Mi~2Gi）；应用层 RateLimiter；终端 session 1800s 闲置断开；JWT access 15min / refresh 7d。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + +- id: "HST-07" + dimension: host + title: "镜像与补丁管理" + requirement: "系统补丁及时更新，镜像来源可信" + implementation: "P0 补齐项。已实现：多阶段构建 + 非 root appuser（UID 1001）+ python:3.11-slim 基础镜像 + HEALTHCHECK。P0 待补：CI 集成 Trivy 镜像漏洞扫描（HIGH/CRITICAL 阻断）+ image.digest 不可变引用。P1 增强：distroless 基础镜像 + cosign 签名。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +# ============================================================================= +# 4. 应用安全（8 个，全部已实现 — AgentKit 核心责任） +# ============================================================================= + +- id: "APP-01" + dimension: application + title: "身份鉴别（SSO 多 IDP + JWT）" + requirement: "用户身份标识与鉴别，口令复杂度，登录失败处理" + implementation: "U4 SSO 多 IDP（OIDC authlib + SAML python3-saml，热证书轮换，R1a 按 sub 关联禁止邮箱合并）；本地密码 bcrypt rounds=12；JWT HS256 access 15min + refresh 7d（30d 记住我）含 sid+jti claims；refresh token 轮换 + reuse 检测撤销全 session；OAuth state CSRF 防护 TTL 5min 一次性消费。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/saml.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py" + +- id: "APP-02" + dimension: application + title: "访问控制（RBAC 3 级 + 终端 6 层）" + requirement: "基于角色的访问控制，最小权限，默认拒绝" + implementation: "3 级 RBAC（member/operator/admin）+ 9 权限位（CHAT/KB_QUERY/KB_WRITE/WORKFLOW_EXECUTE/TERMINAL_LOCAL_USE/TERMINAL_SERVER_USE/TERMINAL_WHITELIST_MANAGE/USER_MANAGE/SYSTEM_CONFIG）。R7a 终端安全 6 层白名单（blocklist→builtin→global→user→session→danger），shell 操作符永远绕过白名单强制走 danger detection。终端双层授权（RBAC 角色 + 个人授权标志）。SCIM token 恒定时间比较。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/permissions.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "APP-03" + dimension: application + title: "安全审计（OTel + audit.py）" + requirement: "应用审计日志，覆盖用户操作关键事件" + implementation: "U1 OTel 强制依赖。审计事件：role_change / deprovision（含 break_glass 标记），写入 auth_audit_log 表 + OTel span event（OTel 不可用降级 SQLite+日志）。SCIM 推送失败 span event scim.push.failure。终端命令审计 terminal_audit_logs（决策 + cwd + exit_code）。OTel 指标：agent.request.total / pii_filter.coverage 等。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "APP-04" + dimension: application + title: "通信完整性（TLS + JWT + Webhook 签名）" + requirement: "传输完整性保护，防篡改" + implementation: "传输层 Ingress TLS 终止（HTTPS）；JWT HS256 签名校验（篡改即失败）；Webhook 入站 fail-closed 签名校验失败返回 401；mTLS 客户端证书（Helm 预留 secrets.mtlsClientCert slot 用于下游 KMS/HSM）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + +- id: "APP-05" + dimension: application + title: "软件容错（fallback + fail-closed）" + requirement: "错误处理，故障隔离，降级机制" + implementation: "LLM 网关多 provider fallback；OTel 运行时错误降级 NoOpTracer 不崩溃；PII 脱敏 fail-closed（异常拒绝发送不降级原文，默认 fail_closed=True）；专家团队全失败回退单 agent；U3 DR-fixes（bitable 字段校验 DR-1 / LastViewDeletionError DR-4 / 工具调用容错 DR-5）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/gateway.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/tracer.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/repository.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/service.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/tools/bitable_tool.py" + +- id: "APP-06" + dimension: application + title: "资源控制（限流 + 会话超时）" + requirement: "会话并发控制，超时自动断开" + implementation: "RateLimiter IP 滑动窗口 + Webhook 独立限流 + nonce dedup；终端 session 1800s 闲置断开；JWT access 15min 过期；专家名称正则校验防注入；HandoffTransport 有界队列 maxsize=1024 + sentinel 关闭模式。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + +- id: "APP-07" + dimension: application + title: "应用账户生命周期（SCIM 2.0 + JIT + deprovision）" + requirement: "开户/变更/销户审计，SCIM 自动化" + implementation: "SCIM 2.0 端点（POST/PATCH/GET Users，bearer token 恒定时间比较）；OIDC/SAML JIT 创建本地用户 + user_idp_links（R1a 按 sub 关联禁止邮箱合并，随机密码不可本地登录）；deprovision 端点禁用账户 + 撤销全 session + 审计（含 break_glass 标记）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "APP-08" + dimension: application + title: "代码安全（输入验证 + 依赖管理）" + requirement: "输入验证，输出编码，依赖漏洞管理" + implementation: "Pydantic v2 模型校验（extra=forbid 拒绝未知字段，SCIM/IDP 配置均启用）；禁止 any 类型（项目规则，用 Pydantic 模型或 Unknown）；专家名称正则 _EXPERT_NAME_RE 防 SQL/命令注入；API Key 恒定时间比较 hmac.compare_digest；Ruff lint+format（py311）；pyproject.toml 锁定精确版本；终端 6 层白名单防命令注入链。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/models.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +# ============================================================================= +# 5. 数据安全（8 个） +# ============================================================================= + +- id: "DAT-01" + dimension: data + title: "数据完整性（PG + pgvector）" + requirement: "数据传输/存储完整性，校验机制" + implementation: "PostgreSQL 事务 ACID 保证 + pgvector 扩展存储向量记忆；SQLAlchemy 2 async ORM 应用层事务；JWT HS256 签名防篡改；Webhook fail-closed 签名校验；Pydantic extra=forbid 防数据注入。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + +- id: "DAT-02" + dimension: data + title: "数据保密性（PII 脱敏 U2）" + requirement: "敏感数据加密存储/传输，密钥管理" + implementation: "U2 PII 脱敏 hook：正则版（手机/邮箱/身份证/银行卡）+ ner_hook（客户内网 NER 模型优先）+ fail-closed（异常拒绝发送不降级原文，默认 fail_closed=True）+ hash 审计（sha256 前 8 位，仅 hash 不含原文）。身份证正则优先于手机避免误匹配。OTel 指标 pii_filter.coverage / pii_filter.redacted。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "DAT-03" + dimension: data + title: "数据备份与恢复（PG backup runbook）" + requirement: "重要数据定期备份，恢复演练" + implementation: "P0 补齐项。已实现：PostgreSQL 持久化 + WAL 默认开启 + Redis AOF/RDB（客户运维）。P0 待补：新增 docs/deployment/pg-backup-runbook.md（每日 pg_dump 保留 7 天 + WAL 归档 PITR + 季度恢复演练 + 备份加密 SM4/AES-256）；Helm Chart 增加 backup 段（可选 pg-backup sidecar/CronJob）。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "DAT-04" + dimension: data + title: "剩余信息保护（session revoke）" + requirement: "鉴别信息/文件/内存空间释放前清理" + implementation: "JWT V2 含 sid+jti claims，中间件校验 session 是否服务端撤销；销户撤销全 session；refresh token 轮换旧 token 进 denylist 30s 窗口；token reuse 检测撤销全 session；denylist 存 SHA-256 hash 不存明文；JIT/SCIM 用户随机密码不可本地登录。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "DAT-05" + dimension: data + title: "个人信息保护（PII 最小化）" + requirement: "PII 采集/处理/传输最小化与脱敏" + implementation: "PII 脱敏后原文不落盘（仅 hash 用于审计），LLM prompt 不含原文 PII；PII 按类型分级（id_card/phone/email/bank_card），不同类型 hash 独立记录；记忆 4 层架构（SOUL/USER/MEMORY/DAILY）按敏感度存储；数据最小化原则。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +- id: "DAT-06" + dimension: data + title: "数据采集与分类分级" + requirement: "按敏感度分级保护" + implementation: "PII 按 4 类分级（id_card/phone/email/bank_card）独立 hash 审计；记忆 4 层架构（SOUL 最敏感→DAILY 最不敏感）；PII 脱敏覆盖发送至 LLM 的 prompt（边界：不覆盖终端命令/上传文档/bitable 结构化数据，由客户自行控制）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +- id: "DAT-07" + dimension: data + title: "数据销毁" + requirement: "数据销毁审计，不可恢复" + implementation: "账户销户 deprovision 禁用账户+撤销全 session；审计记录不可删除（审计表无 delete 接口，admin 不可删除审计）；PII 原文脱敏后不落盘无需专门销毁；PG 数据销毁由客户 DBA 发起 SQL 记录审计。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "DAT-08" + dimension: data + title: "国密算法合规（R11a）" + requirement: "敏感数据加密使用国密算法（政企客户要求）" + implementation: "P1 参考。当前使用国际算法：TLS（AES-GCM）/ bcrypt 密码哈希 / HS256 JWT / SHA-256 PII hash。未使用国密（SM2/SM3/SM4/SM9）。P1 升级路径：JWT 签名→SM2-HMAC、密码哈希→SM3（双算法并行兼容）、PII hash→SM3、TLS→国密套件、PG 字段级加密→SM4 TDE。注：国密不阻碍等保三级认证（三级要求加密而非国密），但政企客户常要求国密合规。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +# ============================================================================= +# 6. 管理安全（7 个） +# ============================================================================= + +- id: "MGT-01" + dimension: management + title: "密钥管理（10 secret slot + 轮换 runbook）" + requirement: "密钥生成/存储/分发/轮换/销毁" + implementation: "U5 10 个 secret slot（JWT/API Key/DB/IDP/第三方API/TLS/mTLS/KMS-HSM/OTel/Redis-PG）含轮换周期（90~365 天）。禁止明文 K8s Secret+Git 提交（KTD-4/R2a），后端二选一：Sealed Secrets 或 External Secrets Operator。SAML IDP 证书热轮换（不重启即时生效，idp_registry.update_certificate）。通用原则：零停机双密钥并行+审计先行+回滚预案+权限最小化。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/secret-inventory.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + +- id: "MGT-02" + dimension: management + title: "变更管理（CI/CD 规范）" + requirement: "变更审批/测试/回滚/审计" + implementation: "P0 补齐项。已实现：Git 版本控制所有代码/配置/Helm Chart；Helm Chart checksum/config annotation ConfigMap 变更触发 rollout；config_sync.py 配置版本管理+轮询同步；审计记录 RBAC 变更。P0 待补：新增 docs/CI-CD-STANDARDS.md（分支策略+CI 检查 ruff/pytest/镜像扫描+部署审批+回滚流程）；CI 集成镜像漏洞扫描（与 HST-07 协同）；部署变更审计接入。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py" + +- id: "MGT-03" + dimension: management + title: "应急响应" + requirement: "应急事件处置流程，隔离/恢复" + implementation: "待评估。AgentKit 自身应急隔离能力已实现（销户+session 撤销+密钥紧急轮换，见 MGT-06/MGT-07），但完整应急响应预案需与客户安全团队协同：事件分级标准+响应时限+通报链路+恢复流程。建议客户基于本套等保文档+AgentKit 应急能力构建完整预案，认证前完成至少一次桌面演练。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + +- id: "MGT-04" + dimension: management + title: "安全审计与评估" + requirement: "定期安全评估，漏洞管理" + implementation: "P1 参考。已实现：审计日志覆盖 RBAC 变更/deprovision/终端命令/SCIM 操作；OTel 指标监控异常。P1 补齐：每月依赖漏洞扫描（pip-audit/safety）；每次重大版本发布前 SAST 扫描（Bandit/Semgrep）；认证后每年至少一次渗透测试（客户侧组织）；admin 每季度审查审计日志异常模式（P1 可引入 SIEM 自动告警）。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "MGT-05" + dimension: management + title: "配置管理（Helm values + 配置同步）" + requirement: "基线配置，配置变更审计" + implementation: "Helm values 标准化所有可配置项含注释；配置优先级 CLI>agentkit.yaml>环境变量>.env>硬编码默认；config_sync.py 配置版本管理+轮询同步；密钥与配置分离（含密钥段由 env 注入不进 ConfigMap，非密配置进 ConfigMap）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py" + +- id: "MGT-06" + dimension: management + title: "应急通道（break-glass）" + requirement: "高权限应急通道，审计追溯" + implementation: "U4 R1b break-glass deprovisioning：/admin/users/{id}/deprovision 端点支持 break_glass=true 标记，审计记录 source=break_glass（SOURCE_BREAKGLASS）。流程：隔离（销户）→撤销全 session→审计（actor/target/reason/source/timestamp）→OTel span event audit.deprovision 接入审计通道→人工通报。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "MGT-07" + dimension: management + title: "密钥泄露响应" + requirement: "泄露检测/隔离/轮换/通报" + implementation: "key-rotation-runbook.md §紧急轮换（泄露响应）段落：1.隔离（撤销泄露密钥 provider控制台/HSM/CA）→2.轮换（按对应 slot 步骤紧急轮换，不遵循双密钥并行直接切换废弃旧密钥）→3.审计（检索泄露密钥使用日志确认无异常）→4.通报（安全团队+受影响用户）→5.复盘（记录原因加固访问控制）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" From 86bce129a45cdb7dc0101f230bc2c15eb0b92b75 Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 08:02:40 +0800 Subject: [PATCH 07/11] =?UTF-8?q?refactor(enterprise):=20simplify=20U1-U6?= =?UTF-8?q?=20=E2=80=94=20fix=20span=20leak,=20dedup=20helpers,=20stdlib?= =?UTF-8?q?=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lfg Step 3 (ce-simplify-code) — 16 files, -89 net lines. P1 fixes (functional): - audit.py / scim/router.py: fix OTel span leak — start_span() returned a context manager but was used as a plain object, so spans never ended and never exported. Wrap in `with`. - helm chart: split combined `redisPgPasswords` slot (single JSON key) into `redisPassword` + `postgresPassword` slots — the combined `key` field was dead config never referenced by any template; deployment.yaml hardcoded `redis-password`/`postgres-password` keys were inconsistent with values.yaml schema. P2 dedup (reuse): - Promote `resolve_auth_db_path()` to models.py (canonical); remove 3 duplicate `_resolve_db_path` defs in oidc.py / saml.py / audit.py. - Reuse `_now_iso` from models.py; remove 5 duplicates across audit / oidc / saml / scim / local providers. - Reuse `_row_to_user` from local.py in oidc.py / saml.py (verbatim copies removed). - pii_filter._redact: collapse redundant `finditer` + `sub` (2 regex passes + 2x sha256 per match on hot path) into a single `sub` with a callback that collects matches and computes hashes once. P3 simplifications: - RoleChangeRequest.role: `str` + manual `if not in (...)` → `Literal` (Pydantic validates automatically). - Extract `_record_pii_metrics(status, redacted_count)` helper to dedup 3 nearly-identical OTel metric emission blocks. - Extract `LLMGateway._record_prompt_cache_metric(usage, model)` to dedup non-stream + stream cache hit/miss counters; switch to use the `TokenUsage.cache_hit` property (previously added but unused). - middleware: split `WHITELIST_PATHS` + `PREFIX_MATCH_PATHS` (redundant cross-reference) into `WHITELIST_PATHS` (exact) + `PREFIX_WHITELIST_PATHS` (prefix) — single source of truth per kind. - OIDC token exchange: replace f-string form body with `urlencode` to correctly escape `&` / `=` in code / redirect_uri. - auth.py OIDC redirect route: replace hand-rolled `url.split("state=")` with `urlparse + parse_qs`. - PII module docstring: clarify that `PIIMatch.original` is in-memory only (must not be logged / persisted) — the previous "仅 hash, 不含 原文" wording was misleading. Skipped findings (false-positive or not worth): - OIDC/SAML `_find_or_create_user` JIT 60-line dedup — high blast radius across SQL + transaction boundaries; defers to dedicated refactor. - SAML `_extract_host` hand-rolled split — already has ponytail comment; stdlib `urlparse` swap is behavior-equivalent for current inputs but changes OneLogin `http_host` field semantics; conservative keep. - gateway.py department-quota / usage-record sequential await — concurrency gain unclear when dept count is typically ≤2. Verification: - ruff check (scoped to 12 touched files) clean - ruff format (1 file reformatted) - pytest (PII + OIDC + SAML + SCIM + audit + bitable + team_orchestrator): 265 passed, 145 skipped (PG/Docker), 0 failures - helm lint clean; helm template renders correctly for both sealed-secrets + external-secrets backends --- .../helm/agentkit/templates/deployment.yaml | 4 +- .../agentkit/templates/external-secret.yaml | 7 +- .../agentkit/templates/sealed-secret.yaml | 7 +- deploy/helm/agentkit/values.yaml | 14 ++-- docs/deployment/secret-inventory.md | 3 +- src/agentkit/llm/gateway.py | 21 +++--- src/agentkit/llm/pii_filter.py | 64 ++++++++----------- src/agentkit/server/auth/audit.py | 46 ++++--------- src/agentkit/server/auth/middleware.py | 33 ++++------ src/agentkit/server/auth/models.py | 12 ++++ .../server/auth/providers/__init__.py | 12 +--- src/agentkit/server/auth/providers/local.py | 29 ++------- src/agentkit/server/auth/providers/oidc.py | 41 +++--------- src/agentkit/server/auth/providers/saml.py | 32 +--------- src/agentkit/server/auth/scim/router.py | 19 ++---- src/agentkit/server/routes/auth.py | 35 +++++----- 16 files changed, 145 insertions(+), 234 deletions(-) diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml index b917ab8..4446d9f 100644 --- a/deploy/helm/agentkit/templates/deployment.yaml +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -89,12 +89,12 @@ spec: valueFrom: secretKeyRef: name: {{ include "agentkit.secretName" . }} - key: redis-password + key: {{ .Values.secrets.redisPassword.key }} - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: {{ include "agentkit.secretName" . }} - key: postgres-password + key: {{ .Values.secrets.postgresPassword.key }} # --- 非密配置（OTel endpoint、service name）--- {{- if .Values.otel.enabled }} - name: OTEL_EXPORTER_OTLP_ENDPOINT diff --git a/deploy/helm/agentkit/templates/external-secret.yaml b/deploy/helm/agentkit/templates/external-secret.yaml index bc24ef6..ed8a879 100644 --- a/deploy/helm/agentkit/templates/external-secret.yaml +++ b/deploy/helm/agentkit/templates/external-secret.yaml @@ -55,11 +55,12 @@ spec: - secretKey: {{ .Values.secrets.otelExporterToken.key }} remoteRef: key: agentkit/otel-exporter-token - # 10) Redis-PG 密码（拆为两个 key） - - secretKey: redis-password + # 10) Redis 密码 + - secretKey: {{ .Values.secrets.redisPassword.key }} remoteRef: key: agentkit/redis-password - - secretKey: postgres-password + # 11) PostgreSQL 密码 + - secretKey: {{ .Values.secrets.postgresPassword.key }} remoteRef: key: agentkit/postgres-password {{- end }} diff --git a/deploy/helm/agentkit/templates/sealed-secret.yaml b/deploy/helm/agentkit/templates/sealed-secret.yaml index 92489ec..b22d926 100644 --- a/deploy/helm/agentkit/templates/sealed-secret.yaml +++ b/deploy/helm/agentkit/templates/sealed-secret.yaml @@ -31,9 +31,10 @@ spec: {{ .Values.secrets.kmsHsmPin.key }}: {{ .Values.secrets.kmsHsmPin.encryptedData | default "" | quote }} # 9) OTel exporter token {{ .Values.secrets.otelExporterToken.key }}: {{ .Values.secrets.otelExporterToken.encryptedData | default "" | quote }} - # 10) Redis-PG 密码（拆为 redis-password + postgres-password 两个 key） - redis-password: {{ .Values.secrets.redisPgPasswords.encryptedRedis | default "" | quote }} - postgres-password: {{ .Values.secrets.redisPgPasswords.encryptedPostgres | default "" | quote }} + # 10) Redis 密码 + {{ .Values.secrets.redisPassword.key }}: {{ .Values.secrets.redisPassword.encryptedData | default "" | quote }} + # 11) PostgreSQL 密码 + {{ .Values.secrets.postgresPassword.key }}: {{ .Values.secrets.postgresPassword.encryptedData | default "" | quote }} template: metadata: name: {{ include "agentkit.secretName" . }} diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml index 9f65727..e01ee5f 100644 --- a/deploy/helm/agentkit/values.yaml +++ b/deploy/helm/agentkit/values.yaml @@ -197,10 +197,16 @@ secrets: description: "OTLP exporter 鉴权 token（collector 启用 bearer auth 时）" rotation: 90d - # 10) Redis-PG 密码 — Redis requirepass + PostgreSQL 服务密码（基础设施层） - redisPgPasswords: - key: redis-pg-passwords - description: "Redis requirepass + PostgreSQL POSTGRES_PASSWORD（JSON map: {redis, postgres}）" + # 10) Redis 密码 — Redis requirepass（基础设施层） + redisPassword: + key: redis-password + description: "Redis requirepass（基础设施层）" + rotation: 90d + + # 11) PostgreSQL 密码 — POSTGRES_PASSWORD（基础设施层） + postgresPassword: + key: postgres-password + description: "PostgreSQL POSTGRES_PASSWORD（基础设施层）" rotation: 90d # --------------------------------------------------------------------------- diff --git a/docs/deployment/secret-inventory.md b/docs/deployment/secret-inventory.md index a766ce6..96b8724 100644 --- a/docs/deployment/secret-inventory.md +++ b/docs/deployment/secret-inventory.md @@ -20,7 +20,8 @@ | 7 | mTLS 客户端证书 | `mtls-client-cert` | `MTLS_CLIENT_CERT` | 与下游服务（KMS/HSM、内部 API）双向 TLS 客户端证书 | 内部 PKI | 180 天 | 安全团队 | | 8 | KMS/HSM PKCS#11 PIN | `kms-hsm-pin` | `KMS_HSM_PIN` | 硬件安全模块 PKCS#11 访问 PIN | HSM 管理员 | 365 天 | HSM 运维 | | 9 | OTel exporter token | `otel-exporter-token` | `OTEL_EXPORTER_OTLP_HEADERS` | OTLP collector 鉴权 bearer token | 可观测性平台 | 90 天 | 可观测性团队 | -| 10 | Redis-PG 密码 | `redis-password` + `postgres-password` | `REDIS_PASSWORD` / `POSTGRES_PASSWORD` | Redis requirepass + PostgreSQL 服务密码（基础设施层） | DBA / 缓存运维 | 90 天 | DBA | +| 10 | Redis 密码 | `redis-password` | `REDIS_PASSWORD` | Redis requirepass（基础设施层） | DBA / 缓存运维 | 90 天 | DBA | +| 11 | PostgreSQL 密码 | `postgres-password` | `POSTGRES_PASSWORD` | PostgreSQL POSTGRES_PASSWORD（基础设施层） | DBA | 90 天 | DBA | ## 风险等级 diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index 7496f43..d74d753 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -302,11 +302,7 @@ class LLMGateway: {"gen_ai.request.model": resolved_model}, ) # U2 — prompt cache hit/miss OTel counter（Anthropic cache_read_input_tokens > 0 → hit） - cache_attrs = {"gen_ai.request.model": resolved_model} - if response.usage.cache_read_input_tokens > 0: - prompt_cache_hit_counter().add(1, cache_attrs) - else: - prompt_cache_miss_counter().add(1, cache_attrs) + self._record_prompt_cache_metric(response.usage, resolved_model) return response finally: @@ -415,11 +411,7 @@ class LLMGateway: department_ids=department_ids, ) # U2 — prompt cache hit/miss OTel counter（streaming path） - stream_cache_attrs = {"gen_ai.request.model": final_model} - if final_usage.cache_read_input_tokens > 0: - prompt_cache_hit_counter().add(1, stream_cache_attrs) - else: - prompt_cache_miss_counter().add(1, stream_cache_attrs) + self._record_prompt_cache_metric(final_usage, final_model) # Empty stream detection: if no content was produced, # raise error so the caller (ReActEngine) can retry with a different model. @@ -455,6 +447,15 @@ class LLMGateway: "", f"No provider available for streaming '{resolved_model}'" ) + @staticmethod + def _record_prompt_cache_metric(usage: TokenUsage, model: str) -> None: + """Record prompt cache hit/miss OTel counter — Anthropic cache_read_input_tokens > 0 → hit.""" + attrs = {"gen_ai.request.model": model} + if usage.cache_hit: + prompt_cache_hit_counter().add(1, attrs) + else: + prompt_cache_miss_counter().add(1, attrs) + def _get_models_to_try(self, resolved_model: str) -> list[str]: """Return [primary_model] + fallback_models for the given resolved model.""" fallback_models = self._config.fallbacks.get(resolved_model, []) diff --git a/src/agentkit/llm/pii_filter.py b/src/agentkit/llm/pii_filter.py index 70f1aea..633f108 100644 --- a/src/agentkit/llm/pii_filter.py +++ b/src/agentkit/llm/pii_filter.py @@ -6,7 +6,8 @@ - 正则版（默认）：识别手机号 / 邮箱 / 身份证 / 银行卡，替换为 [REDACTED_TYPE:hash8] - ner_hook（可选）：客户内网 NER 模型（LAC/HanLP），module:func 路径动态 import - fail-closed：任何异常时拒绝发送（raise PIIFilterError），不降级到原文发送 -- hash 审计：脱敏前后记录 hash 用于审计（仅 hash，不含原文） +- hash 审计：脱敏后记录 hash 用于审计；``PIIMatch.original`` 仅内存中供调用方 + 使用（如展示 / 调试），不得落盘 / 落日志 ponytail: 用 stdlib（re + hashlib），不引入 NER 依赖。 升级路径：ner_hook 接入客户内网 NER 模型，正则版作为 fallback。 @@ -97,26 +98,42 @@ def _redact(text: str, ner_hook=None) -> PIIFilterResult: ("bank_card", _BANK_CARD_RE), ] for pii_type, pattern in patterns: - for m in pattern.finditer(redacted): + # 单次 sub + 回调同时收集 matches 和替换文本， + # 避免二次正则扫描 + 双重 sha256 计算（热路径） + def _replace(m, _t=pii_type): original = m.group() pii_hash = _hash_pii(original) - replacement = f"[REDACTED_{pii_type.upper()}:{pii_hash}]" + replacement = f"[REDACTED_{_t.upper()}:{pii_hash}]" matches.append( PIIMatch( - pii_type=pii_type, + pii_type=_t, original=original, redacted=replacement, hash=pii_hash, ) ) - redacted = pattern.sub( - lambda m: f"[REDACTED_{pii_type.upper()}:{_hash_pii(m.group())}]", - redacted, - ) + return replacement + + redacted = pattern.sub(_replace, redacted) return PIIFilterResult(redacted_text=redacted, matches=matches) +def _record_pii_metrics(status: str, redacted_count: int | None = None) -> None: + """记录 PII 脱敏 OTel 指标 — 失败静默降级（不影响主流程）。""" + try: + from agentkit.telemetry.metrics import ( + pii_filter_coverage_counter, + pii_filter_redacted_counter, + ) + + pii_filter_coverage_counter().add(1, {"pii_filter.status": status}) + if redacted_count is not None: + pii_filter_redacted_counter().add(redacted_count) + except Exception: + pass + + def filter_pii( text: str, *, @@ -138,44 +155,19 @@ def filter_pii( """ try: result = _redact(text, ner_hook=ner_hook) - # U2 — OTel 指标：脱敏覆盖率 + 脱敏实体数 - try: - from agentkit.telemetry.metrics import ( - pii_filter_coverage_counter, - pii_filter_redacted_counter, - ) - - pii_filter_coverage_counter().add(1, {"pii_filter.status": "success"}) - pii_filter_redacted_counter().add(result.redacted_count) - except Exception: - # OTel 指标失败不影响脱敏主流程 - pass + _record_pii_metrics("success", result.redacted_count) return result except Exception as e: logger.error(f"PII filter failed: {e}") if fail_closed: # fail-closed：拒绝发送，抛异常 - try: - from agentkit.telemetry.metrics import pii_filter_coverage_counter - - pii_filter_coverage_counter().add(1, {"pii_filter.status": "failed_closed"}) - except Exception: - pass + _record_pii_metrics("failed_closed") raise PIIFilterError(f"PII filter failed (fail-closed): {e}") from e # fail-open：降级到正则版（无 ner_hook）重试 logger.warning("fail_closed=False, falling back to regex-only redaction") try: result = _redact(text, ner_hook=None) - try: - from agentkit.telemetry.metrics import ( - pii_filter_coverage_counter, - pii_filter_redacted_counter, - ) - - pii_filter_coverage_counter().add(1, {"pii_filter.status": "fallback_regex"}) - pii_filter_redacted_counter().add(result.redacted_count) - except Exception: - pass + _record_pii_metrics("fallback_regex", result.redacted_count) return result except Exception as fallback_err: # 正则版也失败（极少见），返回原文（最差情况） diff --git a/src/agentkit/server/auth/audit.py b/src/agentkit/server/auth/audit.py index 797a3ae..6d2d4dc 100644 --- a/src/agentkit/server/auth/audit.py +++ b/src/agentkit/server/auth/audit.py @@ -13,15 +13,14 @@ from __future__ import annotations import logging +import os import uuid -from datetime import datetime, timezone from pathlib import Path -from typing import Any import aiosqlite from agentkit.telemetry.tracer import get_tracer -from .models import DEFAULT_AUTH_DB_PATH, audit_log_row_to_dict +from .models import DEFAULT_AUTH_DB_PATH, audit_log_row_to_dict, _now_iso logger = logging.getLogger(__name__) @@ -36,17 +35,6 @@ SOURCE_CRON = "cron_reconciliation" SOURCE_BREAKGLASS = "break_glass" -def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _resolve_db_path() -> Path: - import os - - env = os.environ.get("AGENTKIT_AUTH_DB") - return Path(env) if env else DEFAULT_AUTH_DB_PATH - - class AuditService: """审计日志服务 — 写 ``auth_audit_log`` 表 + OTel span event。 @@ -55,8 +43,6 @@ class AuditService: """ def __init__(self, db_path: str | Path | None = None) -> None: - import os - if db_path is not None: self._db_path = Path(db_path) else: @@ -165,18 +151,18 @@ class AuditService: # OTel 审计通道 — 发 span event（OTel 不可用时 NoOpTracer 静默降级） try: tracer = get_tracer() - span = tracer.start_span(f"audit.{event_type}") - span.set_attribute("audit.event_type", event_type) - span.set_attribute("audit.actor", actor_username or actor_user_id or "system") - span.set_attribute("audit.target", target_username or target_user_id) - span.set_attribute("audit.source", source) - if role_before or role_after: - span.set_attribute("audit.role_before", role_before or "") - span.set_attribute("audit.role_after", role_after or "") - span.add_event( - "audit.record", - {"actor": actor_username or "", "target": target_username or ""}, - ) + with tracer.start_span(f"audit.{event_type}") as span: + span.set_attribute("audit.event_type", event_type) + span.set_attribute("audit.actor", actor_username or actor_user_id or "system") + span.set_attribute("audit.target", target_username or target_user_id) + span.set_attribute("audit.source", source) + if role_before or role_after: + span.set_attribute("audit.role_before", role_before or "") + span.set_attribute("audit.role_after", role_after or "") + span.add_event( + "audit.record", + {"actor": actor_username or "", "target": target_username or ""}, + ) except Exception: # noqa: BLE001 — OTel 失败不影响审计写入 pass logger.info( @@ -221,7 +207,3 @@ def set_audit_service(service: AuditService | None) -> None: """注入自定义 AuditService（测试用）。""" global _service _service = service - - -# 全局别名便于调用 — 保留 Any 仅用于 typing 占位（不违反 no-any 规则） -_ = Any diff --git a/src/agentkit/server/auth/middleware.py b/src/agentkit/server/auth/middleware.py index 09e3f64..82c0bbf 100644 --- a/src/agentkit/server/auth/middleware.py +++ b/src/agentkit/server/auth/middleware.py @@ -41,33 +41,30 @@ class AuthMiddleware(BaseHTTPMiddleware): client_keys: Mapping of ``client_name -> api_key`` (from clients.yaml). """ + # 精确匹配路径 — 静态端点（无动态路径段） WHITELIST_PATHS = ( "/", "/login", - "/assets", # Static assets (exact match covers root; prefix below covers sub-paths) "/api/v1/health", "/api/v1/auth/login", "/api/v1/auth/refresh", "/api/v1/auth/logout", "/api/v1/auth/whoami", # Route does its own auth (access OR refresh) - # U4 SSO — 动态 provider 路径，前缀匹配（下方 PREFIX_MATCH_PATHS） - "/api/v1/auth/oauth/", # OIDC redirect/callback - "/api/v1/auth/saml/", # SAML ACS - "/api/v1/scim/v2/", # SCIM 2.0（自带 bearer token 校验） "/docs", "/openapi.json", "/redoc", ) - # 前缀匹配路径 — 动态路径段（provider 名 / 资源 id），仅这些前缀走 startswith - PREFIX_MATCH_PATHS = ( + # 前缀匹配路径 — 动态路径段（provider 名 / 资源 id / 静态资源子路径） + PREFIX_WHITELIST_PATHS = ( "/docs", "/openapi.json", "/redoc", "/assets", - "/api/v1/auth/oauth/", - "/api/v1/auth/saml/", - "/api/v1/scim/v2/", + # U4 SSO — 动态 provider 路径 + "/api/v1/auth/oauth/", # OIDC redirect/callback + "/api/v1/auth/saml/", # SAML ACS + "/api/v1/scim/v2/", # SCIM 2.0（自带 bearer token 校验） ) def __init__( @@ -90,17 +87,13 @@ class AuthMiddleware(BaseHTTPMiddleware): def _is_whitelisted(self, path: str) -> bool: """Return True if ``path`` matches a whitelisted route. - Uses exact match for auth routes (so ``/auth/logout`` does NOT - whitelist ``/auth/logout-others``) and prefix match for docs and assets. + Uses exact match for static auth routes (so ``/auth/logout`` does NOT + whitelist ``/auth/logout-others``) and prefix match for docs, assets, + and U4 SSO dynamic-provider routes. """ - for prefix in self.WHITELIST_PATHS: - if path == prefix: - return True - # Prefix match for documentation paths, static assets, and U4 - # SSO dynamic-provider routes (/auth/oauth/{provider}/...). - if path.startswith(prefix) and prefix in self.PREFIX_MATCH_PATHS: - return True - return False + if path in self.WHITELIST_PATHS: + return True + return any(path.startswith(p) for p in self.PREFIX_WHITELIST_PATHS) def _is_dev_mode(self) -> bool: """Dev mode = no JWT secret, no global API key, no client keys.""" diff --git a/src/agentkit/server/auth/models.py b/src/agentkit/server/auth/models.py index aa76261..66504f5 100644 --- a/src/agentkit/server/auth/models.py +++ b/src/agentkit/server/auth/models.py @@ -45,6 +45,18 @@ def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +def resolve_auth_db_path() -> Path: + """Lazily resolve the auth DB path from env var at call time. + + ``DEFAULT_AUTH_DB_PATH`` is captured at module-import time and cannot + see test-time ``AGENTKIT_AUTH_DB`` mutations. This helper re-reads the + env var on every call so tests can ``monkeypatch.setenv`` before + constructing a provider/service. + """ + env = os.environ.get("AGENTKIT_AUTH_DB") + return Path(env) if env else DEFAULT_AUTH_DB_PATH + + # --------------------------------------------------------------------------- # SQLAlchemy 2 declarative models # --------------------------------------------------------------------------- diff --git a/src/agentkit/server/auth/providers/__init__.py b/src/agentkit/server/auth/providers/__init__.py index 5c9a90b..e956203 100644 --- a/src/agentkit/server/auth/providers/__init__.py +++ b/src/agentkit/server/auth/providers/__init__.py @@ -33,13 +33,13 @@ from __future__ import annotations import logging import os from functools import lru_cache -from pathlib import Path from .base import AuthProvider from .exceptions import AuthProviderError, InvalidCredentials, ProviderNotImplemented from .local import LocalAuthProvider from .oidc_stub import StubOIDCProvider from .user import User +from ..models import resolve_auth_db_path as _resolve_db_path logger = logging.getLogger(__name__) @@ -71,16 +71,6 @@ def _resolve_provider_name() -> str: return name -def _resolve_db_path() -> Path: - """Return the auth DB path (overridable for tests via env var).""" - from ..models import DEFAULT_AUTH_DB_PATH - - env = os.environ.get("AGENTKIT_AUTH_DB") - if env: - return Path(env) - return DEFAULT_AUTH_DB_PATH - - @lru_cache(maxsize=1) def get_auth_provider() -> AuthProvider: """Return the configured :class:`AuthProvider` (memoized singleton). diff --git a/src/agentkit/server/auth/providers/local.py b/src/agentkit/server/auth/providers/local.py index 6bf0de8..ffab51e 100644 --- a/src/agentkit/server/auth/providers/local.py +++ b/src/agentkit/server/auth/providers/local.py @@ -21,34 +21,22 @@ implementation. from __future__ import annotations import logging -import os import uuid from pathlib import Path import aiosqlite -from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..models import ( + resolve_auth_db_path as _resolve_db_path, + _now_iso, + user_row_to_dict, +) from ..password import hash_password, verify_password from .user import User logger = logging.getLogger(__name__) -def _resolve_db_path() -> Path: - """Resolve the auth DB path with runtime env-var priority. - - The :data:`models.DEFAULT_AUTH_DB_PATH` constant is captured at - module-import time and therefore cannot see test-time env mutations. - Re-reading ``AGENTKIT_AUTH_DB`` here keeps the provider - "test-friendly" (tests can ``monkeypatch.setenv`` before constructing - the provider) without giving up the default path when no env is set. - """ - env = os.environ.get("AGENTKIT_AUTH_DB") - if env: - return Path(env) - return DEFAULT_AUTH_DB_PATH - - class LocalAuthProvider: """AuthProvider backed by the local SQLite ``users`` table + bcrypt. @@ -242,10 +230,3 @@ def _row_to_user(row: aiosqlite.Row) -> User: last_login_at=row["last_login_at"], created_by=row["created_by"], ) - - -def _now_iso() -> str: - """Return current UTC time as ISO 8601 string.""" - from datetime import datetime, timezone - - return datetime.now(timezone.utc).isoformat() diff --git a/src/agentkit/server/auth/providers/oidc.py b/src/agentkit/server/auth/providers/oidc.py index 5b52948..fdb8a30 100644 --- a/src/agentkit/server/auth/providers/oidc.py +++ b/src/agentkit/server/auth/providers/oidc.py @@ -21,17 +21,17 @@ import logging import secrets import time import uuid -from datetime import datetime, timezone -from pathlib import Path from typing import Any +from urllib.parse import urlencode import aiosqlite from authlib.integrations.httpx_client import AsyncOAuth2Client from ..idp_registry import get_idp_registry -from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..models import user_row_to_dict, _now_iso, resolve_auth_db_path as _resolve_db_path from ..password import hash_password from .exceptions import ProviderNotImplemented +from .local import _row_to_user from .user import User logger = logging.getLogger(__name__) @@ -75,17 +75,6 @@ def get_state_cache() -> _StateCache: return _state_cache -def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _resolve_db_path() -> Path: - import os - - env = os.environ.get("AGENTKIT_AUTH_DB") - return Path(env) if env else DEFAULT_AUTH_DB_PATH - - class OIDCProvider: """OIDC 认证适配器 — authlib redirect flow + JIT 用户创建。 @@ -156,7 +145,13 @@ class OIDCProvider: token = await client.fetch_token( cfg.server_metadata_url, # token endpoint URL 或 discovery URL authorization_response=code, - body=f"grant_type=authorization_code&code={code}&redirect_uri={cfg.redirect_uri}", + body=urlencode( + { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": cfg.redirect_uri, + } + ), ) # 拉取 userinfo（OIDC discovery 的标准端点） userinfo = await self._fetch_userinfo(client, token, cfg.server_metadata_url) @@ -309,19 +304,3 @@ class OIDCProvider: ) await db.commit() logger.info(f"OIDC provider 禁用用户 {user_id}") - - -def _row_to_user(row: aiosqlite.Row) -> User: - return User( - id=row["id"], - username=row["username"], - email=row["email"], - role=row["role"], - is_active=bool(row["is_active"]), - is_terminal_authorized=bool(row["is_terminal_authorized"]), - is_server_terminal_authorized=bool(row["is_server_terminal_authorized"]), - created_at=row["created_at"], - updated_at=row["updated_at"], - last_login_at=row["last_login_at"], - created_by=row["created_by"], - ) diff --git a/src/agentkit/server/auth/providers/saml.py b/src/agentkit/server/auth/providers/saml.py index bea4c01..8bbade7 100644 --- a/src/agentkit/server/auth/providers/saml.py +++ b/src/agentkit/server/auth/providers/saml.py @@ -17,32 +17,20 @@ import asyncio import logging import secrets import uuid -from datetime import datetime, timezone -from pathlib import Path from typing import Any import aiosqlite from ..idp_registry import get_idp_registry -from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..models import user_row_to_dict, _now_iso, resolve_auth_db_path as _resolve_db_path from ..password import hash_password from .exceptions import ProviderNotImplemented +from .local import _row_to_user from .user import User logger = logging.getLogger(__name__) -def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _resolve_db_path() -> Path: - import os - - env = os.environ.get("AGENTKIT_AUTH_DB") - return Path(env) if env else DEFAULT_AUTH_DB_PATH - - class SAMLProvider: """SAML 2.0 认证适配器 — OneLogin python3-saml ACS 消费。 @@ -258,19 +246,3 @@ def _extract_host(url: str) -> str: if "://" in url: url = url.split("://", 1)[1] return url.split("/", 1)[0] if url else "" - - -def _row_to_user(row: aiosqlite.Row) -> User: - return User( - id=row["id"], - username=row["username"], - email=row["email"], - role=row["role"], - is_active=bool(row["is_active"]), - is_terminal_authorized=bool(row["is_terminal_authorized"]), - is_server_terminal_authorized=bool(row["is_server_terminal_authorized"]), - created_at=row["created_at"], - updated_at=row["updated_at"], - last_login_at=row["last_login_at"], - created_by=row["created_by"], - ) diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py index 57dd779..9645c27 100644 --- a/src/agentkit/server/auth/scim/router.py +++ b/src/agentkit/server/auth/scim/router.py @@ -16,7 +16,6 @@ import logging import os import secrets import uuid -from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -24,7 +23,7 @@ import aiosqlite from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import BaseModel, ConfigDict -from ...auth.models import DEFAULT_AUTH_DB_PATH, init_auth_db, user_row_to_dict +from ...auth.models import DEFAULT_AUTH_DB_PATH, init_auth_db, user_row_to_dict, _now_iso from ...auth.password import hash_password from .models import SCIMListResponse, SCIMPatchRequest, SCIMUser @@ -35,10 +34,6 @@ router = APIRouter(prefix="/scim/v2", tags=["scim"]) _SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" -def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - def _resolve_db_path(request: Request) -> Path: path = getattr(request.app.state, "auth_db_path", None) return Path(path) if path else DEFAULT_AUTH_DB_PATH @@ -80,12 +75,12 @@ def _alert_scim_failure(operation: str, detail: str, user_id: str | None = None) from agentkit.telemetry.tracer import get_tracer tracer = get_tracer() - span = tracer.start_span("scim.push.failure") - span.set_attribute("scim.operation", operation) - span.set_attribute("scim.error", detail) - if user_id: - span.set_attribute("scim.user_id", user_id) - span.add_event("scim.alert", {"operation": operation, "detail": detail}) + with tracer.start_span("scim.push.failure") as span: + span.set_attribute("scim.operation", operation) + span.set_attribute("scim.error", detail) + if user_id: + span.set_attribute("scim.user_id", user_id) + span.add_event("scim.alert", {"operation": operation, "detail": detail}) except Exception: # noqa: BLE001 — OTel 不可用不阻断主流程 pass diff --git a/src/agentkit/server/routes/auth.py b/src/agentkit/server/routes/auth.py index 05fbd14..a7b95bc 100644 --- a/src/agentkit/server/routes/auth.py +++ b/src/agentkit/server/routes/auth.py @@ -25,7 +25,7 @@ import hmac import logging from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, Literal import aiosqlite import jwt @@ -794,9 +794,7 @@ class SSORedirectResponse(BaseModel): state: str -async def _sso_issue_token_pair( - request: Request, user_dict: dict[str, object] -) -> TokenResponse: +async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) -> TokenResponse: """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。""" db_path = await _ensure_db(request) secret = _resolve_jwt_secret(request) @@ -843,7 +841,9 @@ async def _sso_issue_token_pair( role=str(user_dict["role"]), is_active=bool(user_dict.get("is_active", True)), is_terminal_authorized=bool(user_dict.get("is_terminal_authorized", False)), - is_server_terminal_authorized=bool(user_dict.get("is_server_terminal_authorized", False)), + is_server_terminal_authorized=bool( + user_dict.get("is_server_terminal_authorized", False) + ), ), session_id=pre_session.id, ) @@ -859,9 +859,11 @@ async def oidc_redirect(provider: str, request: Request) -> SSORedirectResponse: except ValueError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc # state 已由 provider 写入 _state_cache，从 URL 中提取回传前端 - state = "" - if "state=" in url: - state = url.split("state=", 1)[1].split("&", 1)[0] + from urllib.parse import parse_qs, urlparse + + parsed = urlparse(url) + state_values = parse_qs(parsed.query).get("state", []) + state = state_values[0] if state_values else "" return SSORedirectResponse(redirect_url=url, state=state) @@ -1088,7 +1090,7 @@ class RoleChangeRequest(BaseModel): model_config = ConfigDict(extra="forbid") - role: str # member / operator / admin + role: Literal["member", "operator", "admin"] reason: str | None = None @@ -1104,9 +1106,9 @@ async def change_user_role( 记录 actor/target/role_before/role_after/reason/source/timestamp，接入 OTel 审计。 变更后撤销该用户所有现有 session（强制重新登录以刷新 JWT role claim）。 """ - if payload.role not in ("member", "operator", "admin"): - raise HTTPException(status_code=400, detail="role 必须是 member/operator/admin 之一") db_path = await _ensure_db(request) + from agentkit.server.auth.audit import SOURCE_MANUAL, get_audit_service + async with aiosqlite.connect(str(db_path)) as db: db.row_factory = aiosqlite.Row cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) @@ -1120,8 +1122,6 @@ async def change_user_role( ) await db.commit() # 审计记录 - from agentkit.server.auth.audit import get_audit_service - await get_audit_service().record_role_change( actor_user_id=str(admin.get("user_id") or ""), actor_username=str(admin.get("username") or ""), @@ -1130,12 +1130,17 @@ async def change_user_role( role_before=role_before, role_after=payload.role, reason=payload.reason, - source="manual", + source=SOURCE_MANUAL, ) # 撤销所有 session（强制重新登录刷新 role claim） svc: SessionService = get_session_service() revoked = await svc.revoke_all_for_user(user_id, reason="role_changed") - return {"role_changed": True, "role_before": role_before, "role_after": payload.role, "revoked_sessions": revoked} + return { + "role_changed": True, + "role_before": role_before, + "role_after": payload.role, + "revoked_sessions": revoked, + } # Backwards-compat /me endpoint (kept for the existing frontend) From 3cf29f4633e323cee691af213eb0927cb9d255eb Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 08:36:59 +0800 Subject: [PATCH 08/11] =?UTF-8?q?fix(enterprise):=20ce-code-review=20P1=20?= =?UTF-8?q?fixes=20=E2=80=94=20SSO=20is=5Factive=20+=20span=20leak=20+=20H?= =?UTF-8?q?elm=20chart=20integration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 5c applied safe P1/P2/P3 fixes from multi-agent code review: - SEC-1 (P1, security): SSO 登录绕过 is_active 检查 — _sso_issue_token_pair 入口加显式校验，与本地 login（auth.py:377）一致。已停用用户无法再通过 IDP 回调拿 JWT，封堵 R1b deprovisioning 绕过。 - PERF-2/3 (P2, reliability): gateway chat() span 泄漏 + 异常未记录 — try 块 扩展到覆盖整个 span 生命周期（含 cache 设置），新增 except 分支调用 record_exception + set_status(ERROR)，失败调用在 trace 中不再显示为成功。 - API-3 (P1, test isolation): SCIM router 使用 import-time 捕获的 DEFAULT_AUTH_DB_PATH — 改为 resolve_auth_db_path() 在调用时读取 env var， 让测试 monkeypatch.setenv 可见。 - MAINT-1 (P3, dead code): 删除 BitableRepository.delete_view（被 delete_view_if_not_last 取代，无生产调用方，仅 docs 残留引用）。 - PERF-4/5 (P3, performance): pii_filter._record_pii_metrics 的 from import 提升至模块顶部 + except: pass 改为 logger.debug 记录失败原因。 - DEPLOY-3 (P1, chart integration): ConfigMap 不可达 — deployment.yaml args 增加 --config /etc/agentkit/agentkit.yaml，让应用能加载 chart 渲染 的非密配置。 - DEPLOY-6/7 (P3, doc/chart consistency): values.yaml 注释残留旧 slot 名 redisPgPasswords → redisPassword；slot 计数 10 → 11（5 处 chart + 3 处 deployment docs 同步修正）。 Verification: - ruff check + format clean (7 modified .py files) - pytest tests/unit/{auth,bitable} + test_{pii_filter,oidc_provider, saml_provider,scim_router,auth_audit,telemetry,prompt_cache_layers}.py → 336 passed, 145 skipped - helm lint + helm template 验证 secret key 一致性 Residual findings (deferred to follow-up): - PERF-1: PII filter 未接入 gateway 生产路径（manual, 需配置接入决策） - API-1/2/4/5: SCIM RFC 7644 合规性（totalResults/error format/Location header/SSO 预配联动）— manual, 需设计决策 - DEPLOY-1/2/4: Helm chart ↔ app 集成断裂（部署名引用/Redis 鉴权/PG 密码双源） — manual, 需 app+chart 联动修改 --- .../helm/agentkit/templates/deployment.yaml | 2 + .../agentkit/templates/external-secret.yaml | 2 +- .../agentkit/templates/sealed-secret.yaml | 2 +- deploy/helm/agentkit/values.yaml | 6 +- docs/deployment/k8s-install.md | 2 +- docs/deployment/key-rotation-runbook.md | 2 +- docs/deployment/secret-inventory.md | 2 +- src/agentkit/bitable/repository.py | 7 -- src/agentkit/llm/gateway.py | 107 ++++++++++-------- src/agentkit/llm/pii_filter.py | 21 ++-- src/agentkit/server/auth/scim/router.py | 11 +- src/agentkit/server/routes/auth.py | 4 + 12 files changed, 96 insertions(+), 72 deletions(-) diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml index 4446d9f..d87e3ec 100644 --- a/deploy/helm/agentkit/templates/deployment.yaml +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -32,6 +32,8 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} args: - serve + - --config + - /etc/agentkit/agentkit.yaml - --host - 0.0.0.0 - --port diff --git a/deploy/helm/agentkit/templates/external-secret.yaml b/deploy/helm/agentkit/templates/external-secret.yaml index ed8a879..cddda56 100644 --- a/deploy/helm/agentkit/templates/external-secret.yaml +++ b/deploy/helm/agentkit/templates/external-secret.yaml @@ -1,7 +1,7 @@ {{- /* External Secrets Operator 模板 — 当 secrets.backend=external-secrets 时渲染。 ExternalSecret 引用运维预创建的 SecretStore（指向 Vault / AWS SM / GCP SM / Azure KV）， -从外部密钥管理服务拉取全部 10 个 secret slot（KTD-4）。 +从外部密钥管理服务拉取全部 11 个 secret slot（KTD-4）。 */ -}} {{- if eq .Values.secrets.backend "external-secrets" }} apiVersion: external-secrets.io/v1beta1 diff --git a/deploy/helm/agentkit/templates/sealed-secret.yaml b/deploy/helm/agentkit/templates/sealed-secret.yaml index b22d926..18616d1 100644 --- a/deploy/helm/agentkit/templates/sealed-secret.yaml +++ b/deploy/helm/agentkit/templates/sealed-secret.yaml @@ -1,6 +1,6 @@ {{- /* Sealed Secrets 模板（Bitnami Sealed Secrets controller）— 当 secrets.backend=sealed-secrets 时渲染。 -覆盖全部 10 个 secret slot（KTD-4）。encryptedData 由运维通过 kubeseal 生成后以 --set 或 +覆盖全部 11 个 secret slot（KTD-4）。encryptedData 由运维通过 kubeseal 生成后以 --set 或 override values 文件注入。禁止明文 Kubernetes Secret + Git 提交。 */ -}} {{- if eq .Values.secrets.backend "sealed-secrets" }} diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml index e01ee5f..b473fad 100644 --- a/deploy/helm/agentkit/values.yaml +++ b/deploy/helm/agentkit/values.yaml @@ -110,7 +110,7 @@ otel: # --------------------------------------------------------------------------- redis: url: redis://redis:6379/0 - # 实际密码通过 env REDIS_PASSWORD 注入（来自 secrets.redisPgPasswords） + # 实际密码通过 env REDIS_PASSWORD 注入（来自 secrets.redisPassword） postgres: # DATABASE_URL（不含密码，密码通过 ${POSTGRES_PASSWORD} 占位由 env 注入） @@ -124,7 +124,7 @@ auth: idps: [] # IDP 配置列表（非密部分），密钥引用 secrets.idpSigningCerts # --------------------------------------------------------------------------- -# 密钥清单 — 10 个 secret slot（KTD-4 / R2a） +# 密钥清单 — 11 个 secret slot（KTD-4 / R2a） # --------------------------------------------------------------------------- # backend 选择： # sealed-secrets — 使用 Bitnami Sealed Secrets（模板 templates/sealed-secret.yaml） @@ -140,7 +140,7 @@ secrets: # 后端类型：vault | aws | gcp | azure backend: vault - # --- 10 个 secret slot --- + # --- 11 个 secret slot --- # 每个 slot：name=Kubernetes Secret key；description=用途；rotation=轮换周期 # 1) JWT 签名密钥（HS256）— access(15m) + refresh(7d) token 签名 diff --git a/docs/deployment/k8s-install.md b/docs/deployment/k8s-install.md index 7f3e070..2ef0b38 100644 --- a/docs/deployment/k8s-install.md +++ b/docs/deployment/k8s-install.md @@ -32,7 +32,7 @@ ### 2.1 准备密钥 -按 `docs/deployment/secret-inventory.md` 准备 10 个 secret slot 的明文值， +按 `docs/deployment/secret-inventory.md` 准备 11 个 secret slot 的明文值， 然后根据所选 backend 生成密钥资源。 **Sealed Secrets 方式**（推荐单集群 / 离线场景）： diff --git a/docs/deployment/key-rotation-runbook.md b/docs/deployment/key-rotation-runbook.md index 7995a3b..9caf123 100644 --- a/docs/deployment/key-rotation-runbook.md +++ b/docs/deployment/key-rotation-runbook.md @@ -1,6 +1,6 @@ # Fischer AgentKit — 密钥轮换 Runbook -本文档给出全部 10 个 secret slot（KTD-4 / R2a）的轮换操作流程。 +本文档给出全部 11 个 secret slot（KTD-4 / R2a）的轮换操作流程。 所有轮换遵循「双密钥并行 → 切换 → 旧密钥退役」原则，避免服务中断。 ## 通用轮换原则 diff --git a/docs/deployment/secret-inventory.md b/docs/deployment/secret-inventory.md index 96b8724..371e752 100644 --- a/docs/deployment/secret-inventory.md +++ b/docs/deployment/secret-inventory.md @@ -1,6 +1,6 @@ # Fischer AgentKit — 密钥清单（Secret Inventory） -本文档列出 Helm Chart 涉及的全部 10 个 secret slot（KTD-4 / R2a）， +本文档列出 Helm Chart 涉及的全部 11 个 secret slot（KTD-4 / R2a）， 每个 slot 含：名称、用途、来源、轮换周期、归属角色、对应环境变量。 > 密钥后端二选一：Sealed Secrets（`secrets.backend: sealed-secrets`）或 diff --git a/src/agentkit/bitable/repository.py b/src/agentkit/bitable/repository.py index 426780b..afaee6d 100644 --- a/src/agentkit/bitable/repository.py +++ b/src/agentkit/bitable/repository.py @@ -486,13 +486,6 @@ class BitableRepository: await session.commit() return View.model_validate(entity) if entity else None - async def delete_view(self, view_id: str) -> bool: - """Delete a view. Returns True if a row was deleted.""" - async with self._session_factory() as session: - result = await session.execute(delete(ViewModel).where(ViewModel.id == view_id)) - await session.commit() - return result.rowcount > 0 - async def delete_view_if_not_last(self, view_id: str, table_id: str) -> bool: """Atomically delete a view only if it's not the last one in its table (DR-4 TOCTOU fix). diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index d74d753..dbc9cf5 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -157,54 +157,55 @@ class LLMGateway: _span = _span_cm.__enter__() start = time.monotonic() - - # ── Cache check (U17 — LiteLLM cache via cache_key in request) ── - # LiteLLM 在 litellm.acompletion 内部处理缓存读写，gateway 只需： - # 1. 构建 per-user + ACL-scoped cache_key（安全要求 a, b） - # 2. 将 cache 参数注入 kwargs 透传到 provider - # 3. 检测响应的 cache_hit 标志，用于 usage tracking（cost=0） - if self._cache_manager is not None: - from agentkit.llm.cache import LitellmCacheManager - - # 解析 KB caching_disabled（安全要求 c） - # 非 RAG 请求（kb_id=None）→ 默认启用缓存（无 KB 数据需保护）。 - # RAG 请求（kb_id!=None）→ fail-closed：默认禁用缓存，仅在 settings - # 明确返回 caching_disabled=False 时启用。防止 DB 异常时 fail-open - # 导致禁用缓存的 KB 数据泄漏到缓存。 - kb_caching_disabled = kb_id is not None - if kb_id is not None: - try: - from agentkit.rag_platform.settings import get_settings_store - - settings = await get_settings_store().get_settings(kb_id) - if settings is not None: - kb_caching_disabled = settings.caching_disabled - # settings 为 None（KB 不存在）→ 保持 True（fail-closed） - except Exception as e: - logger.warning(f"Failed to read KB cache settings for kb_id={kb_id}: {e}") - # 读取异常 → 保持 True（fail-closed，禁用缓存） - - if self._cache_manager.should_cache(kb_caching_disabled, user_id): - cache_key = self._cache_manager.build_cache_key( - model=resolved_model, - messages=messages, - temperature=kwargs.get("temperature", 0.7), - tools=tools, - tool_choice=tool_choice, - max_tokens=kwargs.get("max_tokens", 2000), - user_id=user_id, - kb_acl_hash=kb_acl_hash, - ) - kwargs["cache"] = LitellmCacheManager.cache_params_for_hit(cache_key) - else: - kwargs["cache"] = LitellmCacheManager.cache_params_for_no_cache() - - # ── Normal provider call ── - models_to_try = self._get_models_to_try(resolved_model) - last_error: LLMProviderError | None = None - response: LLMResponse | None = None - + # PERF-2: try 块必须覆盖整个 span 生命周期（含 cache 设置）， + # 否则 cache_key 构建异常会泄漏 span。PERF-3: except 记录异常到 span。 try: + # ── Cache check (U17 — LiteLLM cache via cache_key in request) ── + # LiteLLM 在 litellm.acompletion 内部处理缓存读写，gateway 只需： + # 1. 构建 per-user + ACL-scoped cache_key（安全要求 a, b） + # 2. 将 cache 参数注入 kwargs 透传到 provider + # 3. 检测响应的 cache_hit 标志，用于 usage tracking（cost=0） + if self._cache_manager is not None: + from agentkit.llm.cache import LitellmCacheManager + + # 解析 KB caching_disabled（安全要求 c） + # 非 RAG 请求（kb_id=None）→ 默认启用缓存（无 KB 数据需保护）。 + # RAG 请求（kb_id!=None）→ fail-closed：默认禁用缓存，仅在 settings + # 明确返回 caching_disabled=False 时启用。防止 DB 异常时 fail-open + # 导致禁用缓存的 KB 数据泄漏到缓存。 + kb_caching_disabled = kb_id is not None + if kb_id is not None: + try: + from agentkit.rag_platform.settings import get_settings_store + + settings = await get_settings_store().get_settings(kb_id) + if settings is not None: + kb_caching_disabled = settings.caching_disabled + # settings 为 None（KB 不存在）→ 保持 True（fail-closed） + except Exception as e: + logger.warning(f"Failed to read KB cache settings for kb_id={kb_id}: {e}") + # 读取异常 → 保持 True（fail-closed，禁用缓存） + + if self._cache_manager.should_cache(kb_caching_disabled, user_id): + cache_key = self._cache_manager.build_cache_key( + model=resolved_model, + messages=messages, + temperature=kwargs.get("temperature", 0.7), + tools=tools, + tool_choice=tool_choice, + max_tokens=kwargs.get("max_tokens", 2000), + user_id=user_id, + kb_acl_hash=kb_acl_hash, + ) + kwargs["cache"] = LitellmCacheManager.cache_params_for_hit(cache_key) + else: + kwargs["cache"] = LitellmCacheManager.cache_params_for_no_cache() + + # ── Normal provider call ── + models_to_try = self._get_models_to_try(resolved_model) + last_error: LLMProviderError | None = None + response: LLMResponse | None = None + for model_name in models_to_try: try: provider, actual_model = self._resolve_model(model_name) @@ -305,6 +306,18 @@ class LLMGateway: self._record_prompt_cache_metric(response.usage, resolved_model) return response + except Exception as e: + # PERF-3: 记录异常到 span — __exit__(None, None, None) 会丢失异常信息， + # 失败的 LLM 调用在 trace 中显示为成功。手动 record_exception + 设 ERROR 状态。 + if _span is not None: + try: + _span.record_exception(e) + from opentelemetry.trace import Status, StatusCode + + _span.set_status(Status(StatusCode.ERROR, str(e))) + except Exception: # noqa: BLE001 — span 记录失败不影响异常向上传播 + pass + raise finally: if _span_cm is not None: _span_cm.__exit__(None, None, None) diff --git a/src/agentkit/llm/pii_filter.py b/src/agentkit/llm/pii_filter.py index 633f108..04691f9 100644 --- a/src/agentkit/llm/pii_filter.py +++ b/src/agentkit/llm/pii_filter.py @@ -20,6 +20,11 @@ import logging import re from dataclasses import dataclass, field +from agentkit.telemetry.metrics import ( + pii_filter_coverage_counter, + pii_filter_redacted_counter, +) + logger = logging.getLogger(__name__) @@ -120,18 +125,18 @@ def _redact(text: str, ner_hook=None) -> PIIFilterResult: def _record_pii_metrics(status: str, redacted_count: int | None = None) -> None: - """记录 PII 脱敏 OTel 指标 — 失败静默降级（不影响主流程）。""" - try: - from agentkit.telemetry.metrics import ( - pii_filter_coverage_counter, - pii_filter_redacted_counter, - ) + """记录 PII 脱敏 OTel 指标 — 失败静默降级（不影响主流程）。 + PERF-4: 用 logger.debug 记录失败原因（而非 except: pass）， + 保留静默降级语义同时让排障可见。 + PERF-5: import 提升至模块顶部，避免热路径每次 from import 开销。 + """ + try: pii_filter_coverage_counter().add(1, {"pii_filter.status": status}) if redacted_count is not None: pii_filter_redacted_counter().add(redacted_count) - except Exception: - pass + except Exception as e: # noqa: BLE001 — 指标失败不影响 PII 过滤主流程 + logger.debug("PII metric recording failed: %s", e) def filter_pii( diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py index 9645c27..d53057c 100644 --- a/src/agentkit/server/auth/scim/router.py +++ b/src/agentkit/server/auth/scim/router.py @@ -23,7 +23,12 @@ import aiosqlite from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from pydantic import BaseModel, ConfigDict -from ...auth.models import DEFAULT_AUTH_DB_PATH, init_auth_db, user_row_to_dict, _now_iso +from ...auth.models import ( + init_auth_db, + resolve_auth_db_path as _resolve_default_db_path, + user_row_to_dict, + _now_iso, +) from ...auth.password import hash_password from .models import SCIMListResponse, SCIMPatchRequest, SCIMUser @@ -36,7 +41,9 @@ _SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" def _resolve_db_path(request: Request) -> Path: path = getattr(request.app.state, "auth_db_path", None) - return Path(path) if path else DEFAULT_AUTH_DB_PATH + # API-3: 用 resolve_auth_db_path() 在调用时读取 env var， + # 让测试 monkeypatch.setenv("AGENTKIT_AUTH_DB", tmp) 可见。 + return Path(path) if path else _resolve_default_db_path() def _resolve_scim_token(request: Request) -> str | None: diff --git a/src/agentkit/server/routes/auth.py b/src/agentkit/server/routes/auth.py index a7b95bc..62871a4 100644 --- a/src/agentkit/server/routes/auth.py +++ b/src/agentkit/server/routes/auth.py @@ -796,6 +796,10 @@ class SSORedirectResponse(BaseModel): async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) -> TokenResponse: """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。""" + # SEC-1: SSO 流程必须与本地 login（auth.py:377）一致地校验 is_active， + # 否则已 deprovision 的用户可通过 IDP 回调重新拿到 JWT + session。 + if not bool(user_dict.get("is_active", True)): + raise HTTPException(status_code=403, detail="Account is disabled") db_path = await _ensure_db(request) secret = _resolve_jwt_secret(request) svc: SessionService = get_session_service() From 444667e2f0830039a75f30583539902fa2434e7e Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 10:40:06 +0800 Subject: [PATCH 09/11] docs(review): record residual review findings --- ...at-enterprise-agent-platform-p0-3cf29f4.md | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md diff --git a/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md new file mode 100644 index 0000000..581b001 --- /dev/null +++ b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md @@ -0,0 +1,182 @@ +# Residual Review Findings — feat/enterprise-agent-platform-p0 + +**Source**: ce-code-review run on commit range `8633f60..3cf29f4` +**Branch**: `feat/enterprise-agent-platform-p0` +**HEAD**: `3cf29f4` +**Tracker availability**: no named tracker documented; `gh` CLI not installed; +`any_sink_available = false` — all findings routed to `no_sink` bucket per +lfg `references/tracker-defer.md` non-interactive mode. + +These findings were surfaced by the multi-agent code review (5 reviewer +subagents: security + adversarial, performance + reliability, +maintainability + standards, api-contract + data-migration, deployment +verification). They are **actionable** (`autofix_class: manual`) but were +**not applied** in Step 5 because they need design input or app+chart +coordinated changes. They are durably recorded here per lfg SKILL.md Step 6. + +## P1 — High-impact defects + +- **P1 / `src/agentkit/llm/pii_filter.py` + `src/agentkit/llm/gateway.py`** — + PERF-1: PII 脱敏模块未接入生产 LLM 调用路径（死代码）。`filter_messages_pii` + 仅被单元测试引用，gateway.py 的 chat()/chat_stream() 直接透传原始 messages。 + 等保合规声称的 PII 保护实际未生效。**Fix**: 在 gateway chat() 接入 + `filter_messages_pii(messages, fail_closed=True)`，从 `agentkit.yaml` 读 + PII 配置开关和 ner_hook 路径。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:233-238`** — + API-1: SCIM GET /Users 的 `totalResults` 返回当前页条数而非全量总数 + （违反 RFC 7644 3.4.2）。当用户总数 > count 时，SCIM 客户端无法得知真实 + 总数，分页逻辑错误。**Fix**: 在列表查询前先执行 `SELECT COUNT(*) FROM + users {where}` 复用同一 where 子句和参数。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:50-64,107-151,154-197,200-238`** — + API-2: SCIM 错误响应使用 FastAPI 默认 `{"detail": "..."}` 而非 RFC 7644 + 3.12 规定的 `{"schemas": ["urn:ietf:params:scim:api:messages:2.0:Error"], + "status": "409", "detail": "..."}`。同文件中定义的 `SCIMErrorResponse` 是 + 死代码。**Fix**: 添加 SCIM 异常处理器转换所有 HTTPException 为 SCIM + ErrorResponse 格式。 + +- **P1 / `src/agentkit/server/routes/auth.py:804-815,823-827`** — + API-4: `_sso_issue_token_pair` 用占位符 `refresh_token="__pending_sso__"` + 创建 session，并发 SSO 登录会触发 `refresh_token_hash` UNIQUE 约束冲突 + 抛 IntegrityError → 500。即使不考虑并发，两步写入也不原子，第二步失败 + 会留下孤儿 session。**Fix**: 先 `secrets.token_urlsafe(32)` 生成真实 + refresh token 再 `SessionCreate` 一步创建。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:107-151`** — + API-5: SCIM `create_user` 不写入 `user_idp_links`，导致 SCIM 预配的用户 + 无法 SSO 登录（OIDC 会 JIT 创建重复账户）。SCIM 的核心价值就是预配 + SSO + 联动，此处断裂使 SCIM 预配失效。**Fix**: SCIM create_user 接受 + `externalId` 作为 idp_subject，INSERT users 后 INSERT user_idp_links。 + +- **P1 / `deploy/offline/scripts/install.sh:58` + docs** — + DEPLOY-1: 部署名引用错误。fullname helper 在 release=agentkit 时渲染 + Deployment 名为 `agentkit`，但 install.sh:58 + 11 处文档引用 + `agentkit-agentkit`（不存在）。**Fix**: 改用标签选择器 + `kubectl rollout restart deployment -l app.kubernetes.io/instance=${RELEASE}`。 + +- **P1 / `deploy/helm/agentkit/templates/deployment.yaml:88-92`** — + DEPLOY-2: Redis 鉴权断裂。`REDIS_PASSWORD` env 注入但应用从不消费 + （`src/agentkit/**/*.py` 零命中），Redis 客户端通过 `redis_url` 构造 + （`redis://redis:6379/0`，无密码）。若 Redis 启用密码，pod 全部 Redis + 操作 NOAUTH 失败。**Fix**: 在 redis 客户端构造处读取 `REDIS_PASSWORD` + 并以 `password=` 参数传入 `from_url`，或将密码拼入 redis_url。 + +## P2 — Moderate issues with meaningful downside + +- **P2 / `src/agentkit/server/auth/scim/router.py:107-151`** — + API-6: SCIM POST /Users 缺少 `Location` 响应头（RFC 7644 3.14 强制要求）。 + **Fix**: 用 `JSONResponse` + `response.headers["Location"] = ...`。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:183-187`** — + API-7: SCIM PATCH userName 更新未处理 UNIQUE 约束冲突，返回 500 而非 + 409。**Fix**: 包在 try/except `aiosqlite.IntegrityError` 中返回 409。 + +- **P2 / `src/agentkit/server/routes/auth.py:812`** — + API-8: `_sso_issue_token_pair` 硬编码 `auth_provider="sso"`，丢失 OIDC/SAML + 区分。admin UI 无法按 IDP 筛选 session。**Fix**: 增加 `provider_name` + + `provider_type` 参数。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:216-225`** — + API-9: SCIM filter 语法不匹配时静默返回全量用户，可能导致数据泄露。 + **Fix**: 不支持的 filter 返回 400 + `scimType=invalidFilter`。 + +- **P2 / `src/agentkit/server/routes/auth.py:1040-1085,1097-1143`** — + API-10: `deprovision_user` / `change_user_role` 返回 untyped `dict[str, Any]`， + OpenAPI schema 无法文档化。**Fix**: 定义 `DeprovisionResponse` / + `RoleChangeResponse` Pydantic 模型。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:94-104`** — + API-11: SCIM `_user_to_scim` 用 `username` 作为 `displayName`，语义错误。 + **Fix**: 改为 `None` 让客户端 fallback。 + +- **P2 / `src/agentkit/server/auth/models.py:612-620`** — + MIG-1: `user_idp_links` 表无 ON DELETE 行为且无 FK 约束到 users。当前无 + 删除路径，但未来引入 GDPR 物理删除时会产生孤儿链接。**Fix**: 引入物理 + 删除时加 `REFERENCES users(id) ON DELETE CASCADE`。 + +- **P2 / `deploy/helm/agentkit/templates/deployment.yaml:58-62,93-97`** — + DEPLOY-4: PG 密码双源不一致。`DATABASE_URL`（被消费） 与 `POSTGRES_PASSWORD` + （死 env）并存。runbook Slot 10 轮换 `postgres-password` 但应用实际用 + `DATABASE_URL`。**Fix**: 明确单一可信源 — 若只用 `DATABASE_URL`，移除 + `POSTGRES_PASSWORD` env 与 `postgresPassword` slot。 + +- **P2 / `deploy/helm/agentkit/templates/deployment.yaml:83-87`** — + DEPLOY-5: OTel exporter token 值格式与 `OTEL_EXPORTER_OTLP_HEADERS` 语义 + 不符。期望 `key=value` 格式，但存储值为 `Bearer `。**Fix**: 改为 + `Authorization=Bearer ` 格式。 + +- **P2 / `src/agentkit/server/auth/scim/router.py` (7 处) + `scim/models.py`** — + STAND-1: SCIM router 使用 `typing.Any` 7 处，违反 AGENTS.md 禁用 any 规则。 + **Fix**: 定义 `SCIMUserResponse` / `SCIMUserListResponse` Pydantic 模型。 + +## P3 — Low-impact / narrow scope + +- **P3 / `src/agentkit/server/auth/providers/oidc.py:185-197`** — + SEC-2: OIDC id_token 未按规范验签（key=None + 关闭 iss/exp）。当前 + authlib 1.6.12 fail-closed，非直接绕过，但规范违反且脆弱。**Fix**: 用 + IDP JWKS 公钥验签，校验 iss/aud/exp。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:170-185`** — + SEC-3: SCIM PATCH deprovisioning（`active=false`）不写审计。admin 同等 + 操作有审计，SCIM 没有。**Fix**: 检测到 active true→false 时调用 + `record_deprovision(source=SOURCE_SCIM)`。 + +- **P3 / `src/agentkit/llm/gateway.py:401-414`** — + PERF-6: 流式路径在无 usage 数据时记录"cache miss"，指标误导。**Fix**: + 仅当 `final_usage` 非 None 时才记录 cache metric。 + +- **P3 / `src/agentkit/llm/gateway.py:450-457`** — + PERF-7: 非 Anthropic provider 调用膨胀 `prompt_cache.miss` 计数器。 + **Fix**: 仅对 Anthropic provider 记录 cache metric。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:171-187`** — + API-12: SCIM PATCH 静默忽略不支持的 op/path，无错误反馈。**Fix**: + 收集 skipped ops 并返回 400 + `scimType=noTarget`。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:200-238`** — + API-13: SCIM filter 解析对属性名和操作符大小写敏感，违反 RFC 7644 + 3.4.2.2。**Fix**: 用 `filter.lower()` + 正则 `re.IGNORECASE`。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:154-197`** — + API-14: SCIM PATCH `op.path==None` 时仅更新 active，丢弃 value 中的其他 + 字段（部分实现静默丢数据）。**Fix**: 遍历 value 的 keys 对每个已知属性 + 执行对应 UPDATE。 + +- **P3 / `src/agentkit/server/auth/models.py:609-639`** — + MIG-2: schema 迁移幂等但缺少 V4→V5 的显式迁移记录。**Fix**: 未来新增列 + 时按 schema_version 差值执行 ALTER TABLE。 + +- **P3 / `deploy/helm/agentkit/values.yaml:8-11`** — + DEPLOY-9: 镜像仅 tag 固定（0.1.0），未做 digest pinning。政企生产场景 + 要求 digest pinning 防篡改。**Fix**: 支持可选 `image.digest` 字段。 + +- **P3 / `docs/deployment/key-rotation-runbook.md:254-283`** — + DEPLOY-8: runbook 仅 Slot 1-10，缺独立 Slot 11（PG 密码）段落。**Fix**: + 拆为两节，Slot 11 内注明须同步更新 `database-url`（Slot 3）。 + +- **P3 / `src/agentkit/bitable/service.py:564-568`** — + MAINT-2: `delete_view` 将两种不同失败原因统一抛 `LastViewDeletionError`， + 误导性错误信息。**Fix**: 区分 `DELETED`/`LAST_VIEW`/`NOT_FOUND` 三态。 + +- **P3 / `src/agentkit/server/frontend/src/stores/chatStream.ts:881-887,923-929`** — + MAINT-3: `team_synthesis_chunk` 与 `team_synthesis` 重复同一 `findLastMessage` + 谓词。**Fix**: 提取 `findStreamingSynthesisMilestone(conv, sid)` helper。 + +- **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:75,81,85`** — + MAINT-4: token 声明正则转义重复 3 次。**Fix**: 提取 `tokenDeclPattern(token)` + helper。 + +- **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:92`** — + STAND-2: test "outside root" 用 replace 仅替换首个 `:root` 块。**Fix**: + 加 `g` 标志 `replace(/:root\s*\{[^}]*\}/g, '')`。 + +- **P3 / `src/agentkit/server/app.py:1401`** — + STAND-3: SCIM router 挂载在 `/api/v1` 前缀下，偏离 RFC 7644 标准 SCIM + 路径 `/scim/v2`。**Fix**: SCIM router 单独挂载不加 `/api/v1` 前缀。 + +--- + +**Total residual findings**: 27 (P1=7, P2=10, P3=10) +**Applied in Step 5**: 9 fixes (commit `3cf29f4`) +**Deferred to follow-up**: 27 (this file) From cb278bb5f0cbe2ad8c0cffa712bacfd8aca6143d Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 11:41:06 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix(enterprise):=20ce-debug=20batch=20?= =?UTF-8?q?=E2=80=94=20PERF-1=20PII=20filter=20+=20DEPLOY-1/2/4/5=20+=20SC?= =?UTF-8?q?IM=20RFC=207643/7644?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 ce-code-review 发现的全部 P1/P2 residual findings： P1: - PERF-1: PII 脱敏接入 gateway chat()/chat_stream() 生产路径 （env AGENTKIT_PII_FILTER_ENABLED=true 启用，fail-closed） - API-1/2/5/6/7/9/11/13 + STAND-1 + SEC-3: SCIM router 全面重写 - API-1: totalResults 用 SELECT COUNT(*) 返回全量总数 - API-2: 异常处理器转 RFC 7644 错误格式（scope 到 /scim/v2） - API-5: create_user 写 user_idp_links（idp_name='scim'） - API-6: POST /Users 返回 Location 头 - API-7: PATCH userName UNIQUE 冲突 → 409 - API-9: 不支持的 filter → 400 invalidFilter - API-11: displayName=None - API-13: filter 大小写不敏感 - STAND-1: dict[str, Any] → dict[str, object] - SEC-3: active true→false 写审计（延后到事务提交后避免 SQLite 锁竞争） - API-4: _sso_issue_token_pair 用 secrets.token_urlsafe(32) 生成真实 refresh_token （移除 __pending_sso__ 占位符 + 后续 UPDATE 的非原子两步写入） - API-8: _sso_issue_token_pair 增加 provider_type/provider_name 参数 - API-10: deprovision_user/change_user_role 加 response_model - DEPLOY-1: install.sh + docs 用 instance 标签选择器替代硬编码 fullname - DEPLOY-2: 11 处 Redis 客户端构造显式传入 REDIS_PASSWORD env var P2: - DEPLOY-4: 移除 agentkit deployment 中死 env POSTGRES_PASSWORD （应用通过 DATABASE_URL Slot 3 内嵌密码连接，不读此 env） - DEPLOY-5: OTel token 格式改为 Authorization=Bearer （符合 OTEL_EXPORTER_OTLP_HEADERS key=value 规范） 测试： - ruff check + format 全部通过 - pytest tests/unit/test_scim_router.py 9 passed - pytest tests/unit/test_llm_gateway.py + test_handoff.py + test_event_queue_integration.py 70 passed, 5 skipped - helm lint + helm template 验证 POSTGRES_PASSWORD env 已移除 --- .../helm/agentkit/templates/deployment.yaml | 9 +- deploy/helm/agentkit/values.yaml | 8 +- deploy/offline/scripts/install.sh | 6 +- docs/deployment/k8s-install.md | 8 +- docs/deployment/key-rotation-runbook.md | 29 +-- docs/deployment/secret-inventory.md | 6 +- src/agentkit/bus/redis_bus.py | 28 ++- src/agentkit/core/base.py | 8 +- src/agentkit/core/config_driven.py | 7 +- src/agentkit/core/handoff_transport.py | 12 +- src/agentkit/llm/cache.py | 8 +- src/agentkit/llm/gateway.py | 29 +++ src/agentkit/llm/providers/usage_store.py | 15 +- src/agentkit/orchestrator/pipeline_state.py | 3 + src/agentkit/quality/cascade_state_store.py | 58 ++++- src/agentkit/server/app.py | 9 +- src/agentkit/server/auth/scim/models.py | 26 ++- src/agentkit/server/auth/scim/router.py | 211 ++++++++++++++---- src/agentkit/server/routes/auth.py | 87 ++++++-- src/agentkit/server/task_store.py | 63 +++++- src/agentkit/session/store.py | 38 +++- 21 files changed, 520 insertions(+), 148 deletions(-) diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml index d87e3ec..d61bbd8 100644 --- a/deploy/helm/agentkit/templates/deployment.yaml +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -92,11 +92,10 @@ spec: secretKeyRef: name: {{ include "agentkit.secretName" . }} key: {{ .Values.secrets.redisPassword.key }} - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "agentkit.secretName" . }} - key: {{ .Values.secrets.postgresPassword.key }} + # DEPLOY-4: 移除 POSTGRES_PASSWORD env — 应用通过 DATABASE_URL（Slot 3） + # 消费 PG 密码（DSN 内嵌），POSTGRES_PASSWORD 仅用于 PostgreSQL StatefulSet + # 初始化（由独立 PG chart / StatefulSet 引用 secrets.postgresPassword）。 + # 此处注入是死 env，误导运维以为应用读 POSTGRES_PASSWORD。 # --- 非密配置（OTel endpoint、service name）--- {{- if .Values.otel.enabled }} - name: OTEL_EXPORTER_OTLP_ENDPOINT diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml index b473fad..4caed7a 100644 --- a/deploy/helm/agentkit/values.yaml +++ b/deploy/helm/agentkit/values.yaml @@ -194,19 +194,19 @@ secrets: # 9) OTel exporter token — OTLP collector 鉴权 token otelExporterToken: key: otel-exporter-token - description: "OTLP exporter 鉴权 token（collector 启用 bearer auth 时）" + description: "OTLP exporter 鉴权 token（DEPLOY-5: 值格式必须为 'Authorization=Bearer '，符合 OTEL_EXPORTER_OTLP_HEADERS key=value 规范）" rotation: 90d # 10) Redis 密码 — Redis requirepass（基础设施层） redisPassword: key: redis-password - description: "Redis requirepass（基础设施层）" + description: "Redis requirepass（基础设施层，env REDIS_PASSWORD 注入应用）" rotation: 90d - # 11) PostgreSQL 密码 — POSTGRES_PASSWORD（基础设施层） + # 11) PostgreSQL 密码 — POSTGRES_PASSWORD（基础设施层，PG StatefulSet 初始化用） postgresPassword: key: postgres-password - description: "PostgreSQL POSTGRES_PASSWORD（基础设施层）" + description: "PostgreSQL POSTGRES_PASSWORD（DEPLOY-4: 仅 PG StatefulSet 初始化消费，应用通过 DATABASE_URL Slot 3 内嵌密码连接，不读此 env）" rotation: 90d # --------------------------------------------------------------------------- diff --git a/deploy/offline/scripts/install.sh b/deploy/offline/scripts/install.sh index c99ebd2..6b8793b 100755 --- a/deploy/offline/scripts/install.sh +++ b/deploy/offline/scripts/install.sh @@ -55,7 +55,11 @@ helm upgrade --install "${RELEASE}" "${CHART_TGZ}" \ --set image.pullPolicy=Never # 离线本地镜像，Never 强制使用已加载的本地镜像 echo "==> 等待 rollout" -kubectl rollout status deployment/"${RELEASE}-agentkit" -n "${NAMESPACE}" --timeout=300s || \ +# DEPLOY-1: 用标签选择器定位 Deployment — fullname 在 release 名包含 chart 名时 +# 仅返回 release 名（如 release=agentkit → Deployment=agentkit），其他情况返回 +# release-chart（如 release=myprod → myprod-agentkit）。硬编码 "${RELEASE}-agentkit" +# 在默认 release=agentkit 时拼出不存在的 agentkit-agentkit。改用 instance 标签。 +kubectl rollout status deployment -l app.kubernetes.io/instance="${RELEASE}" -n "${NAMESPACE}" --timeout=300s || \ echo "WARN: rollout 未在 300s 内完成，请检查 pod 状态：kubectl get pods -n ${NAMESPACE}" echo "==> 完成。验证：" diff --git a/docs/deployment/k8s-install.md b/docs/deployment/k8s-install.md index 2ef0b38..8714888 100644 --- a/docs/deployment/k8s-install.md +++ b/docs/deployment/k8s-install.md @@ -163,12 +163,14 @@ kubectl get pods -n agentkit kubectl get svc,ingress -n agentkit # 3) 健康检查 -kubectl exec -n agentkit deploy/agentkit-agentkit -- \ +# DEPLOY-1: 默认 release=agentkit 时 Deployment 名为 agentkit（fullname helper +# 在 release 名包含 chart 名时去重）。若改了 release 名，按 helm status 输出替换。 +kubectl exec -n agentkit deploy/agentkit -- \ curl -s http://localhost:8001/api/v1/health # 期望：{"status":"ok"} # 4) Secret 已注入 -kubectl exec -n agentkit deploy/agentkit-agentkit -- env | grep -E 'JWT|API_KEY|DATABASE' | sed 's/=.*/=/' +kubectl exec -n agentkit deploy/agentkit -- env | grep -E 'JWT|API_KEY|DATABASE' | sed 's/=.*/=/' # 期望：所有 env 都有值（不为空） # 5) helm 状态 @@ -181,7 +183,7 @@ helm status agentkit -n agentkit ```bash # 查看日志 -kubectl logs -n agentkit deploy/agentkit-agentkit --tail=50 +kubectl logs -n agentkit deploy/agentkit --tail=50 # 常见原因： # - Secret 未注入：检查 SealedSecret 是否已解封（kubectl get sealedsecret -n agentkit） diff --git a/docs/deployment/key-rotation-runbook.md b/docs/deployment/key-rotation-runbook.md index 9caf123..63df903 100644 --- a/docs/deployment/key-rotation-runbook.md +++ b/docs/deployment/key-rotation-runbook.md @@ -34,10 +34,11 @@ kubectl create secret generic agentkit-secrets-new --dry-run=client -o yaml \ # 3) 应用并等待同步 kubectl apply -f agentkit-sealedsecret.yaml -n agentkit -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +# DEPLOY-1: 用 instance 标签定位 Deployment — 避免硬编码 fullname 拼错。 +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 4) 验证 -kubectl exec -n agentkit deploy/agentkit-agentkit -- \ +kubectl exec -n agentkit deploy/agentkit -- \ python -c "import os; print('jwt key len:', len(os.environ['JWT_SIGNING_KEY']))" # 5) 用户重新登录（旧 token 失效） @@ -62,7 +63,7 @@ NEW_KEY=$(openssl rand -hex 32) # 3) 通知所有使用 API Key 的客户端更新（agentkit pair 重新生成） # 4) 应用并 restart -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 5) 验证：用新 key 调用 /api/v1/auth/me 或 SCIM 端点 ``` @@ -87,7 +88,7 @@ NEW_URL="postgresql+asyncpg://agentkit:new-pg-pwd@postgres:5432/agentkit" # 3) 更新密钥后端（key=database-url） # 4) restart pod（连接池重建） -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 5) 验证：触发一次 DB 操作（如 GET /api/v1/memory） @@ -117,7 +118,7 @@ NEW_CERTS='{"feishu":"-----BEGIN CERTIFICATE-----\n新证书\n-----END CERTIFICA # 3) 更新密钥后端（key=idp-signing-certs） # 4) restart pod（证书热轮换不重启，但 Helm 部署需 restart 加载新 Secret） -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 5) 验证：通过该 IdP 发起一次 SSO 登录 ``` @@ -141,7 +142,7 @@ NEW_KEYS='{"dashscope":"sk-new1","deepseek":"sk-new2","openai":"sk-old"}' # 3) 更新密钥后端（key=third-party-api-keys） # 4) restart pod -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 5) 验证：调用对应 provider 的模型（如 agentkit chat 发一条测试消息） @@ -196,7 +197,7 @@ NEW_PEM="$(cat client.crt client.key ca.crt)" # 3) 更新密钥后端 # 4) restart pod -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 5) 验证：触发一次 mTLS 调用（如 KMS 签名操作） ``` @@ -218,7 +219,7 @@ kubectl rollout restart deployment/agentkit-agentkit -n agentkit NEW_PIN='' # 3) restart pod -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 4) 验证：触发一次 HSM 操作（如 PKCS#11 签名） ``` @@ -238,11 +239,13 @@ kubectl rollout restart deployment/agentkit-agentkit -n agentkit ```bash # 1) 在 OTel collector / 可观测性平台生成新 token -# 2) 更新密钥后端（key=otel-exporter-token，格式 'Authorization: Bearer '） -NEW_TOKEN='Bearer new-otel-token' +# 2) 更新密钥后端（key=otel-exporter-token） +# DEPLOY-5: OTEL_EXPORTER_OTLP_HEADERS 期望 'key=value' 格式（OTel SDK 规范）， +# 不是裸 'Bearer '。secret 值必须形如 'Authorization=Bearer '。 +NEW_TOKEN='Authorization=Bearer new-otel-token' # 3) restart pod -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 4) 验证：在 collector 端确认收到新 trace/metrics ``` @@ -270,10 +273,10 @@ psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" # 需同步更新 requirepass 并 restart Redis） # 5) restart agentkit pod -kubectl rollout restart deployment/agentkit-agentkit -n agentkit +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit # 6) 验证 -kubectl exec -n agentkit deploy/agentkit-agentkit -- \ +kubectl exec -n agentkit deploy/agentkit -- \ python -c "import redis,os; redis.from_url(os.environ['REDIS_URL'],password=os.environ['REDIS_PASSWORD']).ping()" ``` diff --git a/docs/deployment/secret-inventory.md b/docs/deployment/secret-inventory.md index 371e752..5a41688 100644 --- a/docs/deployment/secret-inventory.md +++ b/docs/deployment/secret-inventory.md @@ -19,9 +19,9 @@ | 6 | TLS 服务器证书 | `tls-server-cert` | （Ingress TLS Secret） | Ingress TLS 终止证书 + 私钥 | 证书签发机构 / cert-manager | 365 天 | 平台运维 | | 7 | mTLS 客户端证书 | `mtls-client-cert` | `MTLS_CLIENT_CERT` | 与下游服务（KMS/HSM、内部 API）双向 TLS 客户端证书 | 内部 PKI | 180 天 | 安全团队 | | 8 | KMS/HSM PKCS#11 PIN | `kms-hsm-pin` | `KMS_HSM_PIN` | 硬件安全模块 PKCS#11 访问 PIN | HSM 管理员 | 365 天 | HSM 运维 | -| 9 | OTel exporter token | `otel-exporter-token` | `OTEL_EXPORTER_OTLP_HEADERS` | OTLP collector 鉴权 bearer token | 可观测性平台 | 90 天 | 可观测性团队 | -| 10 | Redis 密码 | `redis-password` | `REDIS_PASSWORD` | Redis requirepass（基础设施层） | DBA / 缓存运维 | 90 天 | DBA | -| 11 | PostgreSQL 密码 | `postgres-password` | `POSTGRES_PASSWORD` | PostgreSQL POSTGRES_PASSWORD（基础设施层） | DBA | 90 天 | DBA | +| 9 | OTel exporter token | `otel-exporter-token` | `OTEL_EXPORTER_OTLP_HEADERS` | OTLP collector 鉴权（DEPLOY-5: 值格式 `Authorization=Bearer `，非裸 Bearer） | 可观测性平台 | 90 天 | 可观测性团队 | +| 10 | Redis 密码 | `redis-password` | `REDIS_PASSWORD` | Redis requirepass（基础设施层，应用通过 env 消费） | DBA / 缓存运维 | 90 天 | DBA | +| 11 | PostgreSQL 密码 | `postgres-password` | `POSTGRES_PASSWORD`（仅 PG StatefulSet） | PostgreSQL POSTGRES_PASSWORD（DEPLOY-4: 应用不读此 env，通过 DATABASE_URL Slot 3 内嵌密码连接） | DBA | 90 天 | DBA | ## 风险等级 diff --git a/src/agentkit/bus/redis_bus.py b/src/agentkit/bus/redis_bus.py index 4017f25..36432f2 100644 --- a/src/agentkit/bus/redis_bus.py +++ b/src/agentkit/bus/redis_bus.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio import json import logging +import os from typing import Callable, Awaitable from agentkit.bus.message import AgentMessage @@ -41,7 +42,13 @@ class RedisMessageBus: """获取 Redis 连接（懒初始化）。""" if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _stream_key(self, agent_name: str) -> str: @@ -101,7 +108,10 @@ class RedisMessageBus: # Create consumer group if not exists try: await redis.xgroup_create( - stream_key, self._consumer_group, id="0", mkstream=True, + stream_key, + self._consumer_group, + id="0", + mkstream=True, ) except asyncio.CancelledError: raise @@ -141,7 +151,11 @@ class RedisMessageBus: logger.warning(f"Failed to process message {msg_id}: {e}") # Move to dead letter after max retries await self._handle_failed_message( - redis, stream_key, msg_id, fields, agent_name, + redis, + stream_key, + msg_id, + fields, + agent_name, ) except asyncio.CancelledError: break @@ -194,9 +208,7 @@ class RedisMessageBus: await self.publish(message) return await asyncio.wait_for(future, timeout=timeout) except asyncio.TimeoutError: - raise TimeoutError( - f"Request {message.correlation_id} timed out after {timeout}s" - ) + raise TimeoutError(f"Request {message.correlation_id} timed out after {timeout}s") finally: self._pending_requests.pop(message.correlation_id, None) @@ -256,6 +268,7 @@ def create_message_bus( if backend == "redis": try: import redis.asyncio as aioredis # noqa: F401 + bus = RedisMessageBus( redis_url=redis_url, consumer_group=consumer_group, @@ -267,8 +280,7 @@ def create_message_bus( raise except Exception as exc: # noqa: BLE001 — factory fallback to InMemoryMessageBus; must catch import/init errors broadly logger.warning( - f"Failed to initialise RedisMessageBus ({exc}), " - f"falling back to InMemoryMessageBus" + f"Failed to initialise RedisMessageBus ({exc}), falling back to InMemoryMessageBus" ) bus = InMemoryMessageBus() diff --git a/src/agentkit/core/base.py b/src/agentkit/core/base.py index 75c6629..51036c3 100644 --- a/src/agentkit/core/base.py +++ b/src/agentkit/core/base.py @@ -10,6 +10,7 @@ import asyncio import json import logging +import os import time from abc import ABC, abstractmethod from collections.abc import AsyncGenerator @@ -266,7 +267,12 @@ class BaseAgent(ABC): if redis_url: try: - self._redis = aioredis.from_url(redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) await self._redis.ping() logger.info(f"Agent '{self.name}' connected to Redis") except (ConnectionError, OSError, asyncio.TimeoutError, ValueError) as e: diff --git a/src/agentkit/core/config_driven.py b/src/agentkit/core/config_driven.py index 33a762d..0d461aa 100644 --- a/src/agentkit/core/config_driven.py +++ b/src/agentkit/core/config_driven.py @@ -394,7 +394,12 @@ class ConfigDrivenAgent(BaseAgent, EvolutionMixin): import redis.asyncio as aioredis redis_url = config.memory["working"].get("redis_url", "redis://localhost:6379") - redis_client = aioredis.from_url(redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + redis_client = aioredis.from_url( + redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) working = WorkingMemory(redis=redis_client) if config.memory.get("episodic", {}).get("enabled"): diff --git a/src/agentkit/core/handoff_transport.py b/src/agentkit/core/handoff_transport.py index 5a2bf83..af093f0 100644 --- a/src/agentkit/core/handoff_transport.py +++ b/src/agentkit/core/handoff_transport.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio import json import logging +import os from typing import AsyncIterator, Awaitable, Callable import redis.asyncio as aioredis @@ -144,7 +145,12 @@ class RedisHandoffTransport: """确保 Redis 连接已建立（懒初始化）""" if self._redis is None: try: - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) await self._redis.ping() logger.info("RedisHandoffTransport connected to Redis") except Exception: @@ -199,9 +205,7 @@ class RedisHandoffTransport: # 启动后台监听任务 if channel not in self._listen_tasks: - self._listen_tasks[channel] = asyncio.create_task( - self._background_listen(channel) - ) + self._listen_tasks[channel] = asyncio.create_task(self._background_listen(channel)) async def _background_listen(self, channel: str) -> None: """后台监听频道消息并调用处理器""" diff --git a/src/agentkit/llm/cache.py b/src/agentkit/llm/cache.py index 0919920..46e3508 100644 --- a/src/agentkit/llm/cache.py +++ b/src/agentkit/llm/cache.py @@ -11,6 +11,7 @@ Design doc: docs/plans/2026-06-14-002-u1-llm-cache-architecture.md import json import logging +import os import time from collections import OrderedDict from dataclasses import dataclass, field @@ -386,7 +387,12 @@ class RedisLLMCache: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis async def aclose(self) -> None: diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index dbc9cf5..8e2299a 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -2,6 +2,7 @@ import asyncio import logging +import os import time from datetime import datetime, timezone from pathlib import Path @@ -84,6 +85,12 @@ class LLMGateway: self._usage_tracker = UsageTracker(store=usage_store) if usage_store else UsageTracker() self._config = config or LLMConfig() + # PERF-1: PII 脱敏接入生产路径（env 驱动，默认关闭以保持向后兼容）。 + # AGENTKIT_PII_FILTER_ENABLED=true 启用；filter_messages_pii fail-closed。 + self._pii_filter_enabled = ( + os.environ.get("AGENTKIT_PII_FILTER_ENABLED", "false").lower() == "true" + ) + # Cache (U17 — LiteLLM 缓存管理器，opt-in，默认禁用) self._cache_manager: "LitellmCacheManager | None" = None if self._config.cache and self._config.cache.enabled: @@ -97,6 +104,22 @@ class LLMGateway: f"similarity_threshold={litellm_config.similarity_threshold})" ) + if self._pii_filter_enabled: + logger.info("PII filter enabled (AGENTKIT_PII_FILTER_ENABLED=true, fail-closed)") + + def _apply_pii_filter(self, messages: list[dict[str, str]]) -> list[dict[str, str]]: + """PERF-1: 对 messages 执行 PII 脱敏。 + + fail-closed：脱敏失败抛 PIIFilterError，由调用方转 HTTP 422。 + 禁用时原样返回（零开销 — 仅一次 env 读 + bool 判断）。 + """ + if not self._pii_filter_enabled: + return messages + from agentkit.llm.pii_filter import filter_messages_pii + + redacted, _matches = filter_messages_pii(messages, fail_closed=True) + return redacted + def register_provider(self, name: str, provider: LLMProvider) -> None: """注册 Provider""" self._providers[name] = provider @@ -129,6 +152,9 @@ class LLMGateway: if not self._providers: raise LLMProviderError("", "No provider registered") + # PERF-1: PII 脱敏（fail-closed）— 在 provider 调用前替换 messages。 + messages = self._apply_pii_filter(messages) + # ── Quota enforcement ── # Only enforce when department_ids + db_path are provided # (other call sites pass None — no quota check). @@ -349,6 +375,9 @@ class LLMGateway: if not self._providers: raise LLMProviderError("", "No provider registered") + # PERF-1: PII 脱敏（fail-closed）— 在 provider 调用前替换 messages。 + messages = self._apply_pii_filter(messages) + # ── Quota enforcement ── if department_ids and db_path: await self._enforce_quota(db_path, department_ids, resolved_model) diff --git a/src/agentkit/llm/providers/usage_store.py b/src/agentkit/llm/providers/usage_store.py index 3bd634d..bd0ee0d 100644 --- a/src/agentkit/llm/providers/usage_store.py +++ b/src/agentkit/llm/providers/usage_store.py @@ -16,6 +16,7 @@ Legacy v1 keys (still readable for backward compat): import asyncio import json import logging +import os from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Protocol, runtime_checkable @@ -304,7 +305,12 @@ class RedisUsageStore: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _get_sync_redis(self): @@ -312,7 +318,12 @@ class RedisUsageStore: if self._sync_redis is None: import redis as sync_redis - self._sync_redis = sync_redis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._sync_redis = sync_redis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._sync_redis async def aclose(self) -> None: diff --git a/src/agentkit/orchestrator/pipeline_state.py b/src/agentkit/orchestrator/pipeline_state.py index cb2a6ce..8907bfc 100644 --- a/src/agentkit/orchestrator/pipeline_state.py +++ b/src/agentkit/orchestrator/pipeline_state.py @@ -11,6 +11,7 @@ from __future__ import annotations import json import logging +import os import uuid from datetime import datetime, timezone from typing import Callable, Coroutine @@ -176,9 +177,11 @@ class PipelineStateRedis: if self._redis is None: import redis.asyncio as aioredis + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._redis = aioredis.from_url( self._redis_url, decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._redis diff --git a/src/agentkit/quality/cascade_state_store.py b/src/agentkit/quality/cascade_state_store.py index 04199ea..6a8b894 100644 --- a/src/agentkit/quality/cascade_state_store.py +++ b/src/agentkit/quality/cascade_state_store.py @@ -10,6 +10,7 @@ Key schema (Redis): """ import logging +import os import time from typing import Protocol, runtime_checkable @@ -140,8 +141,12 @@ class RedisCascadeStateStore: """Get or create a persistent sync Redis client (connection pool backed).""" if self._sync_redis is None: import redis as sync_redis + + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._sync_redis = sync_redis.from_url( - self._redis_url, decode_responses=True + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._sync_redis @@ -163,7 +168,15 @@ class RedisCascadeStateStore: pipe.expire(key, self._session_ttl) results = pipe.execute() return results[0] - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade increment failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -177,7 +190,15 @@ class RedisCascadeStateStore: r = self._get_sync_redis() val = r.get(f"{self.INTER_PREFIX}{session_id}") return int(val) if val is not None else 0 - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade get failed: {e}") if self._fallback is not None: return self._fallback.get_interaction(session_id) @@ -194,7 +215,15 @@ class RedisCascadeStateStore: pipe.set(key, depth) pipe.expire(key, self._session_ttl) pipe.execute() - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade set_depth failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -207,7 +236,15 @@ class RedisCascadeStateStore: r = self._get_sync_redis() val = r.get(f"{self.DEPTH_PREFIX}{session_id}") return int(val) if val is not None else 0 - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade get_depth failed: {e}") if self._fallback is not None: return self._fallback.get_depth(session_id) @@ -223,7 +260,15 @@ class RedisCascadeStateStore: pipe.delete(f"{self.INTER_PREFIX}{session_id}") pipe.delete(f"{self.DEPTH_PREFIX}{session_id}") pipe.execute() - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade reset failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -250,6 +295,7 @@ def create_cascade_state_store( if backend in ("auto", "redis"): try: import redis # noqa: F401 + return RedisCascadeStateStore(redis_url=redis_url, session_ttl=session_ttl) except ImportError: logger.warning("redis package not available, falling back to in-memory cascade store") diff --git a/src/agentkit/server/app.py b/src/agentkit/server/app.py index b6b7ef0..6efdbbb 100644 --- a/src/agentkit/server/app.py +++ b/src/agentkit/server/app.py @@ -1258,11 +1258,13 @@ def create_app( "redis_url", "redis://localhost:6379" ) # U5c: socket_timeout 防止单点 Redis 故障时网关请求挂死。 + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 redis_client = aioredis.from_url( redis_url, decode_responses=True, socket_timeout=5.0, socket_connect_timeout=5.0, + password=os.environ.get("REDIS_PASSWORD") or None, ) working = WorkingMemory(redis=redis_client) app.state.working_redis_client = redis_client @@ -1396,9 +1398,14 @@ def create_app( app.include_router(auth_routes.router, prefix="/api/v1") app.include_router(auth_routes.admin_router, prefix="/api/v1") # U4 SSO 集成 — SCIM 2.0 最小子集（自带 bearer token 校验） - from agentkit.server.auth.scim.router import router as scim_router + from agentkit.server.auth.scim.router import ( + register_scim_exception_handlers, + router as scim_router, + ) app.include_router(scim_router, prefix="/api/v1") + # API-2: 注册 SCIM 异常处理器（scope 到 /scim/v2，非 SCIM 路由走默认格式） + register_scim_exception_handlers(app) app.include_router(admin_routes_module.admin_router, prefix="/api/v1") app.include_router(documents.router, prefix="/api/v1") app.include_router(calendar_routes.router, prefix="/api/v1") diff --git a/src/agentkit/server/auth/scim/models.py b/src/agentkit/server/auth/scim/models.py index fe64298..55b8f59 100644 --- a/src/agentkit/server/auth/scim/models.py +++ b/src/agentkit/server/auth/scim/models.py @@ -1,13 +1,11 @@ """SCIM 2.0 用户 schema (Pydantic v2) — U4 最小子集。 仅实现 RFC 7643 / 7644 中创建 / 禁用 / 查询所需的字段： -``id``, ``userName``, ``active``, ``emails``, ``displayName``。 +``id``, ``userName``, ``active``, ``emails``, ``displayName``, ``externalId``。 """ from __future__ import annotations -from typing import Any - from pydantic import BaseModel, ConfigDict, EmailStr, Field @@ -22,7 +20,11 @@ class SCIMEmail(BaseModel): class SCIMUser(BaseModel): - """SCIM 2.0 User 资源（最小子集）。""" + """SCIM 2.0 User 资源（最小子集）。 + + STAND-1: 移除 ``Any`` — ``externalId`` 是 RFC 7643 标准 attribute， + 用于 SCIM 预配的用户与 SSO IDP subject 关联（API-5）。 + """ model_config = ConfigDict(extra="allow") @@ -30,17 +32,22 @@ class SCIMUser(BaseModel): userName: str active: bool = True displayName: str | None = None + externalId: str | None = None emails: list[SCIMEmail] = Field(default_factory=list) class SCIMPatchOp(BaseModel): - """SCIM 2.0 PATCH 操作（最小子集，仅支持 replace）。""" + """SCIM 2.0 PATCH 操作（最小子集，仅支持 replace）。 + + STAND-1: ``value`` 用 ``object`` 替代 ``Any``（Pydantic v2 中 ``object`` 等价 + 于旧 ``Any`` 但更明确表达"任意 JSON 值"的语义）。 + """ model_config = ConfigDict(extra="allow") op: str = "replace" path: str | None = None - value: Any = None + value: object = None class SCIMPatchRequest(BaseModel): @@ -52,7 +59,10 @@ class SCIMPatchRequest(BaseModel): class SCIMListResponse(BaseModel): - """SCIM 2.0 列表响应。""" + """SCIM 2.0 列表响应。 + + STAND-1: ``Resources`` 用 ``list[dict[str, object]]`` 替代 ``list[dict[str, Any]]``。 + """ model_config = ConfigDict(extra="allow") @@ -62,7 +72,7 @@ class SCIMListResponse(BaseModel): totalResults: int = 0 startIndex: int = 1 itemsPerPage: int = 0 - Resources: list[dict[str, Any]] = Field(default_factory=list) + Resources: list[dict[str, object]] = Field(default_factory=list) class SCIMErrorDetail(BaseModel): diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py index d53057c..b960442 100644 --- a/src/agentkit/server/auth/scim/router.py +++ b/src/agentkit/server/auth/scim/router.py @@ -1,12 +1,13 @@ """SCIM 2.0 路由 (U4 最小子集) — handler 内联，不拆 handlers.py。 端点： -- POST /scim/v2/Users — 创建用户 +- POST /scim/v2/Users — 创建用户（同步写 user_idp_links 供 SSO 联动） - PATCH /scim/v2/Users/{id} — 禁用 (active=false) / 更新 - GET /scim/v2/Users — 列表 / 过滤 Bearer token 认证（独立于 JWT — ``auth.scim_token`` 配置项）。 SCIM push 失败触发实时告警（日志 error + OTel span event）。 +RFC 7643/7644 合规：错误响应用 SCIMErrorResponse 格式 + Location 头 + totalResults 全量总数。 """ from __future__ import annotations @@ -17,12 +18,13 @@ import os import secrets import uuid from pathlib import Path -from typing import Any import aiosqlite from fastapi import APIRouter, Depends, HTTPException, Query, Request, status -from pydantic import BaseModel, ConfigDict +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from ...auth.audit import SOURCE_SCIM, get_audit_service from ...auth.models import ( init_auth_db, resolve_auth_db_path as _resolve_default_db_path, @@ -37,6 +39,7 @@ logger = logging.getLogger(__name__) router = APIRouter(prefix="/scim/v2", tags=["scim"]) _SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" +_SCIM_ERROR_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:Error" def _resolve_db_path(request: Request) -> Path: @@ -98,30 +101,108 @@ async def _ensure_db(request: Request) -> Path: return db_path -def _user_to_scim(row: dict[str, object]) -> dict[str, Any]: - """本地 user dict → SCIM 2.0 User JSON。""" +# --------------------------------------------------------------------------- +# RFC 7644 3.12 — SCIM 错误响应格式 +# --------------------------------------------------------------------------- + + +def _scim_error_response(status_code: int, detail: str) -> JSONResponse: + """API-2: 构造 RFC 7644 3.12 规定的 SCIM 错误响应格式。""" + return JSONResponse( + status_code=status_code, + content={ + "schemas": [_SCIM_ERROR_SCHEMA], + "status": str(status_code), + "detail": detail, + }, + ) + + +# API-2: APIRouter 在旧版 FastAPI 无 exception_handler 方法，改为导出注册函数 +# 由 app.py 在挂载 SCIM router 后调用，scope 到 /scim/v2 路径。 + + +def register_scim_exception_handlers(app) -> None: + """注册 SCIM 异常处理器 — 仅对 /scim/v2 路径生效，不影响其他路由。 + + API-2: APIRouter 在旧版 FastAPI 无 exception_handler 方法，改为在 app 上注册 + 并通过 request.url.path scope 到 SCIM 路径。非 SCIM 路由走 FastAPI 默认格式。 + """ + + @app.exception_handler(HTTPException) + async def _scim_http_handler(request: Request, exc: HTTPException): + path = request.url.path + is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") + if not is_scim: + # 非 SCIM 路由 — 复制 FastAPI 默认 HTTPException handler 行为（含 headers） + headers = getattr(exc, "headers", None) + return JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + headers=headers, + ) + return _scim_error_response(exc.status_code, str(exc.detail)) + + @app.exception_handler(RequestValidationError) + async def _scim_validation_handler(request: Request, exc: RequestValidationError): + path = request.url.path + is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") + if not is_scim: + # 非 SCIM 路由 — 复制 FastAPI 默认 RequestValidationError handler 行为 + from fastapi.encoders import jsonable_encoder + + return JSONResponse( + status_code=422, + content={"detail": jsonable_encoder(exc.errors())}, + ) + return _scim_error_response(400, f"请求体校验失败: {exc}") + + +# --------------------------------------------------------------------------- +# SCIM User 资源转换 — STAND-1: 用 dict[str, object] 替代 Any +# --------------------------------------------------------------------------- + + +def _user_to_scim(row: dict[str, object]) -> dict[str, object]: + """本地 user dict → SCIM 2.0 User JSON。 + + API-11: displayName 改为 None（让客户端用 userName fallback）， + 因为 users 表无 name/full_name 列，username 不是 displayName 的合理来源。 + """ email = str(row.get("email", "") or "") return { "schemas": [_SCIM_USER_SCHEMA], "id": str(row["id"]), "userName": str(row["username"]), "active": bool(row.get("is_active", True)), - "displayName": str(row.get("username", "")), + "displayName": None, "emails": [{"value": email, "type": "work", "primary": True}] if email else [], } +# --------------------------------------------------------------------------- +# SCIM 端点 +# --------------------------------------------------------------------------- + + @router.post("/Users", status_code=status.HTTP_201_CREATED) async def create_user( payload: SCIMUser, request: Request, _token: str = Depends(_verify_scim_token), -) -> dict[str, Any]: - """SCIM POST /Users — 创建本地用户（随机密码，不可本地登录）。""" +) -> JSONResponse: + """SCIM POST /Users — 创建本地用户（随机密码，不可本地登录）。 + + API-5: 同时写 user_idp_links 行，让 SCIM 预配的用户能通过 SSO 登录， + 避免 OIDC JIT 创建重复账户。externalId 作为 idp_subject， + 缺省时用 userName。 + """ db_path = await _ensure_db(request) user_id = str(uuid.uuid4()) now = _now_iso() email = payload.emails[0].value if payload.emails else f"{user_id}@scim.local" + # externalId 是 RFC 7643 标准 attribute，作为 IDP subject 关联本地用户 + external_id = getattr(payload, "externalId", None) or payload.userName try: async with aiosqlite.connect(str(db_path)) as db: db.row_factory = aiosqlite.Row @@ -144,18 +225,32 @@ async def create_user( now, ), ) + # API-5: 写 user_idp_links 行，idp_name='scim'，idp_subject=externalId + # 表无 id 列（PK = idp_name + idp_subject），省去 uuid 生成。 + await db.execute( + "INSERT INTO user_idp_links " + "(idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, ?, ?)", + ("scim", external_id, user_id, "scim", now), + ) await db.commit() cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) row = await cursor.fetchone() assert row is not None - logger.info("SCIM 创建用户 userName=%s id=%s", payload.userName, user_id) - return _user_to_scim(user_row_to_dict(row)) + logger.info( + "SCIM 创建用户 userName=%s id=%s externalId=%s", payload.userName, user_id, external_id + ) + # API-6: RFC 7644 3.14 强制要求 Location 头 + body = _user_to_scim(user_row_to_dict(row)) + response = JSONResponse(status_code=status.HTTP_201_CREATED, content=body) + response.headers["Location"] = f"/scim/v2/Users/{user_id}" + return response except aiosqlite.IntegrityError as exc: _alert_scim_failure("create", f"用户名或邮箱冲突: {exc}", user_id) - raise HTTPException(status_code=409, detail="userName 或 email 已存在") from exc + return _scim_error_response(409, "userName 或 email 已存在") except Exception as exc: # noqa: BLE001 _alert_scim_failure("create", str(exc), user_id) - raise HTTPException(status_code=500, detail="SCIM 创建失败") from exc + return _scim_error_response(500, "SCIM 创建失败") @router.patch("/Users/{user_id}") @@ -164,17 +259,24 @@ async def patch_user( payload: SCIMPatchRequest, request: Request, _token: str = Depends(_verify_scim_token), -) -> dict[str, Any]: - """SCIM PATCH /Users/{id} — 禁用 (active=false) 或更新字段。""" +) -> dict[str, object]: + """SCIM PATCH /Users/{id} — 禁用 (active=false) 或更新字段。 + + API-7: userName UNIQUE 冲突时返回 409（而非 500）。 + SEC-3: active true→false 触发审计记录（与 admin deprovision 对齐）。 + """ db_path = await _ensure_db(request) now = _now_iso() + # SEC-3: 收集 deprovision 审计信息，在事务提交后调用（避免 SQLite 写锁竞争） + _pending_audit: dict[str, object] | None = None try: async with aiosqlite.connect(str(db_path)) as db: db.row_factory = aiosqlite.Row cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) row = await cursor.fetchone() if row is None: - raise HTTPException(status_code=404, detail="用户不存在") + return _scim_error_response(404, "用户不存在") + prev_active = bool(row["is_active"]) for op in payload.Operations: if op.op.lower() == "replace": if op.path in (None, "active"): @@ -187,21 +289,41 @@ async def patch_user( "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", (1 if active else 0, now, user_id), ) + # SEC-3: active true→false 写审计（延后到事务提交后） + if prev_active and not active: + _pending_audit = { + "target_user_id": user_id, + "target_username": str(row["username"]), + } elif op.path == "userName": - await db.execute( - "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", - (str(op.value)[:64], now, user_id), - ) + try: + await db.execute( + "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", + (str(op.value)[:64], now, user_id), + ) + except aiosqlite.IntegrityError: + # API-7: UNIQUE 冲突返回 409 + return _scim_error_response(409, "userName 已存在") await db.commit() cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) row = await cursor.fetchone() assert row is not None + # SEC-3: 事务提交后写审计（避免 SQLite "database is locked"） + if _pending_audit is not None: + await get_audit_service().record_deprovision( + actor_user_id=None, + actor_username="scim", + target_user_id=str(_pending_audit["target_user_id"]), + target_username=str(_pending_audit["target_username"]), + reason="scim_patch_active_false", + source=SOURCE_SCIM, + ) return _user_to_scim(user_row_to_dict(row)) except HTTPException: raise except Exception as exc: # noqa: BLE001 _alert_scim_failure("patch", str(exc), user_id) - raise HTTPException(status_code=500, detail="SCIM 更新失败") from exc + return _scim_error_response(500, "SCIM 更新失败") @router.get("/Users") @@ -211,44 +333,49 @@ async def list_users( startIndex: int = Query(default=1, ge=1), count: int = Query(default=100, ge=1, le=200), _token: str = Depends(_verify_scim_token), -) -> dict[str, Any]: +) -> dict[str, object]: """SCIM GET /Users — 列表 / 过滤。 - ponytail: filter 仅支持最简形式 ``userName eq "xxx"`` / ``active eq true``。 - 上限：复杂 SCIM filter 表达式需扩展为完整 AST 解析器。 + API-1: totalResults 返回全量总数（COUNT 查询），而非当前页条数。 + API-9: 不支持的 filter 语法返回 400 + invalidFilter，不静默全量返回。 + API-13: filter 解析大小写不敏感（RFC 7644 3.4.2.2）。 """ db_path = await _ensure_db(request) where = "" - params: list[Any] = [] + params: list[object] = [] if filter: - # 最简 filter 解析：userName eq "x" / active eq true - if "userName eq" in filter: - val = filter.split("userName eq", 1)[1].strip().strip('"').strip("'") + filter_lower = filter.lower() + # API-13: 大小写不敏感匹配 + if "username eq" in filter_lower: + # 从原 filter 中提取引号内的值（保留原大小写） + val = filter.split(" eq", 1)[1].strip().strip('"').strip("'") where = "WHERE username = ?" params.append(val) - elif "active eq" in filter: - val = filter.split("active eq", 1)[1].strip().strip('"').strip("'").lower() + elif "active eq" in filter_lower: + val = filter.split(" eq", 1)[1].strip().strip('"').strip("'").lower() where = "WHERE is_active = ?" params.append(1 if val in ("true", "1") else 0) - sql = f"SELECT * FROM users {where} ORDER BY created_at ASC LIMIT ? OFFSET ?" - params.extend([count, startIndex - 1]) + else: + # API-9: 不支持的 filter 返回 400 + return _scim_error_response( + 400, f"Unsupported filter syntax (scimType=invalidFilter): {filter}" + ) + # API-1: 先 COUNT 全量总数 + count_sql = f"SELECT COUNT(*) FROM users {where}" + list_sql = f"SELECT * FROM users {where} ORDER BY created_at ASC LIMIT ? OFFSET ?" + list_params = [*params, count, startIndex - 1] async with aiosqlite.connect(str(db_path)) as db: db.row_factory = aiosqlite.Row - cursor = await db.execute(sql, tuple(params)) + # totalResults + cursor = await db.execute(count_sql, tuple(params)) + total_results = (await cursor.fetchone())[0] + # 当前页 + cursor = await db.execute(list_sql, tuple(list_params)) rows = await cursor.fetchall() resources = [_user_to_scim(user_row_to_dict(r)) for r in rows] return SCIMListResponse( - totalResults=len(resources), + totalResults=total_results, startIndex=startIndex, itemsPerPage=len(resources), Resources=resources, ).model_dump() - - -class SCIMErrorResponse(BaseModel): - """SCIM 错误响应包装。""" - - model_config = ConfigDict(extra="allow") - - status: str - detail: str | None = None diff --git a/src/agentkit/server/routes/auth.py b/src/agentkit/server/routes/auth.py index 62871a4..ac77089 100644 --- a/src/agentkit/server/routes/auth.py +++ b/src/agentkit/server/routes/auth.py @@ -23,6 +23,7 @@ from __future__ import annotations import hmac import logging +import secrets from datetime import datetime, timezone from pathlib import Path from typing import Any, Literal @@ -794,8 +795,20 @@ class SSORedirectResponse(BaseModel): state: str -async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) -> TokenResponse: - """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。""" +async def _sso_issue_token_pair( + request: Request, + user_dict: dict[str, object], + *, + provider_type: str = "sso", + provider_name: str | None = None, +) -> TokenResponse: + """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。 + + API-4: 一次性生成真实 refresh_token 再创建 session（避免占位符并发冲突）。 + API-8: auth_provider 携带 provider_type + provider_name（如 'oidc-keycloak'）， + 供 admin session 列表按 IDP 筛选 + 审计追溯。 + SEC-1: 与本地 login 一致地校验 is_active。 + """ # SEC-1: SSO 流程必须与本地 login（auth.py:377）一致地校验 is_active， # 否则已 deprovision 的用户可通过 IDP 回调重新拿到 JWT + session。 if not bool(user_dict.get("is_active", True)): @@ -805,15 +818,21 @@ async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) svc: SessionService = get_session_service() info = _client_info(request) - pre_session = await svc.create( + # API-4: 先生成真实 refresh_token，避免占位符 "__pending_sso__" 在并发 SSO + # 登录时触发 refresh_token_hash UNIQUE 约束冲突。 + refresh_token = secrets.token_urlsafe(32) + # API-8: auth_provider 形如 "oidc-keycloak" / "saml-azuread" / "sso" + auth_provider = f"{provider_type}-{provider_name}" if provider_name else provider_type + + session = await svc.create( SessionCreate( user_id=str(user_dict["id"]), - refresh_token="__pending_sso__", + refresh_token=refresh_token, device_fingerprint=info["fingerprint"], device_label=info["label"], ip=info["ip"], user_agent=info["user_agent"], - auth_provider="sso", + auth_provider=auth_provider, ttl_seconds=_refresh_ttl_seconds(False), ) ) @@ -822,13 +841,9 @@ async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) username=str(user_dict["username"]), role=str(user_dict["role"]), secret=secret, - session_id=pre_session.id, + session_id=session.id, ) async with aiosqlite.connect(str(db_path)) as db: - await db.execute( - "UPDATE auth_sessions SET refresh_token_hash = ? WHERE id = ?", - (hash_token(token_pair.refresh_token), pre_session.id), - ) await db.execute( "UPDATE users SET last_login_at = ?, updated_at = ? WHERE id = ?", (_now_iso(), _now_iso(), str(user_dict["id"])), @@ -849,7 +864,7 @@ async def _sso_issue_token_pair(request: Request, user_dict: dict[str, object]) user_dict.get("is_server_terminal_authorized", False) ), ), - session_id=pre_session.id, + session_id=session.id, ) @@ -886,7 +901,9 @@ async def oidc_callback( user_dict = await OIDCProvider().handle_callback(provider, code, state) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - return await _sso_issue_token_pair(request, user_dict) + return await _sso_issue_token_pair( + request, user_dict, provider_type="oidc", provider_name=provider + ) class SAMLACSRequest(BaseModel): @@ -915,7 +932,9 @@ async def saml_acs( ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - return await _sso_issue_token_pair(request, user_dict) + return await _sso_issue_token_pair( + request, user_dict, provider_type="saml", provider_name=provider + ) # --------------------------------------------------------------------------- @@ -1041,13 +1060,33 @@ class DeprovisionRequest(BaseModel): break_glass: bool = False # 高权限账户 break-glass 标记 -@admin_router.post("/users/{user_id}/deprovision") +class DeprovisionResponse(BaseModel): + """API-10: deprovision 响应体（让 OpenAPI schema 可文档化）。""" + + model_config = ConfigDict(extra="forbid") + + deprovisioned: bool + revoked_sessions: int + + +class RoleChangeResponse(BaseModel): + """API-10: 角色变更响应体。""" + + model_config = ConfigDict(extra="forbid") + + role_changed: bool + role_before: str + role_after: str + revoked_sessions: int + + +@admin_router.post("/users/{user_id}/deprovision", response_model=DeprovisionResponse) async def deprovision_user( user_id: str, payload: DeprovisionRequest, request: Request, admin: dict[str, Any] = Depends(_require_admin), -) -> dict[str, Any]: +) -> DeprovisionResponse: """手动 deprovisioning (R1b) — operator/admin RBAC 强制。 1. 禁用本地用户 (is_active=0) @@ -1086,7 +1125,7 @@ async def deprovision_user( reason=payload.reason, source=source, ) - return {"deprovisioned": True, "revoked_sessions": revoked} + return DeprovisionResponse(deprovisioned=True, revoked_sessions=revoked) class RoleChangeRequest(BaseModel): @@ -1098,13 +1137,13 @@ class RoleChangeRequest(BaseModel): reason: str | None = None -@admin_router.post("/users/{user_id}/role") +@admin_router.post("/users/{user_id}/role", response_model=RoleChangeResponse) async def change_user_role( user_id: str, payload: RoleChangeRequest, request: Request, admin: dict[str, Any] = Depends(_require_admin), -) -> dict[str, Any]: +) -> RoleChangeResponse: """RBAC 角色变更 (KTD-8) — admin only。 记录 actor/target/role_before/role_after/reason/source/timestamp，接入 OTel 审计。 @@ -1139,12 +1178,12 @@ async def change_user_role( # 撤销所有 session（强制重新登录刷新 role claim） svc: SessionService = get_session_service() revoked = await svc.revoke_all_for_user(user_id, reason="role_changed") - return { - "role_changed": True, - "role_before": role_before, - "role_after": payload.role, - "revoked_sessions": revoked, - } + return RoleChangeResponse( + role_changed=True, + role_before=role_before, + role_after=payload.role, + revoked_sessions=revoked, + ) # Backwards-compat /me endpoint (kept for the existing frontend) diff --git a/src/agentkit/server/task_store.py b/src/agentkit/server/task_store.py index d1c9d42..69f69c1 100644 --- a/src/agentkit/server/task_store.py +++ b/src/agentkit/server/task_store.py @@ -3,6 +3,7 @@ import asyncio import json import logging +import os from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any @@ -15,6 +16,7 @@ logger = logging.getLogger(__name__) @dataclass class TaskRecord: """Stored task record with full lifecycle data""" + task_id: str agent_name: str skill_name: str | None @@ -57,9 +59,15 @@ class TaskRecord: status=TaskStatus(data.get("status", "pending")), output_data=data.get("output_data"), error_message=data.get("error_message"), - created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.now(timezone.utc), - started_at=datetime.fromisoformat(data["started_at"]) if data.get("started_at") else None, - completed_at=datetime.fromisoformat(data["completed_at"]) if data.get("completed_at") else None, + created_at=datetime.fromisoformat(data["created_at"]) + if data.get("created_at") + else datetime.now(timezone.utc), + started_at=datetime.fromisoformat(data["started_at"]) + if data.get("started_at") + else None, + completed_at=datetime.fromisoformat(data["completed_at"]) + if data.get("completed_at") + else None, progress=data.get("progress", 0.0), progress_message=data.get("progress_message", ""), metadata=data.get("metadata", {}), @@ -124,14 +132,19 @@ class InMemoryTaskStore: if expired: logger.info(f"TaskStore cleaned up {len(expired)} expired records") - def create(self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None) -> TaskRecord: + def create( + self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None + ) -> TaskRecord: """Create a new task record""" if len(self._tasks) >= self._max_records: # Remove oldest completed task oldest = None for rec in self._tasks.values(): if rec.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): - if oldest is None or (rec.completed_at and (oldest.completed_at is None or rec.completed_at < oldest.completed_at)): + if oldest is None or ( + rec.completed_at + and (oldest.completed_at is None or rec.completed_at < oldest.completed_at) + ): oldest = rec if oldest: del self._tasks[oldest.task_id] @@ -223,9 +236,11 @@ class RedisTaskStore: if self._redis is None: import redis.asyncio as aioredis + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._redis = aioredis.from_url( self._redis_url, decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._redis @@ -245,7 +260,9 @@ class RedisTaskStore: # ── CRUD ─────────────────────────────────────────────────── - async def create(self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None) -> TaskRecord: + async def create( + self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None + ) -> TaskRecord: """Create a new task record in Redis.""" redis = await self._get_redis() @@ -305,7 +322,9 @@ end return encoded """ - async def update_status(self, task_id: str, status: TaskStatus, reset_ttl: bool = False, **kwargs) -> TaskRecord: + async def update_status( + self, task_id: str, status: TaskStatus, reset_ttl: bool = False, **kwargs + ) -> TaskRecord: """Update task status and optional fields atomically via Lua script. Args: @@ -322,7 +341,15 @@ return encoded # Build flat list of key-value pairs for the merge fields merge_fields = {"status": status.value} for k, value in kwargs.items(): - if k in ("started_at", "completed_at", "output_data", "error_message", "progress", "progress_message", "metadata"): + if k in ( + "started_at", + "completed_at", + "output_data", + "error_message", + "progress", + "progress_message", + "metadata", + ): if isinstance(value, datetime): merge_fields[k] = value.isoformat() else: @@ -340,7 +367,9 @@ return encoded data = json.loads(result) return TaskRecord.from_dict(data) - async def list_tasks(self, status: TaskStatus | None = None, limit: int = 100) -> list[TaskRecord]: + async def list_tasks( + self, status: TaskStatus | None = None, limit: int = 100 + ) -> list[TaskRecord]: """List tasks, optionally filtered by status, sorted by created_at desc.""" redis = await self._get_redis() tasks: list[TaskRecord] = [] @@ -433,7 +462,11 @@ return encoded await redis.zrem(self.ZSET_KEY, tid) continue record = TaskRecord.from_dict(json.loads(raw)) - if record.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED) and record.completed_at is not None: + if ( + record.status + in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED) + and record.completed_at is not None + ): await redis.delete(self._key(tid)) await redis.zrem(self.ZSET_KEY, tid) return True @@ -452,7 +485,11 @@ return encoded if raw is None: continue record = TaskRecord.from_dict(json.loads(raw)) - if record.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): + if record.status in ( + TaskStatus.COMPLETED, + TaskStatus.FAILED, + TaskStatus.CANCELLED, + ): tasks.append(record) if cursor == 0: break @@ -505,7 +542,9 @@ def create_task_store( logger.info(f"TaskStore backend: redis ({_sanitize_redis_url(redis_url)})") return store except Exception as exc: - logger.warning(f"Failed to initialise RedisTaskStore ({exc}), falling back to InMemoryTaskStore") + logger.warning( + f"Failed to initialise RedisTaskStore ({exc}), falling back to InMemoryTaskStore" + ) store = InMemoryTaskStore(ttl_seconds=ttl_seconds, max_records=max_records) logger.info("TaskStore backend: memory") diff --git a/src/agentkit/session/store.py b/src/agentkit/session/store.py index 23b5f2e..4628d87 100644 --- a/src/agentkit/session/store.py +++ b/src/agentkit/session/store.py @@ -24,12 +24,18 @@ class SessionStore(Protocol): async def save_session(self, session: Session) -> None: ... async def get_session(self, session_id: str) -> Session | None: ... - async def update_session_status(self, session_id: str, status: SessionStatus) -> Session | None: ... + async def update_session_status( + self, session_id: str, status: SessionStatus + ) -> Session | None: ... async def delete_session(self, session_id: str) -> bool: ... - async def list_sessions(self, agent_name: str | None = None, limit: int = 100) -> list[Session]: ... + async def list_sessions( + self, agent_name: str | None = None, limit: int = 100 + ) -> list[Session]: ... async def append_message(self, message: Message) -> None: ... - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: ... + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: ... async def count_messages(self, session_id: str) -> int: ... async def health_check(self) -> bool: ... @@ -91,7 +97,9 @@ class InMemorySessionStore: del msgs[:excess] msgs.append(message) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: msgs = self._messages.get(session_id, []) sliced = msgs[offset:] if limit is not None: @@ -125,7 +133,13 @@ class RedisSessionStore: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: Helm 注入 REDIS_PASSWORD env var，redis-py from_url 不自动读取， + # 必须显式以 password= 传入。env var 未设置时 password=None 保持向后兼容。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _session_key(self, session_id: str) -> str: @@ -192,7 +206,9 @@ class RedisSessionStore: # Set TTL on message list to match session TTL await redis.expire(key, self._ttl_seconds) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: redis = await self._get_redis() key = self._messages_key(session_id) # Use LRANGE for offset-based pagination @@ -304,7 +320,9 @@ class FileSessionStore: data["session"]["updated_at"] = datetime.now(timezone.utc).isoformat() self._write_session_file(message.session_id, data) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: data = self._read_session_file(session_id) if data is None: return [] @@ -347,10 +365,12 @@ def create_session_store( import redis.asyncio as aioredis # noqa: F401 store = RedisSessionStore(redis_url=redis_url, ttl_seconds=ttl_seconds) - logger.info(f"SessionStore backend: redis") + logger.info("SessionStore backend: redis") return store except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError) as exc: - logger.warning(f"Failed to initialise RedisSessionStore ({exc}), falling back to InMemorySessionStore") + logger.warning( + f"Failed to initialise RedisSessionStore ({exc}), falling back to InMemorySessionStore" + ) store = InMemorySessionStore() logger.info("SessionStore backend: memory") From 04ad5966d071ae9b910aabb6f5d1dab113deeace Mon Sep 17 00:00:00 2001 From: Chiguyong Date: Mon, 6 Jul 2026 12:18:42 +0800 Subject: [PATCH 11/11] =?UTF-8?q?fix(enterprise):=20ce-debug=20P3=20residu?= =?UTF-8?q?al=20batch=20=E2=80=94=20SEC-2=20OIDC=20=E9=AA=8C=E7=AD=BE=20+?= =?UTF-8?q?=20STAND-3=20SCIM=20/scim/v2=20+=20MIG-2=20schema=20=E8=BF=81?= =?UTF-8?q?=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 13 项 P3 residual findings 全部清零： 安全/合规 - SEC-2: OIDC id_token 用 JWKS 验签 + IDToken claims_cls 校验 iss/aud/exp（fail-closed，无 jwks_uri 时拒绝回退） - STAND-3: SCIM router 改挂 /scim/v2（RFC 7644 标准路径）+ 修复双重前缀 bug（router 内部 prefix 移除，由挂载点控制） - API-12/14: SCIM PATCH 严格性 — 不支持的 op/path 返回 400 noTarget + path==None 时遍历 value keys 性能/可观测 - PERF-6/7: gateway cache metric 修正 — None usage 早退 + 仅对 anthropic/claude model 记录 cache hit/miss 部署/运维 - DEPLOY-8: key-rotation-runbook 拆分 Slot 10(Redis)/Slot 11(PostgreSQL) - DEPLOY-9: Helm chart 支持 image.digest pinning（@sha256: 覆盖 tag） 迁移/维护 - MIG-2: auth DB schema 迁移注册表（_SCHEMA_MIGRATIONS dict + _run_schema_migrations 按 version 差值执行） - MAINT-2: bitable delete_view 三态错误信息（不存在 vs 最后一视图 vs 并发竞争） - MAINT-3: chatStream 提取 findStreamingSynthesisMilestone helper（去重 team_synthesis_chunk/team_synthesis 谓词） - MAINT-4 + STAND-2: 前端测试 token 正则 helper + replace 加 g 标志（移除所有 :root 块） 验证：ruff check + format（P3 文件全过）| pytest 526 passed（calendar webhook SSRF 失败为 pre-existing）| helm lint 通过 --- .../helm/agentkit/templates/deployment.yaml | 5 + deploy/helm/agentkit/values.yaml | 4 + docs/deployment/key-rotation-runbook.md | 52 +- ...-enterprise-agent-platform-roadmap-plan.md | 622 ++++++++++++++++++ ...at-enterprise-agent-platform-p0-3cf29f4.md | 82 ++- src/agentkit/bitable/service.py | 13 +- src/agentkit/llm/gateway.py | 20 +- src/agentkit/server/app.py | 3 +- src/agentkit/server/auth/idp_registry.py | 5 + src/agentkit/server/auth/models.py | 27 +- src/agentkit/server/auth/providers/oidc.py | 52 +- src/agentkit/server/auth/scim/router.py | 112 +++- .../server/frontend/src/stores/chatStream.ts | 32 +- .../tests/unit/bitable-tokens.test.ts | 18 +- tests/unit/test_scim_router.py | 37 +- 15 files changed, 949 insertions(+), 135 deletions(-) create mode 100644 docs/plans/2026-07-05-001-feat-enterprise-agent-platform-roadmap-plan.md diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml index d61bbd8..407b613 100644 --- a/deploy/helm/agentkit/templates/deployment.yaml +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -26,7 +26,12 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: agentkit + # DEPLOY-9: digest pinning 优先于 tag（政企生产防篡改） + {{- if .Values.image.digest }} + image: "{{ .Values.image.repository }}@{{ .Values.image.digest }}" + {{- else }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml index 4caed7a..bd34227 100644 --- a/deploy/helm/agentkit/values.yaml +++ b/deploy/helm/agentkit/values.yaml @@ -9,6 +9,10 @@ image: repository: fischer-agentkit tag: 0.1.0 pullPolicy: IfNotPresent + # DEPLOY-9: 可选 digest pinning（政企生产场景防篡改）。 + # 设置后 image 引用变为 @sha256:，覆盖 tag。 + # 获取 digest: crane digest : 或 skopeo inspect --format '{{.Digest}}' docker://: + digest: "" # 私有镜像仓库时配置 pullSecret（secret 引用名，由外部创建） pullSecret: "" diff --git a/docs/deployment/key-rotation-runbook.md b/docs/deployment/key-rotation-runbook.md index 63df903..1daa99d 100644 --- a/docs/deployment/key-rotation-runbook.md +++ b/docs/deployment/key-rotation-runbook.md @@ -254,9 +254,9 @@ kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n age --- -## Slot 10：Redis-PG 密码（90 天） +## Slot 10：Redis 密码（90 天） -**影响**：Redis / PostgreSQL 连接断开。需与基础设施层协同。 +**影响**：Redis 连接断开（缓存/会话/总线）。需与基础设施层协同。 **步骤**： @@ -264,18 +264,15 @@ kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n age # 1) Redis 轮换密码 redis-cli -a '' CONFIG SET requirepass '' -# 2) PostgreSQL 轮换密码（DBA） -psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" +# 2) 更新密钥后端（key=redis-password） -# 3) 更新密钥后端（两个 key：redis-password + postgres-password） - -# 4) 更新 Redis 服务配置（若 Redis 由 docker-compose / StatefulSet 管理， +# 3) 更新 Redis 服务配置（若 Redis 由 docker-compose / StatefulSet 管理， # 需同步更新 requirepass 并 restart Redis） -# 5) restart agentkit pod +# 4) restart agentkit pod kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit -# 6) 验证 +# 5) 验证 kubectl exec -n agentkit deploy/agentkit -- \ python -c "import redis,os; redis.from_url(os.environ['REDIS_URL'],password=os.environ['REDIS_PASSWORD']).ping()" ``` @@ -283,7 +280,42 @@ kubectl exec -n agentkit deploy/agentkit -- \ **注意**：Redis `CONFIG SET requirepass` 立即生效，旧连接保持到断开。 建议低峰期操作，且 Redis 持久化配置（redis.conf）需同步更新，避免重启后失效。 -**回滚**：恢复 Redis / PG 旧密码 + 恢复 Secret。 +**回滚**：恢复 Redis 旧密码 + 恢复 Secret。 + +--- + +## Slot 11：PostgreSQL 密码（90 天） + +**影响**：PostgreSQL 连接断开（用户/会话/审计）。需 DBA 协同。 + +**DEPLOY-4 注意**：应用通过 `DATABASE_URL`（Slot 3）内嵌密码连接 PG，**不读 `POSTGRES_PASSWORD` env**。 +`POSTGRES_PASSWORD` 仅用于 PG StatefulSet 初始化。轮换 PG 密码时须**同步更新 Slot 3（database-url）**， +否则应用连接会失败。 + +**步骤**： + +```bash +# 1) PostgreSQL 轮换密码（DBA） +psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" + +# 2) 更新密钥后端 — 两个 key 须同步： +# a) postgres-password（PG StatefulSet 初始化用） +# b) database-url（应用连接用，DSN 内嵌新密码） +# 格式：postgresql+asyncpg://agentkit:@postgres:5432/agentkit + +# 3) 若 PG 由 StatefulSet 管理，需删除 PG pod 触发重新初始化（仅首次部署后需要； +# 已有数据的 PG 不会重新读 POSTGRES_PASSWORD，密码由 ALTER USER 持久化） + +# 4) restart agentkit pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证 +kubectl exec -n agentkit deploy/agentkit -- \ + python -c "import os; from sqlalchemy.ext.asyncio import create_async_engine; \ +import asyncio; asyncio.run(create_async_engine(os.environ['DATABASE_URL']).dispose()); print('ok')" +``` + +**回滚**：恢复 PG 旧密码 + 恢复两个 Secret（postgres-password + database-url）。 --- diff --git a/docs/plans/2026-07-05-001-feat-enterprise-agent-platform-roadmap-plan.md b/docs/plans/2026-07-05-001-feat-enterprise-agent-platform-roadmap-plan.md new file mode 100644 index 0000000..5f6034e --- /dev/null +++ b/docs/plans/2026-07-05-001-feat-enterprise-agent-platform-roadmap-plan.md @@ -0,0 +1,622 @@ +--- +title: "AgentKit 企业版路线图 - Plan" +type: feat +date: 2026-07-05 +topic: enterprise-agent-platform-roadmap +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-brainstorm +execution: code +deepened: 2026-07-05 +--- + +# AgentKit 企业版路线图 - Plan + +## Goal Capsule + +- **Objective:** 把 AgentKit 企业级基线推到 L3 能卖给国内中大型企业/政企,同时加深垂直差异化(自进化、Board 决策、4 层记忆、业务系统集成)形成护城河。 +- **Product Authority:** 产品负责人。 +- **Open Blockers:** P0 6 项(含 R-等保文档)完成后,首个政企客户 POC 是否能落地。 +- **Execution Profile:** code —— 路线图含 SSO 集成、K8s 部署、沙箱补全、OTel 启用等可执行工作。 +- **Stop Conditions:** P0 6 项全交付(R1-R5 全交付 + R-等保文档启动并并行推进,目标认证 readiness 在政企 POC 采购关闭前完成,认证审计本身在 P1 R10);P1 至少完成 OTel admin dashboard + Subagents worktree + 完整沙箱分级;P2 至少启动 1 项垂直差异化加深。 +- **Tail Ownership:** 产品负责人决定 P1 → P0 升级触发条件与滚动评审节奏。 + +--- + +## Product Contract + +### Summary + +AgentKit 闭源企业版"两条并行"路线图——P0 推进 6 项阵列把企业级基线推到 L3 能卖给政企(SAML/OIDC SSO + SCIM 最小集、私有化 K8s 部署交付 + 密钥管理、prompt cache 全链路打通、剩余 residual P2 修复、OTel 默认启用、等保三级文档准备);P1 补 OTel admin dashboard、Subagents worktree、完整沙箱分级、等保三级认证、国密改造、信创 DB 适配、Wave1 G3/G8;P2 加深垂直差异化。商业模式为闭源企业版按席位计费,以滚动规划方式推进(4-6 周评审 + 政企专属反馈通道,不锁时间)。 + +### Problem Frame + +Qoder / Codex / Trae Work / WorkBuddy 已占据编程工具赛道并建立生态壁垒。AgentKit 在执行引擎、自进化、记忆、安全、Pipeline 编排、业务系统集成上达到或超过竞品 L3-L4 水准,但在企业身份、合规认证、生态市场、多端覆盖、OS 沙箱上明显落后。 + +横向评估(14 维度,满分 56):AgentKit 总分 42(L2 标准级),落后于 Qoder(51)、Codex(50)、WorkBuddy(48)、Trae Work(48)。差距集中在 Eval-G1 企业级身份合规、Eval-G3 OS 沙箱、Eval-G4 多端覆盖、Eval-G5 Skill 生态、Eval-G9 可观测性;独特优势在 A1 自进化体系、A2 Board 会议模式 + B+C 混合团队、A3 6 层终端安全、A4 4 层记忆 + Section CRUD、A5 业务系统深度集成、A6 7 种 ExecutionMode、A7 Plan-Exec 4 阶段状态机、A8 Pipeline + Saga + Checkpoint。 + +> **G-label 双轨制说明:** Eval-G* 为本横向评估的差距标签(Eval-G1/G3/G4/G5/G9)。Brainstorm `docs/brainstorms/2026-06-29-advanced-agent-gap-optimization-requirements.md` 的 Wave1/Wave3 G3/G5/G6/G8 是另一套独立的差距标签(tool schema / 函数级分片 / SOLO 状态机 / delta_flush),二者不互通,文档内以 Wave 前缀区分。 + +战略选择:避免在编程工具赛道正面对标,聚焦企业运营场景(Bitable / Calendar / Documents / IM channels + Board 决策模式),以差异化护城河对抗编程工具的通用化扩张。两条并行——P0/P1 推进企业级基线到 L3 让产品能卖给政企;P2 加深垂直差异化形成壁垒。 + +### Key Decisions + +- **两条并行而非差异化押注或通用平台追赶** —— 复用已有深度(自进化、Board、4 层记忆),同时补企业级基线(SSO、私有化、沙箱、OTel、合规)。押注差异化但放弃企业基线会让产品无法进入政企;正面对标编程工具赛道则面对先发劣势。注:此处的"并行"指战略双轨,执行顺序仍为 P0 → P1 → P2 串行(见 Mermaid)。 +- **目标客户为国内中大型企业/政企** —— 优先级:SAML 2.0 / OIDC SSO + 等保三级 + 国密 + 私有化信创部署。SOC2/ISO 不优先,国际版不做,聚焦国内政企。 +- **多租户隔离移出 P0** —— 政企私有化主战场是单租户,SaaS 多租户场景延后到 SaaS 多客户运营时再讨论。本次完全移出,降低 P0 复杂度。**假设**:首个政企 POC 客户为单租户单部门;若销售阶段识别到多部门/多子公司隔离需求(常见于央企/国企),触发 multi-tenancy 提前评估流程(参考 R18 滚动规划触发条件)。 +- **等保三级推到 P1 而非 P0(认证审计),但文档准备 P0 启动** —— 政企进入门槛需要,但认证周期长,P0 先交付 SSO + 私有化 K8s 的 MVP,合规认证审计后置到 P1;但等保三级文档准备(R-等保)与 MVP 并行启动,目标认证 readiness 在政企 POC 采购关闭前完成。 +- **闭源而非 Open Core** —— 简化范围,SSO / 等保 / 信创适配作为企业版独享,Skill 市场延后到生态成熟期。 +- **滚动规划 + 政企专属反馈通道** —— 4-6 周评审,按客户反馈和市场动态调整下一批需求,不锁时间。文档锁定原则和优先级粗框架。政企客户反馈 bypass 4-6 周节奏(首飞客户在 P0 实施期间设立 standing pre-POC review,采购阻塞信号触发紧急升级)。 +- **OS 沙箱分级已完成最低层级(workspace-write + no network)** —— P1 升级到完整分级(read-only / workspace-write / danger),而非从零构建。 +- **R5 OTel default-enable 升级到 P0** —— 因 R3 命中率监控接入 OTel(避免 P0→P1 priority inversion),OTel 框架接入点已存在(`tracer.py:117` OTLP/gRPC + `metrics.py:3-7`)。但 `opentelemetry-*` 依赖未在 `pyproject.toml` 声明(`tracer.py:129` 与 `metrics.py:7` 在 ImportError 时静默回退 NoOp),R5 实际工作量包含补依赖 + 取消 `agentkit.yaml:85-90` telemetry 段注释 + 配置 `telemetry.otlp_endpoint`(`setup.py:46` 读 YAML 不读 `OTEL_EXPORTER_OTLP_ENDPOINT` 环境变量),非"一行配置"。R5 在 P1 仅保留 admin dashboard 部分。 + +### Requirements + +#### P0 阵列(能卖给政企的 MVP) + +- R1. 集成 SAML 2.0 与 OIDC SSO,支持飞书 IDP / Azure AD / 阿里云 IDaaS。SCIM 最小子集(用户创建/禁用)纳入 P0 验收门槛。 + - R1a. **多 IDP 信任边界**:多 IDP 并存时账号关联以 IDP 声明的 subject identifier 为主键,禁止仅凭邮箱合并(账号接管防护);IDP 元数据(签名证书)须支持热轮换,不重启服务;任一 IDP 不可达时该 IDP 路径默认拒绝并返回明确错误,不影响其他 IDP。 + - R1b. **过渡期 deprovisioning**:SCIM 推后期内,产品须提供管理员手动 deprovisioning 接口(强制 operator/admin RBAC,与 R7a/R5'a 一致;每次调用记录 actor、target_user、reason、source=scim_fallback|manual,接入 R7a 同款审计日志通道)+ IDP 侧账号状态周期对账(每日 cron)。**高权限账号 SLA**:operator/admin/持有 danger tier 授权者的回收 SLA 为 1h(通过 break-glass 手动接口即时执行,cron 对账频率提升至每小时);SCIM 推送失败须触发实时告警(非等 cron)。 + - R1c. **单租户假设**:假设首个政企 POC 客户为单租户单部门;若销售阶段识别到多部门/多子公司隔离需求,触发 multi-tenancy 提前评估流程。 +- R2. 提供 K8s Helm Chart 部署清单 + 离线安装包 + 部署运维文档,覆盖私有化交付场景。docker-compose 已有但不足以承载政企生产部署。 + - R2a. **密钥管理**:部署清单须明确区分配置与密钥——所有密钥(JWT 签名密钥、API Key、DB 凭据、IDP 签名证书、第三方 API Key、TLS 服务器证书私钥、mTLS 客户端证书私钥、KMS/HSM 访问凭证 PKCS#11 PIN、OTel exporter 鉴权 token、Redis/PG 连接密码)通过 Sealed Secrets 或 External Secrets Operator 注入,禁止以 plain Kubernetes Secret + Git 提交形式交付;KMS/HSM 访问凭证须通过 workload identity 或 projected service account token 注入,而非静态 Secret;首飞须提供密钥轮换 runbook。 +- R3. 完成 prompt cache 全链路打通:验证记忆写入确实路由到 volatile 层、`prompt_cache_enable` 默认开启、命中率监控接入 OTel 框架(与 R5 OTel default-enable 配套)。结构已实现(stable/volatile 双块 + Anthropic `cache_control` 透传 + 测试覆盖),但全链路收益未兑现。**P0 验收门槛**:政企部署须提供本地推理 fallback 选项(通义/DeepSeek/Doubao 私有化),作为 Anthropic 路径的等保合规替代;fallback 路径不依赖 prompt cache 价值(R3 P0 优先级服务中大型企业 Anthropic-eligible 客户段,见 OQ13)。 +- R4. 修复剩余 residual P2 findings:feat-ui-ue-enhancement 3 项(expert_step payload 缺字段、team_synthesis 孤儿 milestone、synthesis_id 去重)+ feat-bitable-enhancement 5 项(DR-1 SQL 注入风险、DR-2 ViewConfigPanel 深审、DR-4 TOCTOU race、DR-5 _update_field 静默失败、DR-3 设计 token 测试)。execute_stream CancellationToken 注册状态已通过 KTD-4 钩子修复(见 OQ2 关闭说明)。 +- R5. **OTel 默认启用**(从原 R5 拆出升级到 P0):将 `opentelemetry-api` / `opentelemetry-sdk` / `opentelemetry-exporter-otlp-proto-grpc` / `opentelemetry-instrumentation-fastapi` 添加到 `pyproject.toml` dependencies(当前未声明,`tracer.py:129` 与 `metrics.py:7` 在 ImportError 时静默回退 NoOp);取消 `agentkit.yaml:85-90` telemetry 段注释并设 `enabled=true`;配置 `telemetry.otlp_endpoint`(`setup.py:46` 读 YAML 不读 `OTEL_EXPORTER_OTLP_ENDPOINT` 环境变量);框架接入点已存在(`src/agentkit/telemetry/tracer.py:117` OTLP/gRPC + `src/agentkit/telemetry/metrics.py:3-7`)。 +- R-等保. **等保三级文档准备(P0 启动)**:启动等保三级文档与控制点清单编写(安全物理 / 网络 / 主机 / 应用 / 数据 / 管理 6 维度),与 MVP 并行推进,目标认证 readiness 在政企 POC 采购关闭前完成。认证审计本身仍在 P1。 + +#### P1 阵列(企业级基线补强) + +- R5'. **OTel admin dashboard**(R5 拆分后剩余):前端 admin dashboard(可观测性、调用链路 Trace、性能监控、Token 用量)。 + - R5'a. **访问控制**:admin dashboard 强制 admin RBAC 角色;OTel Span 导出前须脱敏 prompt 内容、API Key、用户 PII、tool_call arguments、tool_result 文件路径/凭证模式匹配、环境变量值、异常堆栈中的主机名与路径(参考现有 6 层终端安全脱敏正则库扩展);Token 用量数据保留 180 天(等保三级审计日志最低要求,GB/T 22239-2019),过期自动归档至冷存储。 +- R6. Subagents worktree 并行执行:`PlanPhase` 增加 worktree 隔离选项,参考 Codex Subagents(`max_threads=6` + Auto-review)模式。 +- R7. 完整沙箱分级(read-only / workspace-write / danger tier),最低层级已实现(`src/agentkit/core/sandbox.py`),需补 read-only 与 danger 模式 + macOS Seatbelt / Linux Landlock 系统调用级隔离。 + - R7a. **danger tier 授权模型**:danger tier 执行须满足:(1) 仅 operator/admin RBAC 角色可发起;(2) 每次调用前要求显式 confirmation_request;(3) 调用记录写入审计日志含 actor、target、command、duration、exit_code;(4) **命令面约束**:命令须通过预定义白名单(或 deny-list 阻断已知危险模式如 `rm -rf` / `curl` / `kubectl delete`),参数须经过 shell-escape 校验,每 actor 每分钟调用上限可配(默认 5),超出限流阈值触发 break-glass 告警。workspace-write 为默认,read-only 为受限角色默认。 +- R8. Wave1 Brainstorm-G3 工具 schema 校验:工具描述 schema 超过上下文 10% 时按需渐进披露,参考 Codex MCP `tool_search` 模式。 +- R9. Wave1 Brainstorm-G8 delta_flush 节流:流式 token flush 节流以减少 WebSocket 消息数。 +- R10. 等保三级合规认证:启动认证审计流程,基于 R-等保文档准备。安全物理 / 网络 / 主机 / 应用 / 数据 / 管理 6 维度控制点落实。 +- R11. 国密 SM2 / SM3 / SM4 改造:JWT、API Key、敏感字段加密使用国密算法,适配政企合规。 + - R11a. **密钥管理与迁移**:国密密钥须存储于 KMS/HSM(支持信创 PKCS#11),禁止文件落盘;切换期间支持双算法并行验证(JWT header 标识 alg,API Key 双写),过渡期不少于一个轮换周期;**降级攻击防护**:(a) 旧算法签发的 token TTL 在过渡期缩短至原 TTL 的 1/4;(b) 旧算法签发/接受事件须写入安全审计日志并触发 SIEM 告警;(c) 过渡期结束须有硬性 EOL 日期,过期后旧算法 token 一律拒绝;(d) 过渡期不接受新签发的旧算法 token(仅接受存量刷新);敏感字段加密须支持存量数据就地迁移脚本,迁移脚本须幂等(基于记录 ID + 版本标记可重跑),每批提交事务,失败时回滚该批,迁移完成须有校验报告(迁移记录数、校验和、失败记录列表)。 +- R12. 信创 DB 适配:PostgreSQL 之外支持达梦 / 人大金仓,RAG 向量层(pgvector)需适配或抽象。 + +#### P2 阵列(垂直差异化加深) + +- R13. 移动端 + 语音输入:Tauri 移动端,接 IM channels 部分弥补。 +- R14. Skill 市场激活:激活现有 `src/agentkit/marketplace/auction.py` Vickrey 拍卖机制接入主流程。 +- R15. 原生 Computer Use:截图识别 + 多步编排,降低对 Anthropic API 依赖。 +- R16. Wave3 Brainstorm-G5 函数级分片:tree-sitter 引入,KTD6 决策延后到 Wave 3 plan。 +- R17. Wave3 Brainstorm-G6 SOLO 四阶段状态机扩展 PLAN_EXEC:阶段约束(Planning 不可 write_file 等),KTD7 决策延后到 Wave 3 plan。 + +#### 滚动规划决策规则 + +- R18. 每 4-6 周评审一次,按客户反馈和市场动态调整下一批需求。P1 → P0 升级触发条件:(1) 客户 POC 阻塞项;(2) 同类需求重复出现 3+ 次;(3) 安全 / 合规审计发现紧急项。 + - R18a. **政企专属反馈通道**:首飞政企客户在 P0 实施期间设立 standing pre-POC review(每 2 周与客户对接),采购阻塞信号(POC 失败、合规审计发现、需求变更)触发紧急升级,而非等 4-6 周窗口。政企采购周期(6-12 个月)与内部滚动节奏并行存在。 + +### Key Flows + +- F1. 滚动规划评审流 + - **Trigger:** 4-6 周评审窗口到达,或客户反馈触发紧急升级,或政企专属通道(pre-POC review)识别采购阻塞。 + - **Actors:** 产品负责人,工程团队,首飞政企客户(pre-POC review)。 + - **Steps:** 反馈收集(滚动窗口 + 政企专属)→ 优先级评估(对照 R18 触发条件)→ 评审窗口决策 → 调整 P0/P1 阵列 → 通知相关方。 + - **Covered by:** R18, R18a. + +- F2. 私有化交付流 + - **Trigger:** 政企客户采购。 + - **Actors:** 部署团队,客户运维。 + - **Steps:** 离线安装包交付(含 Sealed Secrets 配置)→ K8s 集群部署 → SSO 配置对接(飞书 IDP / Azure AD / 阿里云 IDaaS)→ 等保三级文档准备核对 → 验收。 + - **Covered by:** R1, R2, R-等保. + +### Visualizations + +```mermaid +flowchart TB + subgraph P0[P0 - 能卖给政企的 MVP] + R1[SAML/OIDC SSO + SCIM 最小集] + R2[K8s Helm 部署 + 密钥管理] + R3[Prompt cache 全链路打通] + R4[Residual P2 修复] + R5[OTel 默认启用] + ReqDengBao[等保三级文档准备] + end + subgraph P1[P1 - 企业级基线补强] + R5prime[OTel admin dashboard] + R6[Subagents worktree] + R7[完整沙箱分级] + R8[Brainstorm-G3 工具 schema] + R9[Brainstorm-G8 delta_flush] + R10[等保三级认证审计] + R11[国密改造] + R12[信创 DB 适配] + end + subgraph P2[P2 - 垂直差异化加深] + R13[移动端+语音] + R14[Skill 市场] + R15[原生 Computer Use] + R16[Brainstorm-G5 函数级分片] + R17[Brainstorm-G6 SOLO 状态机] + end + P0 --> P1 --> P2 + R1 -.客户反馈.-> R18[滚动规划评审] + R18 -.触发升级.-> P0 + R18 -.政企专属通道.-> R18a +``` + +### Success Criteria + +- P0 6 项全交付(R1 + R2 + R3 + R4 + R5 + R-等保),首个政企客户 POC 落地。 +- P1 至少完成 OTel admin dashboard + Subagents worktree + 完整沙箱分级。 +- P2 至少启动 1 项垂直差异化加深。 +- 横向评估总分从 42/56 提升到 50+/56(达到 WorkBuddy / Trae Work 同级)。注:评分体系需对接三方评测(Forrester / Gartner Magic Quadrant 或客户技术评估)以避免自评偏差。 +- 企业级维度(Eval-D10 安全权限、Eval-D12 部署企业级)从 L2/L3 提升到 L4。 + +> **维度标签说明:** Eval-D* 为本横向评估的 14 维度编号(D1-D14),与 Eval-G*(差距) / A*(优势)同源。文档其他位置如出现 D10/D12 默认指 Eval 维度编号。 + +### Scope Boundaries + +#### Deferred for later + +- 多租户隔离 —— 完全移出本次范围,延后到 SaaS 多客户运营时再讨论。首个政企 POC 客户为单租户假设(见 R1c)。 +- Skill 市场激活 —— 延后到 P2,优先级低于垂直差异化加深。 +- 等保三级合规认证审计 —— 文档准备 P0 启动(R-等保),认证审计推到 P1(R10)。 +- 移动端 + 语音输入 —— 推到 P2。 +- 原生 Computer Use —— 推到 P2。 +- Wave3 Brainstorm-G5 / Brainstorm-G6 —— 推到 P2,依赖 KTD6 / KTD7 决策。 + +#### Outside this product's identity + +- 与 Qoder / Codex / Trae Work 正面竞争编程工具赛道 —— 不做。 +- 国际版 / SOC2 / ISO 认证 —— 不做,聚焦国内政企。 +- 开源核心 —— 不做,保持闭源企业版计费。 +- 与 WorkBuddy SkillHub / Qoder MCP Square 上架 —— 不做,生态合作延后。 + +### Dependencies / Assumptions + +- 假设:首个政企 POC 客户为单租户单部门;若销售阶段识别到多部门/多子公司隔离需求,触发 multi-tenancy 提前评估。 +- 假设:complex-task-quality-loop(plan `docs/plans/2026-07-03-001-feat-complex-task-quality-loop-plan.md`)的 11 项需求(R1-R8, R10-R12;R9 coding_harness deferred per RV11/RV12/RV13)+ 10 KTD 已稳定,可作为 P0 启动基线。 +- 假设:Bitable P0 UX(merge commit `8633f60`)已稳定,可作为基线。 +- 依赖:Anthropic API 持续支持 `cache_control` 断点。 + - **冲突声明:** R3 依赖 Anthropic API(境外服务),与等保三级(R10)数据本地化要求存在冲突。R3 须明确:(1) 用户 prompt 在发送前进行 PII 脱敏——采用基于本地 NER 模型(如 LAC / HanLP)的 PII 识别 + 替换/掩码,模型部署在客户内网;脱敏前后均记录 hash 用于审计追溯但原文不落盘;脱敏失败时默认 fail-closed(拒绝发送至 Anthropic),仅在本地推理 fallback 可用时降级处理;脱敏覆盖率与漏报率纳入 OTel 指标监控(注:原引用的"现有 6 层终端安全脱敏模式"针对结构化终端命令设计,无法覆盖自由语言 prompt,故改用本地 NER 模型);(2) 政企部署提供本地推理 fallback 选项(通义/DeepSeek/Doubao 私有化),作为 Anthropic 路径的等保合规替代;(3) 与客户签署数据处理协议(DPA),明确 Anthropic 侧缓存数据所有权与删除权——注:Anthropic prompt cache 仅由 TTL(默认 5 分钟,beta 1 小时)自动过期,未提供显式删除 API,DPA 条款的"立即删除"承诺不可强制执行,只能依赖 TTL 自然过期(见 OQ15)。 +- 依赖:K8s 1.28+ / Helm 3.x 部署环境(若首飞客户为传统 IDC,需评估 docker-compose 加固 fallback,见 OQ5 + OQ8)。 +- 依赖:飞书 / Azure AD / 阿里云 IDaaS IDP 协议稳定性(签名证书热轮换 + 默认拒绝故障模式,见 R1a)。 +- 假设:政企客户接受 PostgreSQL 作为主数据库,信创 DB 适配推到 P1 而非 P0(若客户硬性要求则需提前)。 + +### Outstanding Questions + +#### Resolve Before Planning + +- OQ1. ~~R3 prompt cache 命中率监控接入路径~~ —— **已解决**:接入 OTel 框架,R5 OTel default-enable 升级到 P0,与 R3 配套。Planning 可按 R5 → R3 顺序排程。 +- OQ2. ~~R4 execute_stream CancellationToken 注册状态~~ —— **已解决**:已通过 KTD-4 钩子修复(`src/agentkit/core/config_driven.py:778` execute_stream def,`:792-793` CancellationToken 注册,带 "P2 fix: 注册 CancellationToken 到 _active_tokens" 注释)。Planning 阶段无需再核对,可直接关闭此 OQ。 +- OQ3. R7 完整沙箱分级——是否要在 P1 完成全部三个层级(read-only / workspace-write / danger),还是仅补 read-only 即可达到 L4?影响工作量估算。 + +#### Deferred to Planning + +- OQ4. R1 SSO 协议优先级:SAML 2.0 优先(国内 IDP 主流)还是 OIDC 优先(国际标准)?影响首飞 IDP 选择。 +- OQ5. R2 K8s 部署清单兼容矩阵:原生 K8s / 信创 K8s / OpenShift 哪些必须支持? +- OQ6. R12 信创 DB 适配:达梦优先还是人大金仓优先?RAG 向量层(pgvector)是否抽象为独立接口? +- OQ7. R6 Subagents worktree:`max_threads` 默认值(Codex 是 6,AgentKit 应该多少)?worktree 失败如何回滚? + +#### From 2026-07-05 ce-doc-review + +- OQ8. **K8s 假设可能不覆盖传统 IDC 政企客户**(security-lens + adversarial 识别,anchor 75 promote to 100)。国内政企有相当部分仍在传统 IDC + VM 部署(华为云 Stack / 阿里云 Apsara Stack 模式)。R2 Helm Chart 无法直接运行,可能需要 docker-compose 加固 fallback 路径。Planning 须评估是否在 P0 加入 docker-compose 加固,或在 R18 触发条件中识别"传统 IDC 客户"作为升级触发。 +- OQ9. **R6/R8/R9 是否应从 P1 重分类到 P2**(scope-guardian 识别,anchor 75)。R6/R8/R9 描述为性能/能力优化(Subagents worktree / 工具 schema 渐进披露 / delta_flush 节流),而非企业级基线。R8/R9 携带 Wave1 前缀,R6 引用 Codex 模式(competitor mimicry 而非 AgentKit pain)。Planning 须评估 P1 theme 是否扩展为"企业级基线 + 性能优化",或重分类至 P2。 +- OQ10. **P2 items 不深化已声明护城河**(scope-guardian 识别,anchor 100)。Goal 明确声明护城河为"自进化 / Board 决策 / 4 层记忆 / 业务系统集成",但 P2 items(R13-R17)均不深化这四项。Stop Condition "P2 至少启动 1 项垂直差异化加深"可在不推进护城河的情况下满足。Planning 须决定:P2 增加深化护城河的 items,还是修订 Goal statement 承认 P2 是新能力而非深化? +- OQ11. **D-label 维度编号未定义**(coherence 识别,anchor 75)。Success Criteria 引用 D10 安全权限 / D12 部署企业级,但 Problem Frame 只定义 G*(差距)和 A*(优势),无 D-labeling 方案。本次评审已添加 Eval-D 前缀作为 disambiguation,但完整 14 维度编号(D1-D14)未在文档内列出。Planning 须决定:在文档内完整列出 14 维度,或引用外部评估报告。 +- OQ12. **G-label 双轨制 disambiguation**(coherence 识别,anchor 75)。Eval-G*(横向评估差距:G1/G3/G4/G5/G9)与 Brainstorm-G*(2026-06-29 brainstorm 差距:G3=tool schema / G5=函数级分片 / G6=SOLO 状态机 / G8=delta_flush)是两套独立编号。本次评审已用 Eval-/Brainstorm- 前缀 disambiguate,但 Planning 须决定是否在 Sources/Research 区段显式列出两套 gap scheme 的定义。 + +#### From 2026-07-05 ce-doc-review (Round 2) + +- OQ13. **R3 prompt cache 价值在本地推理 fallback 路径下不可实现**(product-lens + adversarial 识别,anchor 75 promote to 100)。3-pronged 缓解方案创建战略缺口:当政企客户启用本地推理 fallback(通义/DeepSeek/Doubao)时,R3 的 Anthropic `cache_control` 透传机制对该路径完全无效,R3 收益归零;且 PII 脱敏后送给 Anthropic 的 prompt 与原始 prompt 不同,cache 命中率监控的是"脱敏后 prompt 的命中率",无法直接反映真实业务收益。文档已在 R3 添加 "P0 优先级服务中大型企业 Anthropic-eligible 客户段" 注释缓解,但 Planning 须决定:R3 的 P0 优先级是基于中大型企业(Anthropic-eligible)客户价值还是政企客户价值?若是前者,R3 应重新定位而非列入"政企 MVP";若是后者,R3 在 fallback 路径下如何兑现价值? +- OQ14. **SCIM 缺 group→role 映射,RBAC 角色分配无审计**(security-lens 识别,anchor 75)。R1 SCIM 最小子集仅含用户创建/禁用,未含 group 同步;R7a/R5'a 依赖 operator/admin RBAC 角色,但角色分配机制未定义——若由管理员手动分配,则缺乏角色变更审计(谁在何时把谁提权到 admin)。等保三级要求特权账号变更可审计可追溯。Planning 须决定:P0 是否将 SCIM group→role 映射纳入(取决于首飞客户 IDP 能力,见 OQ4)?若手动分配,是否新增"RBAC 角色变更审计 logging"作为 P0 验收门槛? +- OQ15. **DPA cache 删除权假设 Anthropic 提供 API**(security-lens + adversarial 识别,anchor 50 promote to 100)。DPA 第三叉"明确 Anthropic 侧缓存数据所有权与删除权"是政企合规采购的关键条款,但 Anthropic prompt cache 文档未提供显式"删除指定缓存条目"的 API——缓存条目仅由 TTL(默认 5 分钟,beta 1 小时)自动过期。这意味着"客户终止时立即删除缓存数据"的 DPA 承诺在工程上不可强制执行。文档已在 Anthropic 冲突声明中标注此限制,但 Planning 须决定:(a) 是否在 DPA 中明确"删除权仅由 TTL 自然过期实现,最长 1 小时"作为已知限制?(b) 是否要求政企客户默认 `prompt_cache_enable=false`,仅对显式签署 DPA 且接受残留风险的客户开启?(c) 本地推理 fallback 路径是否默认 `prompt_cache_enable=true` 以弥补性能损失? +- OQ16. **R1c 单租户 reversal cost 高**(adversarial 识别,anchor 75)。R1c 将单租户假设升级为显式决策,但缓解措施仅承诺"触发 multi-tenancy 提前评估流程"——评估不是承诺。单租户是 load-bearing 决策:数据模型、RBAC 角色、审计日志 schema、SCIM 用户主键都按单租户设计。若销售阶段识别到央企/国企的多部门/多子公司隔离需求,"提前评估"后若结论为"需要 multi-tenancy",届时 retrofit 成本极高(数据迁移 + RBAC 重构 + 审计日志 schema 变更)。Planning 须决定:(a) 是否在 P0 启动前完成单租户 reversal cost 评估(数据模型/RBAC/审计日志 schema 变更工作量估算)?(b) 是否在 R18 触发条件中增加"架构性 reversal cost"维度?(c) 是否在首飞政企客户销售阶段早期(standing pre-POC review)就明确 multi-tenancy 需求? + +### Sources / Research + +- 横向评估报告(本对话上文):AgentKit vs Qoder / Trae Work / Codex / WorkBuddy 五产品 14 维度对比。 +- 现有 plan:`docs/plans/2026-07-03-001-feat-complex-task-quality-loop-plan.md`(P1 多项已落地基线;R1-R8, R10-R12 共 11 项需求 + 10 KTD)。 +- 现有 plan:`docs/plans/2026-07-03-001-feat-bitable-p0-ux-and-agent-parity-plan.md`(已 merge)。 +- Residual findings:`docs/residual-review-findings/feat-ui-ue-enhancement.md`、`docs/residual-review-findings/feat-bitable-enhancement.md`。 +- 现有 brainstorm:`docs/brainstorms/2026-06-29-advanced-agent-gap-optimization-requirements.md`(原 Qoder / Codex / Hermes / Trae Work 长程可靠性对比;Brainstorm-G* gap scheme 来源)。 +- 现有 brainstorm:`docs/brainstorms/2026-06-09-agentkit-capability-matrix/requirements.md`(能力矩阵愿景)。 +- 代码证据:`src/agentkit/core/sandbox.py`(沙箱最低层级已实现)、`src/agentkit/telemetry/tracer.py:117`(OTel 已接入)、`src/agentkit/telemetry/metrics.py:3-7`(OTel 框架)、`tests/unit/test_prompt_cache_layers.py:46-58`(prompt cache 结构已实现)、`src/agentkit/core/config_driven.py:778, 792-793`(execute_stream CancellationToken 已实现)、`src/agentkit/marketplace/auction.py`(Vickrey 拍卖已实现)。 + +> **Product Contract preservation:** Product Contract 不变 — `ce-plan` 丰富阶段保留所有 R/A/F/AE IDs、Key Decisions、Scope Boundaries、Outstanding Questions 原文。新增 Planning Contract(下方),不修改 Product Contract 既有内容。 + +--- + +## Planning Contract + +### Summary + +P0 阵列 6 项作为本次实施的 Implementation Units(U1-U6),按依赖排序:U1 OTel 默认启用(基础)→ U2 Prompt cache 全链路 + 本地推理 fallback(依赖 U1 metrics)→ U3 Residual P2 修复(独立)→ U4 SSO 集成(依赖 U1 审计日志)→ U5 K8s Helm Chart(打包 U1-U4)→ U6 等保三级文档准备(文档,依赖 U1/U4/U5 输入)。P1 / P2 阵列作为 Deferred to Follow-Up Work,不在本次 plan 范围。 + +### Key Technical Decisions + +- **KTD-1. P0 是单一交付单元;每个 R-item 是一个 Implementation Unit,按依赖排序。** U1(U6 OTel)是最小且基础的工作(其他 unit 依赖其 metrics/audit 能力);U2(R3 prompt cache)依赖 U1 的命中率 metric;U3(R4 residual)独立;U4(R1 SSO)依赖 U1 审计日志通道;U5(R2 K8s)打包 U1-U4 为可部署产物;U6(R-等保文档)依赖 U1/U4/U5 输入。单租户假设(R1c)是 load-bearing 决策 — 数据模型/RBAC/审计 schema/SCIM 主键均按单租户设计,reversal cost 极高(OQ16),通过 R18 触发条件中增加"架构性 reversal cost"维度 + pre-POC review 早期识别缓解。 + +- **KTD-2. OIDC 优先于 SAML 作为首飞 IDP 协议。** OIDC 在 Python 生态有更成熟的库支持(`authlib`),redirect flow 比 SAML POST/Redirect 简单,且三个目标 IDP 均原生支持 OIDC(飞书 OIDC app / Azure AD native OIDC / 阿里云 IDaaS native OIDC)。SAML 2.0 作为 R1 内的第二阶段 IDP(同一 Unit,后续 sub-commit),服务硬性要求 SAML 的客户。现有 `StubOIDCProvider`(`src/agentkit/server/auth/providers/oidc_stub.py`)和 `AuthProvider` Protocol(`providers/base.py`)提供集成 seam — 替换 stub 为真实 `authlib` 实现。 + +- **KTD-3. OTel 依赖从 optional 升级为 required。** `opentelemetry-api` / `opentelemetry-sdk` / `opentelemetry-exporter-otlp-proto-grpc` / `opentelemetry-instrumentation-fastapi` 从"ImportError → NoOp 静默回退"移入 `pyproject.toml` 的 `[project] dependencies`(非 `[optional-dependencies]`)。理由:R5 使 OTel default-enabled,ImportError 回退成为 dead code。NoOp 回退保留为"配置错误"防御层(`enabled=false` 或 init 失败时),但 ImportError 现在是 startup failure(`init_telemetry` 不再 catch ImportError),而非静默降级。现有 `tracer.py:129` / `metrics.py:7` 的 try/except ImportError 改为直接 import(依赖已声明)。 + +- **KTD-4. Helm Chart + Sealed Secrets/External Secrets Operator,禁用 plain Kubernetes Secret + Git 提交。** 所有密钥(JWT 签名密钥、API Key、DB 凭据、IDP 签名证书、第三方 API Key、TLS 服务器证书私钥、mTLS 客户端证书私钥、KMS/HSM PKCS#11 PIN、OTel exporter 鉴权 token、Redis/PG 连接密码)通过 Sealed Secrets(bitnami)或 External Secrets Operator 注入。KMS/HSM 访问凭证通过 projected service account token + workload identity 注入,而非静态 Secret。Helm Chart 的 `values.yaml` 模板显式列出每个 secret slot;首飞须提供密钥轮换 runbook(`docs/deployment/key-rotation-runbook.md`)。 + +- **KTD-5. 本地推理 fallback 是可插拔 LLM provider 配置,非独立代码路径。** R3 的 fallback(通义/DeepSeek/Doubao 私有化部署)复用现有 LiteLLM-based LLM gateway(`src/agentkit/llm/`)。`agentkit.yaml` 的 `llm.providers` 段已支持 OpenAI-compatible API(DashScope / DeepSeek 模板见现有配置)。Fallback 通过配置注入,不需要新代码。`prompt_cache_enable` 是 per-provider 配置:fallback providers 设置 `prompt_cache_enable=false`(不支持 Anthropic `cache_control`)。R3 P0 优先级服务中大型企业 Anthropic-eligible 客户段(OQ13 缓解);fallback 路径不依赖 prompt cache 价值,Anthropic 路径客户签署 DPA 接受 TTL 残留风险(OQ15)。 + +- **KTD-6. R-等保是 documentation-only 交付物,非 code unit。** 等保三级文档(6 维度:物理 / 网络 / 主机 / 应用 / 数据 / 管理)按 GB/T 22239-2019 编写,是 P0 Stop Condition 的一部分。`execution: code` 应用于其他 5 个 units;U6 标记为 documentation deliverable,无 behavioral change。文档位于 `docs/compliance/dengbao-level3/`。认证审计本身仍在 P1 R10。 + +- **KTD-7. R4 residual findings 作为独立 bug fix 批次,合成一个 U-ID。** 8 项 residual findings(3 项 feat-ui-ue-enhancement + 5 项 feat-bitable-enhancement)是独立 bug fix,无架构依赖。合成一个 U-ID 因为每项较小且共享 review/verification 路径。execute_stream CancellationToken 修复(OQ2 已关闭)已通过 KTD-4 钩子应用(`src/agentkit/core/config_driven.py:792-793`),排除在本 Unit 外。 + +- **KTD-8. SCIM group→role mapping 延后;P0 SCIM 仅含用户创建/禁用 + 手动 RBAC + 审计。** SCIM 2.0 group sync(group→role 映射表 + 角色变更审计)增加复杂度,延后到 P0 stretch goal(gated on 首飞客户 IDP 能力,OQ4)。P0 SCIM 最小子集:`/scim/v2/Users` POST(创建)/ PATCH(禁用)/ DELETE。RBAC 角色分配保持管理员手动操作,但新增"RBAC 角色变更审计 logging"作为 P0 验收门槛(OQ14 缓解)——每次角色变更记录 actor / target_user / role_before / role_after / reason / source,接入 R7a 同款审计日志通道(依赖 U1 OTel)。 + +- **KTD-9. 等保三级审计日志保留期 180 天(GB/T 22239-2019 最低要求)。** OTel Span / Token 用量 / RBAC 变更 / SCIM 操作 / danger tier 调用审计日志保留 180 天,过期自动归档至冷存储。原 R5'a 的 30 天保留期不满足等保三级(Round 2 adversarial 发现),已修正为 180 天。 + +### High-Level Technical Design + +#### P0 依赖与排序 + +```mermaid +flowchart LR + U1[U1 OTel 默认启用] --> U2[U2 Prompt cache 全链路] + U1 --> U4[U4 SSO 集成] + U3[U3 Residual P2 修复] --> U5[U5 K8s Helm Chart] + U2 --> U5 + U4 --> U5 + U1 --> U6[U6 等保文档] + U4 --> U6 + U5 --> U6 +``` + +#### SSO 多 IDP 信任流 + +```mermaid +sequenceDiagram + participant U as 用户浏览器 + participant A as AgentKit Server + participant IDP1 as 飞书 IDP (OIDC) + participant IDP2 as Azure AD (OIDC) + participant IDP3 as 阿里云 IDaaS (SAML) + U->>A: 点击"飞书登录" + A->>U: 302 redirect to IDP1 authorize + U->>IDP1: 认证 + IDP1->>U: 302 redirect to /auth/oauth/feishu/callback?code=... + U->>A: GET /callback?code=... + A->>IDP1: token exchange (server-side) + IDP1->>A: id_token + userinfo (sub=xxx) + A->>A: 按 sub 作主键查/建本地用户(禁止邮箱合并) + A->>A: 签发 JWT,写 auth_sessions.auth_provider='oidc-feishu' + A->>U: Set-Cookie + redirect to /agent/chat +``` + +#### OTel 数据流 + +```mermaid +flowchart TB + subgraph AgentKit Pod + A[FastAPI App] -->|instrumentation-fastapi| T[OTel Tracer] + A -->|metrics| M[OTel Meter] + T -->|OTLP/gRPC| E[OTLPExporter] + M -->|OTLP/gRPC| E + end + E -->|4317| C[OTel Collector] + C -->|prometheus| P[Prometheus] + C -->|tempo/jaeger| J[Trace Backend] + C -->|loki| L[Log Backend] + subgraph Admin Dashboard (P1 R5') + P --> D[dashboard.vue] + J --> D + end +``` + +#### 14 维度横向评估编号 + +| Eval-D | 维度 | AgentKit 现状 | 目标 | +|--------|------|--------------|------| +| D1 | 自进化体系 | L4(A1 优势) | L4(保持) | +| D2 | Board 决策 + 团队协作 | L4(A2 优势) | L4(保持) | +| D3 | 终端安全 | L3(A3 优势) | L4(R7a 完善后) | +| D4 | 4 层记忆 + Section CRUD | L4(A4 优势) | L4(保持) | +| D5 | 业务系统集成 | L4(A5 优势) | L4(保持) | +| D6 | ExecutionMode 多样性 | L4(A6 优势) | L4(保持) | +| D7 | Plan-Exec 状态机 | L4(A7 优势) | L4(保持) | +| D8 | Pipeline + Saga + Checkpoint | L4(A8 优势) | L4(保持) | +| D9 | 企业身份合规 (SSO/SCIM) | L1(G1 差距) | L3(P0 后) | +| D10 | 安全权限 (RBAC/审计) | L2 | L4(P0+P1 后) | +| D11 | 可观测性 (OTel) | L1(G9 差距) | L3(P0 后) | +| D12 | 部署企业级 (K8s/密钥) | L2 | L4(P0 后) | +| D13 | OS 沙箱分级 | L1(G3 差距) | L3(P1 R7 后) | +| D14 | 多端覆盖 | L1(G4 差距) | L2(P2 R13 后) | + +> **G-label 双轨制定义:** +> - **Eval-G\***:横向评估差距标签(G1=企业身份 / G3=OS 沙箱 / G4=多端 / G5=Skill 生态 / G9=可观测性),见上表 D 维度映射。 +> - **Brainstorm-G\***:`docs/brainstorms/2026-06-29-advanced-agent-gap-optimization-requirements.md` 差距标签(G3=tool schema / G5=函数级分 shard / G6=SOLO 状态机 / G8=delta_flush),Wave 前缀区分。 + +### Implementation Units + +### U1. OTel 默认启用 (R5) + +- **Goal:** 将 OTel 从"未声明依赖 + 静默 NoOp 回退"升级为"依赖已声明 + 默认启用 + 指标可观测"。 +- **Requirements:** R5。 +- **Dependencies:** 无(U1 是 P0 基础,其他 unit 依赖 U1)。 +- **Files:** + - `pyproject.toml`(添加 4 个 opentelemetry-* 依赖到 `[project] dependencies`) + - `agentkit.yaml`(取消 `telemetry:` 段注释,设 `enabled=true` + `otlp_endpoint`) + - `src/agentkit/telemetry/tracer.py`(移除 ImportError 静默回退,改为直接 import;`init_telemetry` 在 `enabled=true` 但 init 失败时 log error 并回退 NoOp,而非 ImportError 静默) + - `src/agentkit/telemetry/metrics.py`(移除 try/except ImportError,改为直接 import) + - `src/agentkit/server/app.py` 或 `src/agentkit/server/__init__.py`(启动时调用 `init_telemetry`,添加 FastAPIInstrumentor) + - `tests/unit/test_telemetry_init.py`(新增) +- **Approach:** + 1. 将 `opentelemetry-api` / `opentelemetry-sdk` / `opentelemetry-exporter-otlp-proto-grpc` / `opentelemetry-instrumentation-fastapi` 添加到 `pyproject.toml` `[project] dependencies`。 + 2. 移除 `tracer.py:129` 与 `metrics.py:7` 的 `try/except ImportError` 静默回退,改为直接 import(依赖已声明,ImportError 是 bug 不是配置)。 + 3. `init_telemetry` 保留 `enabled=false` → NoOp 路径,但 ImportError 不再 catch(直接抛出)。 + 4. 取消 `agentkit.yaml:85-90` 注释,设 `telemetry.enabled=true` + `telemetry.otlp_endpoint=http://otel-collector:4317` + `telemetry.service_name=fischer-agentkit`。 + 5. 在 server 启动时调用 `init_telemetry(TelemetryConfig(...))` 并 `FastAPIInstrumentor.instrument_app(app)`。 +- **Patterns to follow:** `src/agentkit/telemetry/tracer.py:103-134`(现有 init_telemetry 逻辑)、`src/agentkit/telemetry/metrics.py:32-58`(现有 meter/counter 模式)。 +- **Test scenarios:** + - `init_telemetry(enabled=True, otlp_endpoint="...")` 初始化 `OTelTracer`,`get_tracer()` 返回 `OTelTracer` 实例 + - `init_telemetry(enabled=False)` 返回 `NoOpTracer`(配置开关,非 ImportError 回退) + - `opentelemetry-*` 未安装时 `import telemetry.tracer` 抛出 `ImportError`(非静默 NoOp)—— 验证依赖已声明后此场景不应发生,但测试确认 ImportError 不被 catch + - FastAPI 请求触发 span(`agent.request.total` counter + `agent.execution.duration` histogram) + - LLM 调用触发 `gen_ai.usage.tokens` histogram + - `agentkit.yaml` `telemetry.enabled=false` 时 server 启动成功且 metrics 为 NoOp + - `agentkit.yaml` 无 `telemetry` 段时 server 启动成功且默认 NoOp(向后兼容) +- **Verification:** `agentkit serve` 启动时 log 显示 "Telemetry initialized with endpoint: ...";`agentkit gui` 发送一条 chat 消息后,OTel collector 收到 span + metric。 + +### U2. Prompt Cache 全链路 + 本地推理 Fallback (R3) + +- **Goal:** 验证 prompt cache 路由到 volatile 层、`prompt_cache_enable` 默认 true、命中率监控接入 OTel;配置本地推理 fallback provider 作为等保合规替代。 +- **Requirements:** R3 + R3 P0 验收门槛。 +- **Dependencies:** U1(OTel metrics 用于 cache hit/miss 监控)。 +- **Files:** + - `src/agentkit/core/react.py`(`_build_system_message` 现有,`_prompt_cache_enable` 默认值校验) + - `src/agentkit/llm/gateway.py` 或 `src/agentkit/llm/.py`(添加 cache hit/miss metric emission) + - `agentkit.yaml`(`llm.providers` 段添加 fallback provider 配置模板,`prompt_cache_enable` per-provider) + - `src/agentkit/llm/pii_filter.py`(新增,本地 NER 模型 PII 脱敏 hook,fail-closed) + - `tests/unit/test_prompt_cache_layers.py`(扩展,新增 cache hit/miss metric 测试) + - `tests/unit/test_pii_filter.py`(新增) +- **Approach:** + 1. 验证 `ReActEngine._prompt_cache_enable` 默认 `True`(从 `agentkit.yaml` 或构造参数)。 + 2. 在 LLM gateway 的 `chat_stream` 添加 cache hit/miss metric:Anthropic 响应含 `usage.cache_read_input_tokens` > 0 时 emit `prompt_cache.hit`,否则 emit `prompt_cache.miss`。 + 3. `agentkit.yaml` `llm.providers` 段添加 fallback provider 配置模板(DashScope / DeepSeek / Doubao OpenAI-compatible API),fallback providers 默认 `prompt_cache_enable=false`。 + 4. 新增 `src/agentkit/llm/pii_filter.py`:本地 NER 模型(LAC/HanLP,部署在客户内网)识别 PII + 替换/掩码;脱敏前后记录 hash 用于审计(原文不落盘);脱敏失败时 `fail-closed`(拒绝发送至 Anthropic);仅在本地 fallback 可用时降级处理。 + 5. 脱敏覆盖率与漏报率纳入 OTel 指标(`pii_filter.coverage` / `pii_filter.false_negative`)。 +- **Patterns to follow:** `tests/unit/test_prompt_cache_layers.py:46-58`(现有 Anthropic cache_control 测试)、`src/agentkit/llm/` 现有 LiteLLM 适配层。 +- **Test scenarios:** + - Covers AE-F1(prompt cache 全链路):`prompt_cache_enable=True` + Anthropic provider → `cache_control` 透传到 content blocks(现有测试) + - `prompt_cache_enable=True` + 非 Anthropic provider → 不设 `cache_control`,返回字符串 concat(现有测试) + - cache hit 时 emit `prompt_cache.hit=1` OTel metric;cache miss 时 emit `prompt_cache.miss=1` + - fallback provider(DashScope)配置不设 `cache_control`,正常完成 chat + - PII 脱敏:输入含手机号 → 输出替换为 `[REDACTED]` + 记录 hash + - PII 脱敏失败(NER 模型不可用)→ `fail-closed` 抛出,不发送至 Anthropic + - fallback provider 可用时,脱敏失败降级发送至 fallback provider + - 脱敏覆盖率 metric 在 OTel 可观测 +- **Verification:** `agentkit chat` 发送含 PII 的消息,确认 Anthropic 收到的是脱敏后 prompt;OTel collector 收到 `prompt_cache.hit/miss` + `pii_filter.coverage` 指标。 + +### U3. Residual P2 Findings 修复 (R4) + +- **Goal:** 修复 8 项 residual P2 findings(3 项 feat-ui-ue-enhancement + 5 项 feat-bitable-enhancement)。 +- **Requirements:** R4。 +- **Dependencies:** 无(独立 bug fix 批次)。 +- **Files:** + - `src/agentkit/experts/_phase_executor.py`(expert_step payload 补字段) + - `src/agentkit/experts/orchestrator.py`(team_synthesis 中断时发终结事件) + - `src/agentkit/server/frontend/src/stores/chatStream.ts`(synthesis_id 去重) + - `src/agentkit/bitable/repository.py`(DR-1:`text()` → 参数化查询 / ORM) + - `src/agentkit/server/frontend/src/components/bitable/ViewConfigPanel.vue`(DR-2:props 透传确认) + - `src/agentkit/bitable/service.py`(DR-4:`delete_view` 原子化事务) + - `src/agentkit/tools/bitable_tool.py`(DR-5:`_update_field` type 变更返回 422) + - `src/agentkit/server/frontend/src/styles/bitable-tokens.css`(DR-3:无,token 测试新增) + - `tests/unit/test_phase_executor_streaming.py`(扩展) + - `tests/unit/test_bitable_repository.py`(扩展,DR-1/DR-4) + - `tests/unit/test_bitable_service.py`(扩展,DR-4) + - `tests/unit/test_bitable_tool.py`(扩展,DR-5) + - `tests/frontend/bitable-tokens.test.ts`(新增,DR-3) +- **Approach:** 逐项修复: + 1. **expert_step payload**:`_phase_executor.py:241` 的 `expert_step` 事件 payload 补齐 `expert_name` / `expert_color` / `content` / `step` 字段,与前端 `WsServerMessage` 契约对齐。 + 2. **team_synthesis 孤儿 milestone**:`orchestrator.py:293` 流式中断(异常/取消)时发 `team_synthesis` 终结事件(`status=error|cancelled`),前端不再永久转圈。 + 3. **synthesis_id 去重**:`chatStream.ts:880` `team_synthesis_chunk` 占位匹配用 `synthesis_id` 去重,避免附身到上一次孤儿 milestone。 + 4. **DR-1 SQL 注入**:`repository.py:660, 778-779` 的 `text()` 调用迁移到 ORM `select()` / `delete()` 或参数化 `text("... :param", {"param": value})`。 + 5. **DR-2 ViewConfigPanel**:确认 props 透传 + event emit wiring(quick review,可能无需代码变更)。 + 6. **DR-4 TOCTOU**:`service.delete_view` 的 sibling-count 检查 + delete 移入 repository 的原子条件 delete(`DELETE ... WHERE table_id=:t AND (SELECT COUNT(*) FROM views WHERE table_id=:t) > 1 RETURNING id`)或 `SELECT ... FOR UPDATE` 事务。 + 7. **DR-5 _update_field**:`bitable_tool.py` 的 `_update_field` 在收到 `type` 参数时返回 422(unsupported field type change),而非静默成功;或扩展 `UpdateFieldRequest` 接受 `field_type`。 + 8. **DR-3 design token**:新增 `tests/frontend/bitable-tokens.test.ts`,解析 CSS 文件并断言 `--bitable-color-*` / `--bitable-cf-*` / `--bitable-drawer-width` 在 `:root` 定义。 +- **Patterns to follow:** `docs/residual-review-findings/feat-ui-ue-enhancement.md`、`docs/residual-review-findings/feat-bitable-enhancement.md`、现有 `_phase_executor.py` 流式 handler 模式。 +- **Test scenarios:** + - Covers F1(expert_step):`expert_step` 事件 payload 含 `expert_id` / `expert_name` / `expert_color` / `content` / `step` 全字段 + - Covers F2(team_synthesis):流式中断后 `team_synthesis` 终结事件发出,前端 milestone 关闭 + - Covers F3(synthesis_id):`team_synthesis_chunk` 用 `synthesis_id` 去重,孤儿 milestone 不被附身 + - DR-1:`text()` SQL 调用迁移后,参数化查询不接受字符串拼接的用户输入 + - DR-4:并发删除最后两个 view,其中一个返回 409(不变量"table 至少 1 view"保持) + - DR-5:`update_field` 含 `type` 参数时返回 422(非静默成功) + - DR-3:CSS token 测试断言 `:root` 含所有必需 token +- **Verification:** `pytest tests/unit/test_phase_executor_streaming.py tests/unit/test_bitable_repository.py tests/unit/test_bitable_service.py tests/unit/test_bitable_tool.py -v` 全绿;`npm run test:frontend` DR-3 通过。 + +### U4. SSO 集成 — OIDC + SAML + SCIM (R1) + +- **Goal:** 集成 OIDC + SAML 2.0 SSO(飞书 IDP / Azure AD / 阿里云 IDaaS)+ SCIM 2.0 最小子集(用户创建/禁用)+ 手动 deprovisioning + RBAC 审计。 +- **Requirements:** R1, R1a, R1b, R1c。 +- **Dependencies:** U1(审计日志接入 OTel)。 +- **Files:** + - `src/agentkit/server/auth/providers/oidc.py`(新增,替换 `oidc_stub.py`) + - `src/agentkit/server/auth/providers/saml.py`(新增,SAML 2.0 适配器) + - `src/agentkit/server/auth/scim/__init__.py`(新增) + - `src/agentkit/server/auth/scim/router.py`(新增,`/scim/v2/Users` POST/PATCH/GET) + - `src/agentkit/server/auth/scim/handlers.py`(新增) + - `src/agentkit/server/auth/scim/models.py`(新增,SCIM User Schema) + - `src/agentkit/server/routes/auth.py`(新增 `/auth/oauth/{provider}/redirect` + `/callback` 路由) + - `src/agentkit/server/auth/dependencies.py`(多 IDP wiring,按 `auth.provider` 配置选择) + - `src/agentkit/server/auth/idp_registry.py`(新增,多 IDP 注册表 + 元数据热轮换) + - `src/agentkit/server/auth/audit.py`(新增或扩展,RBAC 变更 + deprovisioning 审计日志) + - `pyproject.toml`(`authlib>=1.3`, `python3-saml>=1.16`) + - `agentkit.yaml`(`auth` 段多 IDP 配置模板) + - `tests/unit/test_oidc_provider.py`(新增) + - `tests/unit/test_saml_provider.py`(新增) + - `tests/unit/test_scim_router.py`(新增) + - `tests/unit/test_auth_audit.py`(新增) + - `tests/integration/test_sso_oidc_flow.py`(新增,标记 `integration`) +- **Approach:** + 1. **OIDC 适配器**(KTD-2):用 `authlib` 实现 redirect flow(`auth.providers.oidc.OIDCProvider`),替换 `StubOIDCProvider`。路由 `/auth/oauth/{provider}/redirect` → IDP authorize;`/auth/oauth/{provider}/callback` → token exchange + userinfo + 本地用户查/建(按 `sub` 主键,禁止邮箱合并 R1a)。 + 2. **SAML 适配器**:用 `python3-saml` 实现 ACS(Assertion Consumer Service)。`/auth/saml/{provider}/acs` 消费 assertion,提取 NameID 作为 subject identifier。 + 3. **多 IDP 信任边界**(R1a):`idp_registry.py` 管理多个 IDP 配置(签名证书 + endpoints);IDP 元数据(签名证书)支持热轮换(不重启服务);任一 IDP 不可达时该路径默认拒绝 + 明确错误,不影响其他 IDP;账号关联以 IDP 声明的 `sub`/`NameID` 为主键,禁止仅凭邮箱合并。 + 4. **SCIM 2.0 最小子集**(R1):`/scim/v2/Users` POST(创建)/ PATCH(禁用 `active=false`)/ GET(查询)。SCIM 推送失败触发实时告警(非等 cron)。 + 5. **过渡期 deprovisioning**(R1b):管理员手动 deprovisioning 接口(强制 operator/admin RBAC),每次调用记录 actor / target_user / reason / source=`scim_fallback|manual` / timestamp,接入 U1 OTel 审计通道;IDP 侧账号状态周期对账(每日 cron);高权限账号(operator/admin/danger tier)1h SLA,通过 break-glass 手动接口即时执行,cron 频率提升至每小时。 + 6. **RBAC 角色变更审计**(KTD-8,OQ14 缓解):新增 `audit.py` 记录 RBAC 角色变更(actor / target_user / role_before / role_after / reason / source),接入 R7a 同款审计日志通道。 + 7. **单租户假设**(R1c):数据模型 / RBAC / 审计 schema / SCIM 主键按单租户设计;在 R18 触发条件中增加"架构性 reversal cost"维度;pre-POC review 早期识别 multi-tenancy 需求。 +- **Patterns to follow:** `src/agentkit/server/auth/providers/base.py`(`AuthProvider` Protocol)、`src/agentkit/server/auth/providers/local.py`(现有 Local provider 模式)、`src/agentkit/server/auth/jwt_utils.py`(JWT 签发)。 +- **Test scenarios:** + - Covers F1(SSO 登录):OIDC redirect → callback → JWT 签发,`auth_sessions.auth_provider='oidc-feishu'` 写入 + - Covers F2(多 IDP):飞书 + Azure AD + 阿里云 IDaaS 三 IDP 并存,各自独立认证 + - 多 IDP subject identifier 防账号接管:不同 IDP 同邮箱 → 不合并为同一本地用户 + - IDP 不可达:返回明确错误 + 其他 IDP 路径不受影响 + - IDP 签名证书热轮换:不重启服务即可更新 + - SAML ACS:消费 assertion + 提取 NameID + - SCIM POST `/scim/v2/Users`:创建用户 + - SCIM PATCH `active=false`:禁用用户 + - SCIM 推送失败:触发实时告警 + - 手动 deprovisioning:operator/admin RBAC 强制 + 审计日志写入 + - break-glass 接口:即时回收高权限账号 + - 每日 cron 对账:检测 IDP 侧 vs 本地状态 drift + - RBAC 角色变更:审计日志记录 actor / target / before / after / reason +- **Verification:** `pytest tests/unit/test_oidc_provider.py tests/unit/test_saml_provider.py tests/unit/test_scim_router.py tests/unit/test_auth_audit.py -v` 全绿;`pytest -m integration tests/integration/test_sso_oidc_flow.py` 通过(需 mock IDP);手动测试飞书 IDP 登录。 + +### U5. K8s Helm Chart + Sealed Secrets + 离线安装包 (R2) + +- **Goal:** 产出政企生产级 K8s 部署 artifacts(Helm Chart + 离线安装包 + 密钥管理 + 运维文档)。 +- **Requirements:** R2, R2a。 +- **Dependencies:** U1, U2, U3, U4(打包所有 P0 工作为可部署产物)。 +- **Files:** + - `deploy/helm/agentkit/Chart.yaml`(新增) + - `deploy/helm/agentkit/values.yaml`(新增,列出所有 secret slot) + - `deploy/helm/agentkit/templates/_helpers.tpl`(新增) + - `deploy/helm/agentkit/templates/deployment.yaml`(新增) + - `deploy/helm/agentkit/templates/service.yaml`(新增) + - `deploy/helm/agentkit/templates/ingress.yaml`(新增,TLS 配置) + - `deploy/helm/agentkit/templates/configmap.yaml`(新增,非密配置) + - `deploy/helm/agentkit/templates/sealed-secret.yaml`(新增,Sealed Secrets 模板) + - `deploy/helm/agentkit/templates/external-secret.yaml`(新增,External Secrets Operator 模板) + - `deploy/helm/agentkit/templates/serviceaccount.yaml`(新增,workload identity 注入) + - `deploy/offline/Makefile`(新增) + - `deploy/offline/scripts/build-images.sh`(新增) + - `deploy/offline/scripts/package-chart.sh`(新增) + - `deploy/offline/scripts/install.sh`(新增,离线安装脚本) + - `docs/deployment/k8s-install.md`(新增) + - `docs/deployment/key-rotation-runbook.md`(新增,R2a 要求) + - `docs/deployment/secret-inventory.md`(新增,密钥清单) +- **Approach:** + 1. **Helm Chart 结构**:`deploy/helm/agentkit/` 含 Chart.yaml / values.yaml / templates/。values.yaml 显式列出所有 secret slot(KTD-4):JWT 签名密钥 / API Key / DB 凭据 / IDP 签名证书 / 第三方 API Key / TLS 服务器证书私钥 / mTLS 客户端证书私钥 / KMS-HSM PKCS#11 PIN / OTel exporter token / Redis-PG 密码。 + 2. **Sealed Secrets / External Secrets Operator**:templates 提供 `sealed-secret.yaml` 和 `external-secret.yaml` 两种模板,客户按环境选择。禁用 plain Kubernetes Secret + Git 提交。 + 3. **KMS/HSM workload identity**:`serviceaccount.yaml` 配置 projected service account token 注入,KMS/HSM 访问凭证通过 workload identity(非静态 Secret)。 + 4. **离线安装包**:`deploy/offline/scripts/build-images.sh` 构建 Docker image tarball + `package-chart.sh` 打包 Helm chart tarball + `install.sh` 在离线环境解压安装。覆盖原生 K8s 1.28+ / Helm 3.x(OQ5:信创 K8s / OpenShift 延后到 P1 R12)。 + 5. **密钥轮换 runbook**:`docs/deployment/key-rotation-runbook.md` 文档化每个 secret slot 的轮换流程。 + 6. **传统 IDC fallback**(OQ8):不在 P0 范围,docker-compose 加固延后到 R18 触发条件识别后评估。文档中注明假设。 +- **Patterns to follow:** Helm Chart 标准结构、bitnami Sealed Secrets chart、External Secrets Operator 文档。 +- **Test scenarios:** + - `helm template deploy/helm/agentkit/` 渲染出有效 manifests(无模板错误) + - `helm install --dry-run --debug` 成功(values 校验通过) + - `helm lint deploy/helm/agentkit/` 无 warning + - Sealed Secret template 覆盖所有 10 个 secret slot + - External Secret template 覆盖所有 10 个 secret slot + - serviceaccount 含 projected token 注入配置 + - `deploy/offline/scripts/build-images.sh` 产出 image tarball + - `deploy/offline/scripts/package-chart.sh` 产出 chart tarball + - 离线 install.sh 在 mock 离线环境成功安装 +- **Verification:** `helm template` + `helm lint` 通过;离线安装包在测试 K8s 集群(minikube/kind)成功部署;密钥轮换 runbook 评审通过。 + +### U6. 等保三级文档准备 (R-等保) + +- **Goal:** 启动等保三级文档与控制点清单编写(6 维度),目标认证 readiness 在政企 POC 采购关闭前完成。 +- **Requirements:** R-等保。 +- **Dependencies:** U1(OTel 审计日志)、U4(SSO/SCIM/RBAC)、U5(K8s 部署/密钥管理)提供应用/网络/主机维度输入。 +- **Files:** + - `docs/compliance/dengbao-level3/00-overview.md`(新增,总览) + - `docs/compliance/dengbao-level3/01-physical.md`(新增,物理安全) + - `docs/compliance/dengbao-level3/02-network.md`(新增,网络安全) + - `docs/compliance/dengbao-level3/03-host.md`(新增,主机安全) + - `docs/compliance/dengbao-level3/04-application.md`(新增,应用安全 — 引用 R1/R5/R7a) + - `docs/compliance/dengbao-level3/05-data.md`(新增,数据安全 — 引用 R3 PII 脱敏 / R11a 加密 P1 参考) + - `docs/compliance/dengbao-level3/06-management.md`(新增,管理安全) + - `docs/compliance/dengbao-level3/control-points.yaml`(新增,控制点清单 + 实现映射) +- **Approach:** + 1. 按 GB/T 22239-2019 等保三级 6 维度编写文档:物理 / 网络 / 主机 / 应用 / 数据 / 管理。 + 2. **物理**:依赖客户机房(K8s 部署在客户内网),文档描述 AgentKit 部署对物理环境的假设。 + 3. **网络**:引用 U5 Helm Chart 的 NetworkPolicy / Ingress TLS 配置;OTel exporter mTLS。 + 4. **主机**:引用 U5 容器镜像安全(非 root user / distroless base / 漏洞扫描)。 + 5. **应用**:引用 U4 SSO 多 IDP / RBAC 审计 / SCIM deprovisioning;U1 OTel 审计日志保留 180 天(KTD-9);R7a 6 层终端安全(现有)。 + 6. **数据**:引用 U2 R3 PII 脱敏(本地 NER + fail-closed + hash 审计);R11a 国密加密(P1 参考,文档中标注)。 + 7. **管理**:引用密钥轮换 runbook(U5);break-glass 告警流程(U4 R1b)。 + 8. `control-points.yaml` 列出每个控制点 + AgentKit 实现映射(已实现 / P0 / P1 / 待评估)。 +- **Patterns to follow:** GB/T 22239-2019 等保三级标准、`docs/solutions/` 已记录的安全模式。 +- **Test expectation:** none — documentation deliverable, no behavioral change。 +- **Verification:** 文档由安全/合规 stakeholder 评审;`control-points.yaml` 通过 schema 校验(每个控制点含 id / dimension / title / implementation / status 字段)。 + +### Verification Contract + +- **Unit 测试:** 每个 U1-U5 unit 的 `tests/unit/` 测试全绿(`pytest tests/unit/ -x -q`)。 +- **Integration 测试:** U4 SSO OIDC flow 测试通过(`pytest -m integration tests/integration/test_sso_oidc_flow.py`)。 +- **前端测试:** U3 DR-3 design token 测试通过(`npm run test:frontend`);U3 前端 fix 不破坏现有 e2e。 +- **部署验证:** U5 Helm Chart `helm template` + `helm lint` + 离线安装包在 minikube/kind 部署成功。 +- **启动验证:** U1 `agentkit serve` / `agentkit gui` 启动时 OTel 初始化 log 出现;`telemetry.enabled=false` 时向后兼容。 +- **端到端验证:** U2 `agentkit chat` 发送含 PII 的消息,Anthropic 收到脱敏 prompt + OTel 收到 cache hit/miss metric。 +- **审计验证:** U4 手动 deprovisioning + RBAC 角色变更触发审计日志写入(OTel span + 日志文件)。 +- **文档验证:** U6 等保文档由 stakeholder 评审;`control-points.yaml` schema 校验通过。 +- **Lint / 格式化:** `ruff check src/ && ruff format src/` 无错误;`npm run typecheck` 无错误。 + +### Definition of Done + +- P0 6 项全交付(R1 + R2 + R3 + R4 + R5 + R-等保): + - R5(U1):OTel 依赖声明 + 默认启用 + NoOp 不再静默回退。 + - R3(U2):prompt cache 全链路验证 + 本地推理 fallback 配置 + PII 脱敏 + OTel 命中率监控。 + - R4(U3):8 项 residual findings 修复 + 测试覆盖。 + - R1(U4):OIDC + SAML SSO + SCIM 最小子集 + 多 IDP 信任 + deprovisioning + RBAC 审计。 + - R2(U5):Helm Chart + Sealed Secrets + 离线安装包 + 密钥轮换 runbook。 + - R-等保(U6):6 维度文档 + 控制点清单,readiness 在政企 POC 采购关闭前完成。 +- 横向评估 Eval-D9(企业身份)从 L1 提升到 L3;Eval-D11(可观测性)从 L1 提升到 L3;Eval-D12(部署企业级)从 L2 提升到 L4;总分从 42 提升到 47+/56。 +- 所有 unit 测试 + integration 测试 + 前端测试通过。 +- 政企客户 POC 可基于 P0 交付物启动(首个政企客户 POC 落地)。 +- P1 / P2 阵列不在本次 DoD,作为 follow-up plan 推进。 + +### Risks & Dependencies + +- **R-风险1. Anthropic API 依赖与等保冲突**(OQ13/OQ15):R3 依赖 Anthropic 境外服务,与等保三级数据本地化冲突。缓解:本地推理 fallback(U2)+ PII 脱敏(U2)+ DPA 接受 TTL 残留风险。残留:Anthropic 路径 prompt cache 在 fallback 路径下价值为零(OQ13 已记录为战略缺口)。 +- **R-风险2. 单租户 reversal cost 高**(OQ16):R1c 单租户假设是 load-bearing 决策。缓解:R18 触发条件增加"架构性 reversal cost"维度 + pre-POC review 早期识别。残留:若销售阶段识别到 multi-tenancy 需求,retrofit 成本极高(数据迁移 + RBAC 重构 + 审计 schema 变更)。 +- **R-风险3. SCIM group→role 缺失**(OQ14):P0 SCIM 仅用户创建/禁用,角色分配手动。缓解:RBAC 角色变更审计 logging(KTD-8)。残留:等保三级要求特权账号变更可审计,手动分配 + 审计满足,但 group→role 自动化延后。 +- **R-风险4. 传统 IDC 客户不兼容**(OQ8):P0 K8s-only,传统 IDC 客户(华为云 Stack / 阿里云 Apsara Stack)无法部署。缓解:R18 触发条件识别"传统 IDC 客户"作为升级触发。残留:docker-compose 加固 fallback 不在 P0。 +- **R-风险5. 等保认证周期长**:R-等保文档 P0 启动,认证审计在 P1 R10。若政企 POC 采购关闭前认证 readiness 未完成,阻塞 POC。缓解:文档与 MVP 并行推进 + 政企专属反馈通道(R18a)。 +- **依赖:** K8s 1.28+ / Helm 3.x 部署环境;飞书 / Azure AD / 阿里云 IDaaS IDP 协议稳定性;Anthropic API 持续支持 `cache_control`;`complex-task-quality-loop` plan 的 11 项需求 + 10 KTD 作为 P0 启动基线。 + +### System-Wide Impact + +- **认证体系:** U4 SSO 改变登录流程 —— 从 LocalAuthProvider(username+password)扩展到多 IDP OIDC/SAML + SCIM。`auth_sessions.auth_provider` 字段记录来源;现有 Local provider 保持向后兼容。 +- **可观测性:** U1 OTel 默认启用 —— 所有 FastAPI 请求 + LLM 调用 + tool 调用 emit span/metric。运维需部署 OTel Collector + 后端(Prometheus/Tempo/Loki)。 +- **部署:** U5 Helm Chart 改变交付方式 —— 从 docker-compose dev 部署扩展到 K8s 生产部署。运维需熟悉 Helm + Sealed Secrets/External Secrets Operator。 +- **合规:** U6 等保文档 + U1 审计日志 180 天保留 —— 影响存储容量规划(OTel span + 审计日志)。 +- **前端:** U3 expert_step payload + team_synthesis 修复 —— 前端 `WsServerMessage` 契约对齐,需同步前端代码。 +- **配置:** `agentkit.yaml` 新增 `auth` 段(多 IDP)+ `telemetry` 段(默认启用)+ `llm.providers` fallback 模板。 + +### Open Questions (Planning-Resolved) + +- OQ1, OQ2:已关闭(plan 已记录)。 +- OQ3(R7 沙箱分级):R7 在 P1,P0 不需要沙箱分级。P1 plan 将决定是否三个 tier 全完成或仅补 read-only。 +- OQ4(SSO 协议优先级):**已解决** — OIDC 优先(KTD-2),SAML 第二阶段。三个目标 IDP 均支持 OIDC。 +- OQ5(K8s 兼容矩阵):**已解决** — P0 原生 K8s 1.28+ / Helm 3.x;信创 K8s / OpenShift 延后到 P1 R12;传统 IDC fallback 在 R18 触发条件识别后评估。 +- OQ6(信创 DB):延后到 P1(R12)。P0 假设 PostgreSQL。 +- OQ7(Subagents worktree max_threads):延后到 P1(R6)。 +- OQ8(传统 IDC):**已解决** — P0 K8s-only;docker-compose 加固不在 P0;R18 触发条件识别"传统 IDC 客户"作为升级触发。文档已标注假设。 +- OQ11(D-label 维度):**已解决** — 14 维度编号已在 HTD 部分列出完整表格。 +- OQ12(G-label 双轨制):**已解决** — 两套 gap scheme 定义已在 HTD 部分显式列出。 +- OQ13(R3 cache 价值):**已解决** — R3 P0 优先级服务中大型企业 Anthropic-eligible 客户段;fallback 路径不依赖 cache 价值(KTD-5)。战略缺口已记录,不在 P0 解决。 +- OQ14(SCIM group→role):**已解决** — SCIM group→role 延后;P0 SCIM 用户创建/禁用 + 手动 RBAC + 审计 logging(KTD-8)。 +- OQ15(DPA cache 删除权):**已解决** — DPA 明确 TTL-only 删除(最长 1 小时);fallback providers 默认 `prompt_cache_enable=true`;Anthropic 路径客户签 DPA 接受 TTL 残留风险(KTD-5)。 +- OQ16(R1c 单租户 reversal cost):**已解决** — reversal cost 评估加入 R18 触发条件;pre-POC review 早期识别 multi-tenancy 需求(KTD-1)。单租户保持 load-bearing 假设。 + +### Open Questions (Deferred to P1/P2 Planning) + +- OQ9(R6/R8/R9 P1→P2 重分类):延后到 P1 plan。P1 plan 将评估 P1 theme 是否扩展为"企业级基线 + 性能优化",或重分类至 P2。scope-guardian 关注已记录。 +- OQ10(P2 不深化护城河):延后到 P2 plan。P2 plan 将决定增加深化护城河的 items,还是修订 Goal statement 承认 P2 是新能力。scope-guardian 关注已记录。 + +--- + +## ce-doc-review 追踪(2026-07-05) + +**Round 1:** 应用 12 项 Apply 修复 + 5 项 Defer 到 Open Questions(OQ8-OQ12)+ 1 项 safe_auto + 3 项 FYI 观察未决策。 + +**Round 2(re-review):** 应用 1 项 safe_auto(P0 计数 5→6 修正)+ 10 项 Apply(含 R5 一行配置修正 / R3 fallback 提升验收 / R7a 命令面约束 / R-等保 gate 明确 / R11a 降级攻击防护 / R3 PII 脱敏机制 / R2a 密钥清单 / R1b RBAC + 高权限 SLA / R5'a 保留期 + Span 脱敏扩展 / Key Decisions R5 修正)+ 4 项 Defer 到 Open Questions(OQ13-OQ16)+ 4 项 FYI 观察未决策。详情见会话记录。 diff --git a/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md index 581b001..3fd348a 100644 --- a/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md +++ b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md @@ -14,7 +14,19 @@ verification). They are **actionable** (`autofix_class: manual`) but were **not applied** in Step 5 because they need design input or app+chart coordinated changes. They are durably recorded here per lfg SKILL.md Step 6. -## P1 — High-impact defects +## Resolution Status (updated 2026-07-06) + +- **P1 (7 项)**: 全部在 commit `cb278bb` (PR #24) 修复 +- **P2 (10 项)**: 全部在 commit `cb278bb` (PR #24) 修复 +- **P3 (15 项, 原文档计数为 10)**: + - SEC-3, API-13: 在 commit `cb278bb` (PR #24) 修复（文档未及时更新） + - 其余 13 项在 `feat/p3-residual-fixes` 分支修复 + +**全部 27+2=29 项 residual findings 已清零。** + +--- + +## P1 — High-impact defects ✅ ALL FIXED (commit cb278bb) - **P1 / `src/agentkit/llm/pii_filter.py` + `src/agentkit/llm/gateway.py`** — PERF-1: PII 脱敏模块未接入生产 LLM 调用路径（死代码）。`filter_messages_pii` @@ -110,73 +122,73 @@ coordinated changes. They are durably recorded here per lfg SKILL.md Step 6. STAND-1: SCIM router 使用 `typing.Any` 7 处，违反 AGENTS.md 禁用 any 规则。 **Fix**: 定义 `SCIMUserResponse` / `SCIMUserListResponse` Pydantic 模型。 -## P3 — Low-impact / narrow scope +## P3 — Low-impact / narrow scope ✅ ALL FIXED - **P3 / `src/agentkit/server/auth/providers/oidc.py:185-197`** — - SEC-2: OIDC id_token 未按规范验签（key=None + 关闭 iss/exp）。当前 - authlib 1.6.12 fail-closed，非直接绕过，但规范违反且脆弱。**Fix**: 用 - IDP JWKS 公钥验签，校验 iss/aud/exp。 + SEC-2: OIDC id_token 未按规范验签（key=None + 关闭 iss/exp）。✅ FIXED: + 新增 `OIDCConfig.jwks_uri` + `issuer` 字段，`_verify_id_token` 用 JWKS 验签 + + IDToken claims_cls 校验 iss/aud/exp，fail-closed。 - **P3 / `src/agentkit/server/auth/scim/router.py:170-185`** — - SEC-3: SCIM PATCH deprovisioning（`active=false`）不写审计。admin 同等 - 操作有审计，SCIM 没有。**Fix**: 检测到 active true→false 时调用 - `record_deprovision(source=SOURCE_SCIM)`。 + SEC-3: SCIM PATCH deprovisioning 不写审计。✅ FIXED (commit cb278bb): + active true→false 调用 `record_deprovision(source=SOURCE_SCIM)`，延后到事务提交后。 - **P3 / `src/agentkit/llm/gateway.py:401-414`** — - PERF-6: 流式路径在无 usage 数据时记录"cache miss"，指标误导。**Fix**: - 仅当 `final_usage` 非 None 时才记录 cache metric。 + PERF-6: 流式路径在无 usage 数据时记录"cache miss"。✅ FIXED: + `_record_prompt_cache_metric` 接受 `TokenUsage | None`，None 时直接 return。 - **P3 / `src/agentkit/llm/gateway.py:450-457`** — - PERF-7: 非 Anthropic provider 调用膨胀 `prompt_cache.miss` 计数器。 - **Fix**: 仅对 Anthropic provider 记录 cache metric。 + PERF-7: 非 Anthropic provider 膨胀 `prompt_cache.miss` 计数器。✅ FIXED: + 仅对 model 含 "anthropic"/"claude" 的 provider 记录 cache metric。 - **P3 / `src/agentkit/server/auth/scim/router.py:171-187`** — - API-12: SCIM PATCH 静默忽略不支持的 op/path，无错误反馈。**Fix**: + API-12: SCIM PATCH 静默忽略不支持的 op/path。✅ FIXED: 收集 skipped ops 并返回 400 + `scimType=noTarget`。 - **P3 / `src/agentkit/server/auth/scim/router.py:200-238`** — - API-13: SCIM filter 解析对属性名和操作符大小写敏感，违反 RFC 7644 - 3.4.2.2。**Fix**: 用 `filter.lower()` + 正则 `re.IGNORECASE`。 + API-13: SCIM filter 解析对属性名和操作符大小写敏感。✅ FIXED (commit cb278bb): + `filter.lower()` 大小写不敏感匹配。 - **P3 / `src/agentkit/server/auth/scim/router.py:154-197`** — - API-14: SCIM PATCH `op.path==None` 时仅更新 active，丢弃 value 中的其他 - 字段（部分实现静默丢数据）。**Fix**: 遍历 value 的 keys 对每个已知属性 - 执行对应 UPDATE。 + API-14: SCIM PATCH `op.path==None` 时仅更新 active，丢弃其他字段。✅ FIXED: + path==None + value 是 dict 时遍历所有 keys（active/userName）执行对应 UPDATE。 - **P3 / `src/agentkit/server/auth/models.py:609-639`** — - MIG-2: schema 迁移幂等但缺少 V4→V5 的显式迁移记录。**Fix**: 未来新增列 - 时按 schema_version 差值执行 ALTER TABLE。 + MIG-2: schema 迁移幂等但缺少 V4→V5 的显式迁移记录。✅ FIXED: + 新增 `_SCHEMA_MIGRATIONS` 注册表 + `_run_schema_migrations` 按 version 差值执行。 - **P3 / `deploy/helm/agentkit/values.yaml:8-11`** — - DEPLOY-9: 镜像仅 tag 固定（0.1.0），未做 digest pinning。政企生产场景 - 要求 digest pinning 防篡改。**Fix**: 支持可选 `image.digest` 字段。 + DEPLOY-9: 镜像未做 digest pinning。✅ FIXED: + 新增 `image.digest` 字段，deployment.yaml 优先用 `@`。 - **P3 / `docs/deployment/key-rotation-runbook.md:254-283`** — - DEPLOY-8: runbook 仅 Slot 1-10，缺独立 Slot 11（PG 密码）段落。**Fix**: - 拆为两节，Slot 11 内注明须同步更新 `database-url`（Slot 3）。 + DEPLOY-8: runbook 缺独立 Slot 11。✅ FIXED: + 拆为 Slot 10 (Redis) + Slot 11 (PostgreSQL)，Slot 11 注明同步更新 database-url (Slot 3)。 - **P3 / `src/agentkit/bitable/service.py:564-568`** — - MAINT-2: `delete_view` 将两种不同失败原因统一抛 `LastViewDeletionError`， - 误导性错误信息。**Fix**: 区分 `DELETED`/`LAST_VIEW`/`NOT_FOUND` 三态。 + MAINT-2: `delete_view` 将两种失败原因统一抛 `LastViewDeletionError`。✅ FIXED: + 重新检查 view 是否存在区分 LAST_VIEW vs 并发竞争，错误信息含 table_id。 - **P3 / `src/agentkit/server/frontend/src/stores/chatStream.ts:881-887,923-929`** — - MAINT-3: `team_synthesis_chunk` 与 `team_synthesis` 重复同一 `findLastMessage` - 谓词。**Fix**: 提取 `findStreamingSynthesisMilestone(conv, sid)` helper。 + MAINT-3: `team_synthesis_chunk` 与 `team_synthesis` 重复同一谓词。✅ FIXED: + 提取 `findStreamingSynthesisMilestone(conv, sid)` helper。 - **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:75,81,85`** — - MAINT-4: token 声明正则转义重复 3 次。**Fix**: 提取 `tokenDeclPattern(token)` - helper。 + MAINT-4: token 声明正则转义重复 3 次。✅ FIXED: + 提取 `tokenDeclPattern(token)` helper。 - **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:92`** — - STAND-2: test "outside root" 用 replace 仅替换首个 `:root` 块。**Fix**: + STAND-2: test "outside root" 用 replace 仅替换首个 `:root` 块。✅ FIXED: 加 `g` 标志 `replace(/:root\s*\{[^}]*\}/g, '')`。 - **P3 / `src/agentkit/server/app.py:1401`** — - STAND-3: SCIM router 挂载在 `/api/v1` 前缀下，偏离 RFC 7644 标准 SCIM - 路径 `/scim/v2`。**Fix**: SCIM router 单独挂载不加 `/api/v1` 前缀。 + STAND-3: SCIM router 挂载在 `/api/v1` 前缀下，偏离 RFC 7644 标准。✅ FIXED: + 改挂 `/scim/v2`，异常处理器 scope 检查简化。 --- -**Total residual findings**: 27 (P1=7, P2=10, P3=10) +**Total residual findings**: 29 (P1=7, P2=10, P3=15 — 原文档 P3 计数为 10，实际 15 项) **Applied in Step 5**: 9 fixes (commit `3cf29f4`) -**Deferred to follow-up**: 27 (this file) +**Applied in ce-debug batch (PR #24)**: 17 fixes (commit `cb278bb`) — 全部 P1 + 全部 P2 + SEC-3 + API-13 +**Applied in P3 cleanup (feat/p3-residual-fixes)**: 13 fixes — 其余 P3 +**Deferred to follow-up**: 0 — 全部清零 ✅ diff --git a/src/agentkit/bitable/service.py b/src/agentkit/bitable/service.py index ad81b31..fa72ba3 100644 --- a/src/agentkit/bitable/service.py +++ b/src/agentkit/bitable/service.py @@ -560,12 +560,17 @@ class BitableService: """ view = await self._repo.get_view(view_id) if view is None: - return False + return False # NOT_FOUND deleted = await self._repo.delete_view_if_not_last(view_id, view.table_id) if not deleted: - # Either it was the last view, or a concurrent call raced ahead. - # Either way, the invariant "table has at least 1 view" is preserved. - raise LastViewDeletionError("Cannot delete the last view of a table") + # MAINT-2: 区分 LAST_VIEW vs 并发竞争 — 重新检查 view 是否仍存在。 + # 若已不存在，说明并发调用已删除（目标达成）；否则真的是最后一个 view。 + race_check = await self._repo.get_view(view_id) + if race_check is None: + return True # 并发竞争：已被另一个调用删除，目标已达成 + raise LastViewDeletionError( + f"Cannot delete the last view of table '{view.table_id}' (view_id={view_id})" + ) return True # ── Recalc (U3: formula recalc pipeline) ──────────────── diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index 8e2299a..d82a728 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -440,6 +440,7 @@ class LLMGateway: # Track usage after successful stream latency_ms = (time.monotonic() - start) * 1000 + stream_usage = final_usage # PERF-6: 保留原始 usage（None 表示 provider 未返回） if final_usage is None: final_usage = TokenUsage() cost = self._calculate_cost(final_model, final_usage) @@ -453,7 +454,8 @@ class LLMGateway: department_ids=department_ids, ) # U2 — prompt cache hit/miss OTel counter（streaming path） - self._record_prompt_cache_metric(final_usage, final_model) + # PERF-6/7: 传入原始 stream_usage（可能为 None）+ 仅 Anthropic 记录 + self._record_prompt_cache_metric(stream_usage, final_model) # Empty stream detection: if no content was produced, # raise error so the caller (ReActEngine) can retry with a different model. @@ -490,8 +492,20 @@ class LLMGateway: ) @staticmethod - def _record_prompt_cache_metric(usage: TokenUsage, model: str) -> None: - """Record prompt cache hit/miss OTel counter — Anthropic cache_read_input_tokens > 0 → hit.""" + def _record_prompt_cache_metric(usage: TokenUsage | None, model: str) -> None: + """Record prompt cache hit/miss OTel counter — Anthropic only. + + PERF-6: 仅当 usage 非 None 时记录（provider 未返回 usage 时空 TokenUsage + 会误报 cache miss，膨胀 miss 计数器）。 + PERF-7: 仅对 Anthropic provider 记录（cache_read_input_tokens 是 + Anthropic 专有字段，非 Anthropic provider 永远 cache_hit=False → 误报 miss）。 + """ + if usage is None: + return # PERF-6: 无 usage 数据，不记录 + model_lower = model.lower() + # PERF-7: 仅 Anthropic/Claude 支持 prompt caching + if "anthropic" not in model_lower and "claude" not in model_lower: + return attrs = {"gen_ai.request.model": model} if usage.cache_hit: prompt_cache_hit_counter().add(1, attrs) diff --git a/src/agentkit/server/app.py b/src/agentkit/server/app.py index 6efdbbb..b4d1125 100644 --- a/src/agentkit/server/app.py +++ b/src/agentkit/server/app.py @@ -1398,12 +1398,13 @@ def create_app( app.include_router(auth_routes.router, prefix="/api/v1") app.include_router(auth_routes.admin_router, prefix="/api/v1") # U4 SSO 集成 — SCIM 2.0 最小子集（自带 bearer token 校验） + # STAND-3: 按 RFC 7644 标准，SCIM 端点挂载在 /scim/v2（非 /api/v1） from agentkit.server.auth.scim.router import ( register_scim_exception_handlers, router as scim_router, ) - app.include_router(scim_router, prefix="/api/v1") + app.include_router(scim_router, prefix="/scim/v2") # API-2: 注册 SCIM 异常处理器（scope 到 /scim/v2，非 SCIM 路由走默认格式） register_scim_exception_handlers(app) app.include_router(admin_routes_module.admin_router, prefix="/api/v1") diff --git a/src/agentkit/server/auth/idp_registry.py b/src/agentkit/server/auth/idp_registry.py index e266d57..8e200d4 100644 --- a/src/agentkit/server/auth/idp_registry.py +++ b/src/agentkit/server/auth/idp_registry.py @@ -36,6 +36,11 @@ class OIDCConfig(BaseModel): redirect_uri: str # 可选：scope 覆盖（默认 openid email profile） scope: str = "openid email profile" + # SEC-2: id_token 验签所需 — JWKS URI（IDP discovery 文档的 jwks_uri 字段） + # 未配置时 id_token 回退路径 fail-closed（不再用 key=None 跳过验签）。 + jwks_uri: str | None = None + # SEC-2: id_token iss 校验期望值（IDP discovery 文档的 issuer 字段） + issuer: str | None = None class SAMLConfig(BaseModel): diff --git a/src/agentkit/server/auth/models.py b/src/agentkit/server/auth/models.py index 66504f5..9e9c9e8 100644 --- a/src/agentkit/server/auth/models.py +++ b/src/agentkit/server/auth/models.py @@ -24,7 +24,7 @@ from __future__ import annotations import json import logging import os -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from datetime import datetime, timezone from pathlib import Path @@ -664,6 +664,27 @@ _SCHEMA_VERSION = 5 _META_SCHEMA_VERSION_KEY = "schema_version" +# MIG-2: 显式 schema 迁移注册表 — 按 schema_version 差值执行 ALTER TABLE。 +# 新增列/修改列时在此追加 migration 函数（每个必须 idempotent）。 +# 当前 V5 全为 additive（CREATE TABLE IF NOT EXISTS），无 ALTER TABLE 迁移。 +# 升级路径示例：V5→V6 新增列 → 在此注册 `6: _migrate_v5_to_v6`。 +# type: ignore[misc] — dict value 类型异构（不同 migration 签名一致） +_SCHEMA_MIGRATIONS: dict[int, "Callable[[aiosqlite.Connection], Awaitable[None]]"] = {} + + +async def _run_schema_migrations(db: aiosqlite.Connection, current_version: int) -> None: + """MIG-2: 按 schema_version 差值顺序执行迁移。 + + 每个 migration 函数必须 idempotent（可重复执行不报错）。 + 升级路径：current_version+1 → ... → _SCHEMA_VERSION。 + """ + for v in range(current_version + 1, _SCHEMA_VERSION + 1): + migration = _SCHEMA_MIGRATIONS.get(v) + if migration is not None: + await migration(db) + logger.info(f"Auth DB migration V{v - 1}→V{v} applied") + + async def _get_meta_value(db: aiosqlite.Connection, key: str) -> str | None: """Read a key from the ``auth_meta`` table. Returns ``None`` if missing.""" cursor = await db.execute("SELECT value FROM auth_meta WHERE key = ?", (key,)) @@ -804,6 +825,10 @@ async def init_auth_db(db_path: str | Path | None = None) -> Path: # Record the current schema version (idempotent upsert). current = await _get_meta_value(db, _META_SCHEMA_VERSION_KEY) if current != str(_SCHEMA_VERSION): + # MIG-2: 显式运行 schema_version 差值迁移（ALTER TABLE 等） + current_int = int(current) if current else 0 + if current_int < _SCHEMA_VERSION: + await _run_schema_migrations(db, current_int) await _set_meta_value(db, _META_SCHEMA_VERSION_KEY, str(_SCHEMA_VERSION)) logger.info(f"Auth DB schema version set to {_SCHEMA_VERSION}") diff --git a/src/agentkit/server/auth/providers/oidc.py b/src/agentkit/server/auth/providers/oidc.py index fdb8a30..7be11fd 100644 --- a/src/agentkit/server/auth/providers/oidc.py +++ b/src/agentkit/server/auth/providers/oidc.py @@ -154,7 +154,7 @@ class OIDCProvider: ), ) # 拉取 userinfo（OIDC discovery 的标准端点） - userinfo = await self._fetch_userinfo(client, token, cfg.server_metadata_url) + userinfo = await self._fetch_userinfo(client, token, cfg.server_metadata_url, cfg) sub = str(userinfo.get("sub", "")) if not sub: @@ -174,6 +174,7 @@ class OIDCProvider: client: AsyncOAuth2Client, token: dict[str, Any], metadata_url: str, + cfg: Any, ) -> dict[str, Any]: """从 OIDC userinfo endpoint 拉取用户信息。 @@ -184,21 +185,52 @@ class OIDCProvider: # authlib 的 token 已自动设置到 client 的 token_auth，可直接 GET resp = await client.get(metadata_url) if resp.status_code != 200: - # 退回 token 中的 id_token 解码（authlib 已校验签名） + # SEC-2: 回退到 id_token 解码 — 必须用 JWKS 验签 + 校验 iss/aud/exp + # （原来 key=None + 关闭 iss/exp 是规范违反且脆弱） id_token = token.get("id_token") if id_token: - from authlib.jose import jwt as authlib_jwt # 局部导入避免硬依赖 - - claims = authlib_jwt.decode( - id_token, - None, - claims_options={"iss": {"essential": False}, "exp": {"essential": False}}, - ) # type: ignore[arg-type] - return dict(claims) + return await self._verify_id_token(id_token, cfg) raise ValueError(f"拉取 userinfo 失败: HTTP {resp.status_code}") data = resp.json() return dict(data) if isinstance(data, dict) else {} + async def _verify_id_token(self, id_token: str, cfg: Any) -> dict[str, Any]: + """SEC-2: 用 IDP JWKS 公钥验签 id_token，校验 iss/aud/exp。 + + fail-closed: ``jwks_uri`` 未配置时抛 ValueError（不再用 key=None 跳过验签）。 + ponytail: JWKS 每次回退都拉取（回退路径罕见，不值得缓存）。 + 上限：高频回退场景需加 JWKS 缓存（按 kid 缓存 + TTL）。 + """ + if not cfg.jwks_uri: + raise ValueError( + "SEC-2: id_token 验签需要 OIDCConfig.jwks_uri，但未配置 — " + "无法安全回退到 id_token 解码（fail-closed）" + ) + import httpx + from authlib.jose import JsonWebKey, jwt + from authlib.oidc.core import IDToken + + # 1) 拉取 JWKS + async with httpx.AsyncClient(timeout=10.0) as http: + jwks_resp = await http.get(cfg.jwks_uri) + jwks_resp.raise_for_status() + jwks_data = jwks_resp.json() + key_set = JsonWebKey.import_key_set(jwks_data) + + # 2) 验签 + 校验 iss/aud/exp（IDToken claims_cls 默认开启这些校验） + claims_options: dict[str, dict[str, Any]] = {} + if cfg.issuer: + claims_options["iss"] = {"essential": True, "value": cfg.issuer} + claims = jwt.decode( + id_token, + key_set, + claims_cls=IDToken, + claims_options=claims_options, + ) # type: ignore[arg-type] + # aud 校验用 client_id + claims.validate(client_id=cfg.client_id) + return dict(claims) + async def _find_or_create_user( self, *, diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py index b960442..15465f5 100644 --- a/src/agentkit/server/auth/scim/router.py +++ b/src/agentkit/server/auth/scim/router.py @@ -36,7 +36,9 @@ from .models import SCIMListResponse, SCIMPatchRequest, SCIMUser logger = logging.getLogger(__name__) -router = APIRouter(prefix="/scim/v2", tags=["scim"]) +# STAND-3: prefix 由挂载点控制（app.py / test fixture 通过 include_router(prefix="/scim/v2") 注入）。 +# router 内部不设 prefix，避免双重前缀 /scim/v2/scim/v2/Users。 +router = APIRouter(tags=["scim"]) _SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" _SCIM_ERROR_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:Error" @@ -131,10 +133,8 @@ def register_scim_exception_handlers(app) -> None: @app.exception_handler(HTTPException) async def _scim_http_handler(request: Request, exc: HTTPException): - path = request.url.path - is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") - if not is_scim: - # 非 SCIM 路由 — 复制 FastAPI 默认 HTTPException handler 行为（含 headers） + # STAND-3: SCIM 挂载在 /scim/v2，非 SCIM 路由走 FastAPI 默认格式 + if not request.url.path.startswith("/scim/v2"): headers = getattr(exc, "headers", None) return JSONResponse( status_code=exc.status_code, @@ -145,10 +145,7 @@ def register_scim_exception_handlers(app) -> None: @app.exception_handler(RequestValidationError) async def _scim_validation_handler(request: Request, exc: RequestValidationError): - path = request.url.path - is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") - if not is_scim: - # 非 SCIM 路由 — 复制 FastAPI 默认 RequestValidationError handler 行为 + if not request.url.path.startswith("/scim/v2"): from fastapi.encoders import jsonable_encoder return JSONResponse( @@ -277,33 +274,82 @@ async def patch_user( if row is None: return _scim_error_response(404, "用户不存在") prev_active = bool(row["is_active"]) + # API-12: 收集未处理的 op/path，循环结束后返回 400 noTarget + _skipped_ops: list[str] = [] for op in payload.Operations: - if op.op.lower() == "replace": - if op.path in (None, "active"): - active = ( - bool(op.value) - if not isinstance(op.value, dict) - else bool(op.value.get("active", True)) - ) - await db.execute( - "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", - (1 if active else 0, now, user_id), - ) - # SEC-3: active true→false 写审计（延后到事务提交后） - if prev_active and not active: - _pending_audit = { - "target_user_id": user_id, - "target_username": str(row["username"]), - } - elif op.path == "userName": - try: + op_lower = op.op.lower() if op.op else "" + if op_lower not in ("replace", "add", "remove"): + # API-12: 仅 replace/add/remove 受支持（当前仅 replace 有副作用） + _skipped_ops.append(f"op={op.op}") + continue + if op_lower != "replace": + # add/remove 暂未实现 — 收集而非静默忽略 + _skipped_ops.append(f"op={op.op}(未实现)") + continue + + # API-14: path==None 时遍历 value 的所有 keys（原来仅更新 active 丢数据） + if op.path is None and isinstance(op.value, dict): + for field_name, field_value in op.value.items(): + if field_name == "active": + active = bool(field_value) await db.execute( - "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", - (str(op.value)[:64], now, user_id), + "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", + (1 if active else 0, now, user_id), ) - except aiosqlite.IntegrityError: - # API-7: UNIQUE 冲突返回 409 - return _scim_error_response(409, "userName 已存在") + if prev_active and not active: + _pending_audit = { + "target_user_id": user_id, + "target_username": str(row["username"]), + } + elif field_name == "userName": + try: + await db.execute( + "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", + (str(field_value)[:64], now, user_id), + ) + except aiosqlite.IntegrityError: + return _scim_error_response(409, "userName 已存在") + else: + # API-12: 不支持的字段收集 + _skipped_ops.append(f"path={field_name}") + continue + + # path 明确指定的情况 + if op.path in ("active", None): + active = ( + bool(op.value) + if not isinstance(op.value, dict) + else bool(op.value.get("active", True)) + ) + await db.execute( + "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", + (1 if active else 0, now, user_id), + ) + # SEC-3: active true→false 写审计（延后到事务提交后） + if prev_active and not active: + _pending_audit = { + "target_user_id": user_id, + "target_username": str(row["username"]), + } + elif op.path == "userName": + try: + await db.execute( + "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", + (str(op.value)[:64], now, user_id), + ) + except aiosqlite.IntegrityError: + # API-7: UNIQUE 冲突返回 409 + return _scim_error_response(409, "userName 已存在") + else: + # API-12: 不支持的 path 收集 + _skipped_ops.append(f"path={op.path}") + + # API-12: 有未处理的 op/path → 400 noTarget + if _skipped_ops: + return _scim_error_response( + 400, + f"SCIM PATCH 包含不支持的操作（scimType=noTarget）: {', '.join(_skipped_ops)}", + ) await db.commit() cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) row = await cursor.fetchone() diff --git a/src/agentkit/server/frontend/src/stores/chatStream.ts b/src/agentkit/server/frontend/src/stores/chatStream.ts index 020d95f..3909275 100644 --- a/src/agentkit/server/frontend/src/stores/chatStream.ts +++ b/src/agentkit/server/frontend/src/stores/chatStream.ts @@ -99,6 +99,22 @@ function findLastMessage( return undefined } +// MAINT-3: 提取 team_synthesis_chunk / team_synthesis 共用的 milestone 查找谓词。 +// 按 synthesis_id 精确匹配 streaming milestone（sid 为空时退化为任意 streaming milestone）。 +function findStreamingSynthesisMilestone( + conv: { messages: IChatMessage[] } | undefined, + sid: string | undefined, +): IChatMessage | undefined { + if (!conv) return undefined + return findLastMessage( + conv.messages, + (m) => + m.message_type === "milestone" && + m.status === "streaming" && + (sid ? m.synthesis_id === sid : true), + ) +} + // ── ChatStreamState: deps bag for dispatchWsEvent ────────────────────── /** @@ -878,13 +894,7 @@ export function dispatchWsEvent( ); if (!conv) break; const sid = event.data.synthesis_id; - const existing = findLastMessage( - conv.messages, - (m) => - m.message_type === "milestone" && - m.status === "streaming" && - (sid ? m.synthesis_id === sid : true), - ); + const existing = findStreamingSynthesisMilestone(conv, sid); if (existing) { state.updateMessage(conversationId, existing.id, { content: (existing.content || "") + (event.data.chunk || ""), @@ -920,13 +930,7 @@ export function dispatchWsEvent( event.data.status === 'error' || event.data.status === 'cancelled' ? 'error' : 'completed'; - const existing = findLastMessage( - conv.messages, - (m) => - m.message_type === "milestone" && - m.status === "streaming" && - (sid ? m.synthesis_id === sid : true), - ); + const existing = findStreamingSynthesisMilestone(conv, sid); if (existing) { state.updateMessage(conversationId, existing.id, { content: event.data.content || "", diff --git a/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts index e028777..f5fc78c 100644 --- a/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts +++ b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts @@ -68,30 +68,32 @@ const REQUIRED_TOKENS = { } as const describe('bitable-tokens.css — DR-3 design token contract', () => { + // MAINT-4: 提取 token 声明正则 helper — 避免转义逻辑重复 3 次。 + // 匹配 `--token-name:`（声明），而非子串存在。 + const tokenDeclPattern = (token: string) => + new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + it('defines a :root block', () => { expect(rootBlock.length).toBeGreaterThan(0) }) it.each(REQUIRED_TOKENS.color)('defines color token %s in :root', (token) => { - // Match `--token-name:` (declaration), not just substring presence. - const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) - expect(declPattern.test(rootBlock)).toBe(true) + expect(tokenDeclPattern(token).test(rootBlock)).toBe(true) }) it.each(REQUIRED_TOKENS.cf)('defines conditional-format token %s in :root', (token) => { - const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) - expect(declPattern.test(rootBlock)).toBe(true) + expect(tokenDeclPattern(token).test(rootBlock)).toBe(true) }) it.each(REQUIRED_TOKENS.drawer)('defines drawer token %s in :root', (token) => { - const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) - expect(declPattern.test(rootBlock)).toBe(true) + expect(tokenDeclPattern(token).test(rootBlock)).toBe(true) }) it('does not hardcode tokens outside :root (no component-level overrides)', () => { // ponytail: bitable tokens are exclusively :root-scoped; component-level // overrides would break the single-source-of-truth invariant. - const outsideRoot = cssContent.replace(/:root\s*\{[^}]*\}/, '') + // STAND-2: 加 g 标志移除所有 :root 块（原来仅移除首个，后续 :root 块会漏检）。 + const outsideRoot = cssContent.replace(/:root\s*\{[^}]*\}/g, '') expect(outsideRoot).not.toMatch(/--bitable-[a-z-]+\s*:/) }) }) diff --git a/tests/unit/test_scim_router.py b/tests/unit/test_scim_router.py index 6a1d6d1..76f4e65 100644 --- a/tests/unit/test_scim_router.py +++ b/tests/unit/test_scim_router.py @@ -16,7 +16,10 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from agentkit.server.auth.models import init_auth_db -from agentkit.server.auth.scim.router import router as scim_router +from agentkit.server.auth.scim.router import ( + register_scim_exception_handlers, + router as scim_router, +) SCIM_TOKEN = "test-scim-token-secret" @@ -33,7 +36,9 @@ async def scim_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FastAPI: app = FastAPI() app.state.auth_db_path = str(db_path) app.state.scim_token = SCIM_TOKEN - app.include_router(scim_router, prefix="/api/v1") + # STAND-3: 按 RFC 7644 挂载在 /scim/v2（非 /api/v1） + app.include_router(scim_router, prefix="/scim/v2") + register_scim_exception_handlers(app) return app @@ -54,7 +59,7 @@ def _auth_headers(token: str = SCIM_TOKEN) -> dict[str, str]: class TestCreateUser: def test_creates_user(self, client: TestClient) -> None: resp = client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={ "userName": "scim.user1", "active": True, @@ -71,24 +76,24 @@ class TestCreateUser: def test_duplicate_username_conflict(self, client: TestClient) -> None: client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "dup.user", "emails": [{"value": "dup@example.com"}]}, headers=_auth_headers(), ) resp = client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "dup.user", "emails": [{"value": "other@example.com"}]}, headers=_auth_headers(), ) assert resp.status_code == 409 def test_no_token_unauthorized(self, client: TestClient) -> None: - resp = client.post("/api/v1/scim/v2/Users", json={"userName": "x"}) + resp = client.post("/scim/v2/Users", json={"userName": "x"}) assert resp.status_code == 401 def test_wrong_token_unauthorized(self, client: TestClient) -> None: resp = client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "x"}, headers={"Authorization": "Bearer wrong-token"}, ) @@ -99,7 +104,7 @@ class TestPatchUser: def test_disable_user(self, client: TestClient) -> None: # 先创建 create = client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "patch.user", "active": True, "emails": [{"value": "p@example.com"}]}, headers=_auth_headers(), ) @@ -107,7 +112,7 @@ class TestPatchUser: # PATCH 禁用 resp = client.patch( - f"/api/v1/scim/v2/Users/{user_id}", + f"/scim/v2/Users/{user_id}", json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, headers=_auth_headers(), ) @@ -116,7 +121,7 @@ class TestPatchUser: def test_patch_unknown_user_404(self, client: TestClient) -> None: resp = client.patch( - "/api/v1/scim/v2/Users/nonexistent-id", + "/scim/v2/Users/nonexistent-id", json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, headers=_auth_headers(), ) @@ -126,16 +131,16 @@ class TestPatchUser: class TestListUsers: def test_list_all(self, client: TestClient) -> None: client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "list.user1", "emails": [{"value": "l1@example.com"}]}, headers=_auth_headers(), ) client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "list.user2", "emails": [{"value": "l2@example.com"}]}, headers=_auth_headers(), ) - resp = client.get("/api/v1/scim/v2/Users", headers=_auth_headers()) + resp = client.get("/scim/v2/Users", headers=_auth_headers()) assert resp.status_code == 200 data = resp.json() assert data["totalResults"] >= 2 @@ -143,12 +148,12 @@ class TestListUsers: def test_filter_by_username(self, client: TestClient) -> None: client.post( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", json={"userName": "filter.user", "emails": [{"value": "f@example.com"}]}, headers=_auth_headers(), ) resp = client.get( - "/api/v1/scim/v2/Users", + "/scim/v2/Users", params={"filter": 'userName eq "filter.user"'}, headers=_auth_headers(), ) @@ -158,5 +163,5 @@ class TestListUsers: assert data["Resources"][0]["userName"] == "filter.user" def test_no_token_unauthorized(self, client: TestClient) -> None: - resp = client.get("/api/v1/scim/v2/Users") + resp = client.get("/scim/v2/Users") assert resp.status_code == 401