diff --git a/agentkit.yaml b/agentkit.yaml index ef3f66f..e27c1a9 100644 --- a/agentkit.yaml +++ b/agentkit.yaml @@ -28,6 +28,24 @@ llm: coding: bailian-coding/qwen3-coder-plus chat: deepseek/deepseek-chat reasoning: deepseek/deepseek-reasoner + # U2/R3 — 模型 fallback 链：主模型失败时按序尝试 fallback。 + # 等保合规场景：fallback provider 在客户内网（DashScope/DeepSeek OpenAI-compatible API）， + # 不依赖 Anthropic 境外服务。fallback providers 默认 prompt_cache_enable=false + #（非 Anthropic provider 无 cache_control 透传，走自动前缀缓存）。 + fallbacks: + default: [fast] # default 失败 → fast（同 provider 内降级） + # 等保合规 fallback（注释，按需启用）：将主模型 fallback 到本地推理 provider + # default: [chat, fast] # default 失败 → deepseek-chat → qwen-turbo + # U2/R3 — PII 脱敏 hook（fail-closed）。 + # 启用后，所有发送至 LLM 的 prompt 先经过 pii_filter 脱敏： + # - 正则识别手机号/邮箱/身份证/银行卡，替换为 [REDACTED_TYPE:hash8] + # - 脱敏前后记录 hash 用于审计（原文不落盘） + # - 任何异常时 fail-closed（拒绝发送至 LLM） + # - 留 ner_hook 接入客户内网 NER 模型（LAC/HanLP），未配置时用正则版 + pii_filter: + enabled: false # 默认关闭；等保合规场景设 true + ner_hook: null # 可选：自定义 NER 函数路径（module:func） + fail_closed: true # 脱敏失败时拒绝发送（true）或降级发送（false） # G4/U1: Auxiliary model for cost-sensitive tasks (summarization). # When set, ContextCompressor tries this alias first, falling back to # the main model on failure or empty content. Commented to preserve @@ -70,19 +88,64 @@ fallback_chain: # Tests run in-memory for isolation; production / staging should use Redis. session: backend: ${AGENTKIT_SESSION_BACKEND:-memory} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} bus: backend: ${AGENTKIT_BUS_BACKEND:-memory} - redis_url: ${REDIS_URL:-redis://127.0.0.1:6379/0} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} task_store: backend: ${AGENTKIT_TASK_STORE_BACKEND:-memory} + redis_url: ${REDIS_URL:-redis://127.0.0.1:6391/0} skills: {auto_discover: true, paths: ["./configs/skills"]} experts: {paths: ["./configs/experts"]} board: {max_rounds: 5, default_template: private_board, parallel_speech: true, history_compression_threshold: 20} logging: {level: INFO, format: text} router: {classifier: heuristic, auction_enabled: false} -# OTel 可观测性 — 默认注释（OTel 为可选依赖，未安装时 telemetry/metrics.py 返回 NoOp）。 -# 启用：pip install opentelemetry-sdk opentelemetry-exporter-otlp，取消注释并指向 collector。 -# 未配置时所有指标（请求量/延迟/token 消耗）静默丢弃，形成监控盲区。 -# telemetry: -# otlp_endpoint: http://localhost:4317 # OTLP gRPC 端点 -# service_name: fischer-agentkit +# U4 SSO 集成 — 多 IDP 信任边界 (R1a) + SCIM 2.0 + 手动 deprovisioning。 +# OIDC (authlib) + SAML 2.0 (python3-saml) 并存，按 IDP sub/NameID 关联，禁止邮箱合并。 +# 单租户设计 (R1c)。证书热轮换不重启，单 IDP 故障不影响其他。 +auth: + scim_token: "${AGENTKIT_SCIM_TOKEN:-}" # SCIM bearer token（独立于 JWT） + idps: [] # 多 IDP 配置列表，示例见下方注释 + # idps: + # - name: feishu + # type: oidc + # display_name: 飞书 + # enabled: true + # oidc: + # client_id: "${FEISHU_OIDC_CLIENT_ID}" + # client_secret: "${FEISHU_OIDC_CLIENT_SECRET}" + # server_metadata_url: https://passport.feishu.cn/suite/passport/oauth/authorize + # redirect_uri: https://your-host/api/v1/auth/oauth/feishu/callback + # scope: "openid email profile" + # - name: azure_ad + # type: oidc + # display_name: Azure AD + # enabled: true + # oidc: + # client_id: "${AZURE_AD_CLIENT_ID}" + # client_secret: "${AZURE_AD_CLIENT_SECRET}" + # server_metadata_url: https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration + # redirect_uri: https://your-host/api/v1/auth/oauth/azure_ad/callback + # - name: aliyun_idaas + # type: saml + # display_name: 阿里云 IDaaS + # enabled: true + # saml: + # entity_id: https://idaas.aliyun.com/idp/xxx + # sso_url: https://idaas.aliyun.com/sso/xxx + # idp_cert: | + # -----BEGIN CERTIFICATE----- + # ...IDP 签名证书 PEM... + # -----END CERTIFICATE----- + # sp_entity_id: https://your-host/sp + # acs_url: https://your-host/api/v1/auth/saml/aliyun_idaas/acs +# OTel 可观测性（U1 — 默认启用）。 +# OTel 依赖已升级为 required（pyproject.toml [project] dependencies）， +# ImportError 静默回退已移除 — 缺包将硬失败。设 enabled=false 可显式关闭。 +# otlp_endpoint 指向 OTel collector（容器名 otel-collector / jaeger）。 +telemetry: + enabled: true + otlp_endpoint: http://localhost:4317 # OTLP gRPC 端点（开发环境；生产指向集群 collector） + service_name: fischer-agentkit + export_traces: true + export_metrics: true diff --git a/deploy/helm/agentkit/Chart.yaml b/deploy/helm/agentkit/Chart.yaml new file mode 100644 index 0000000..4dfcbe6 --- /dev/null +++ b/deploy/helm/agentkit/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: agentkit +description: Fischer AgentKit — 政企生产级 K8s Helm Chart（含 Sealed Secrets / External Secrets 双模板 + 离线安装支持） +type: application +version: 0.1.0 +appVersion: "0.1.0" +kubeVersion: ">=1.28-0" +keywords: + - agentkit + - llm + - enterprise + - k8s +maintainers: + - name: Fischer Team +icon: "" diff --git a/deploy/helm/agentkit/templates/_helpers.tpl b/deploy/helm/agentkit/templates/_helpers.tpl new file mode 100644 index 0000000..149987a --- /dev/null +++ b/deploy/helm/agentkit/templates/_helpers.tpl @@ -0,0 +1,65 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "agentkit.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified app name — release-name + chart-name（避免多 release 命名冲突）。 +*/}} +{{- define "agentkit.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart name + version label。 +*/}} +{{- define "agentkit.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels。 +*/}} +{{- define "agentkit.labels" -}} +helm.sh/chart: {{ include "agentkit.chart" . }} +{{ include "agentkit.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels。 +*/}} +{{- define "agentkit.selectorLabels" -}} +app.kubernetes.io/name: {{ include "agentkit.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name — 若未指定则用 fullname。 +*/}} +{{- define "agentkit.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "agentkit.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Secret backend 名称（Sealed Secret / External Secret 共用的目标 Secret 名）。 +*/}} +{{- define "agentkit.secretName" -}} +{{- printf "%s-secrets" (include "agentkit.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/deploy/helm/agentkit/templates/configmap.yaml b/deploy/helm/agentkit/templates/configmap.yaml new file mode 100644 index 0000000..b63a245 --- /dev/null +++ b/deploy/helm/agentkit/templates/configmap.yaml @@ -0,0 +1,44 @@ +{{- /* +非密配置 ConfigMap — 渲染 agentkit.yaml 的非密段（server/session/bus/task_store/ +skills/experts/logging）。含密钥的段（llm/auth/telemetry headers）由 env 注入， +不进 ConfigMap。 +*/ -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +data: + agentkit.yaml: | + server: + host: {{ .Values.config.server.host | quote }} + port: {{ .Values.config.server.port }} + workers: {{ .Values.config.server.workers }} + rate_limit: {{ .Values.config.server.rate_limit }} + session: + backend: {{ .Values.config.session.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + bus: + backend: {{ .Values.config.bus.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + task_store: + backend: {{ .Values.config.task_store.backend | quote }} + redis_url: {{ .Values.redis.url | quote }} + skills: + auto_discover: {{ .Values.config.skills.auto_discover }} + paths: + {{- range .Values.config.skills.paths }} + - {{ . | quote }} + {{- end }} + experts: + paths: + {{- range .Values.config.experts.paths }} + - {{ . | quote }} + {{- end }} + logging: + level: {{ .Values.config.logging.level | quote }} + format: {{ .Values.config.logging.format | quote }} + # DATABASE_URL / REDIS_PASSWORD / LLM provider keys / JWT signing key / + # SCIM token / OTel token 等 — 全部由 env 注入（来自 Secret）。 + # 见 deployment.yaml env / envFrom。 diff --git a/deploy/helm/agentkit/templates/deployment.yaml b/deploy/helm/agentkit/templates/deployment.yaml new file mode 100644 index 0000000..d61bbd8 --- /dev/null +++ b/deploy/helm/agentkit/templates/deployment.yaml @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "agentkit.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "agentkit.selectorLabels" . | nindent 8 }} + annotations: + # ConfigMap 变更触发 rollout + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ include "agentkit.serviceAccountName" . }} + {{- with .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ . }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: agentkit + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + args: + - serve + - --config + - /etc/agentkit/agentkit.yaml + - --host + - 0.0.0.0 + - --port + - {{ .Values.service.apiPort | quote }} + ports: + - name: api + containerPort: {{ .Values.service.apiPort }} + protocol: TCP + - name: gui + containerPort: {{ .Values.service.guiPort }} + protocol: TCP + env: + # --- 密钥从统一 Secret 注入（Sealed Secret / External Secret 渲染）--- + - name: JWT_SIGNING_KEY + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.jwtSigningKey.key }} + - name: API_KEY + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.apiKey.key }} + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.dbCredentials.key }} + - name: IDP_SIGNING_CERTS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.idpSigningCerts.key }} + - name: THIRD_PARTY_API_KEYS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.thirdPartyApiKeys.key }} + - name: MTLS_CLIENT_CERT + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.mtlsClientCert.key }} + - name: KMS_HSM_PIN + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.kmsHsmPin.key }} + - name: OTEL_EXPORTER_OTLP_HEADERS + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.otelExporterToken.key }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "agentkit.secretName" . }} + key: {{ .Values.secrets.redisPassword.key }} + # DEPLOY-4: 移除 POSTGRES_PASSWORD env — 应用通过 DATABASE_URL（Slot 3） + # 消费 PG 密码（DSN 内嵌），POSTGRES_PASSWORD 仅用于 PostgreSQL StatefulSet + # 初始化（由独立 PG chart / StatefulSet 引用 secrets.postgresPassword）。 + # 此处注入是死 env，误导运维以为应用读 POSTGRES_PASSWORD。 + # --- 非密配置（OTel endpoint、service name）--- + {{- if .Values.otel.enabled }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} + - name: OTEL_SERVICE_NAME + value: {{ .Values.otel.serviceName | quote }} + {{- end }} + envFrom: + # agentkit.yaml 从 ConfigMap 注入（通过 AGENTKIT_CONFIG 环境变量指向挂载路径） + - configMapRef: + name: {{ include "agentkit.fullname" . }} + volumeMounts: + - name: config + mountPath: /etc/agentkit + readOnly: true + - name: tmp + mountPath: /tmp + resources: + {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + {{- toYaml .Values.probes.liveness | nindent 12 }} + readinessProbe: + {{- toYaml .Values.probes.readiness | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ include "agentkit.fullname" . }} + - name: tmp + emptyDir: {} diff --git a/deploy/helm/agentkit/templates/external-secret.yaml b/deploy/helm/agentkit/templates/external-secret.yaml new file mode 100644 index 0000000..cddda56 --- /dev/null +++ b/deploy/helm/agentkit/templates/external-secret.yaml @@ -0,0 +1,66 @@ +{{- /* +External Secrets Operator 模板 — 当 secrets.backend=external-secrets 时渲染。 +ExternalSecret 引用运维预创建的 SecretStore（指向 Vault / AWS SM / GCP SM / Azure KV）， +从外部密钥管理服务拉取全部 11 个 secret slot（KTD-4）。 +*/ -}} +{{- if eq .Values.secrets.backend "external-secrets" }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ include "agentkit.secretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + secretStoreRef: + name: {{ .Values.secrets.externalSecret.secretStoreName | quote }} + kind: SecretStore + target: + name: {{ include "agentkit.secretName" . }} + creationPolicy: Owner + data: + # 1) JWT 签名密钥 + - secretKey: {{ .Values.secrets.jwtSigningKey.key }} + remoteRef: + key: agentkit/jwt-signing-key + # 2) API Key + - secretKey: {{ .Values.secrets.apiKey.key }} + remoteRef: + key: agentkit/api-key + # 3) DATABASE_URL + - secretKey: {{ .Values.secrets.dbCredentials.key }} + remoteRef: + key: agentkit/database-url + # 4) IDP 签名证书 + - secretKey: {{ .Values.secrets.idpSigningCerts.key }} + remoteRef: + key: agentkit/idp-signing-certs + # 5) 第三方 API Key + - secretKey: {{ .Values.secrets.thirdPartyApiKeys.key }} + remoteRef: + key: agentkit/third-party-api-keys + # 6) TLS 服务器证书 + - secretKey: {{ .Values.secrets.tlsServerCert.key }} + remoteRef: + key: agentkit/tls-server-cert + # 7) mTLS 客户端证书 + - secretKey: {{ .Values.secrets.mtlsClientCert.key }} + remoteRef: + key: agentkit/mtls-client-cert + # 8) KMS/HSM PKCS#11 PIN + - secretKey: {{ .Values.secrets.kmsHsmPin.key }} + remoteRef: + key: agentkit/kms-hsm-pin + # 9) OTel exporter token + - secretKey: {{ .Values.secrets.otelExporterToken.key }} + remoteRef: + key: agentkit/otel-exporter-token + # 10) Redis 密码 + - secretKey: {{ .Values.secrets.redisPassword.key }} + remoteRef: + key: agentkit/redis-password + # 11) PostgreSQL 密码 + - secretKey: {{ .Values.secrets.postgresPassword.key }} + remoteRef: + key: agentkit/postgres-password +{{- end }} diff --git a/deploy/helm/agentkit/templates/ingress.yaml b/deploy/helm/agentkit/templates/ingress.yaml new file mode 100644 index 0000000..82fa697 --- /dev/null +++ b/deploy/helm/agentkit/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className | quote }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host | quote }} + secretName: {{ .Values.ingress.tls.secretName | quote }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + # GUI 走根路径，API 走 /api 前缀 + - path: /api + pathType: Prefix + backend: + service: + name: {{ include "agentkit.fullname" . }} + port: + name: api + - path: / + pathType: Prefix + backend: + service: + name: {{ include "agentkit.fullname" . }} + port: + name: gui +{{- end }} diff --git a/deploy/helm/agentkit/templates/sealed-secret.yaml b/deploy/helm/agentkit/templates/sealed-secret.yaml new file mode 100644 index 0000000..18616d1 --- /dev/null +++ b/deploy/helm/agentkit/templates/sealed-secret.yaml @@ -0,0 +1,43 @@ +{{- /* +Sealed Secrets 模板（Bitnami Sealed Secrets controller）— 当 secrets.backend=sealed-secrets 时渲染。 +覆盖全部 11 个 secret slot（KTD-4）。encryptedData 由运维通过 kubeseal 生成后以 --set 或 +override values 文件注入。禁止明文 Kubernetes Secret + Git 提交。 +*/ -}} +{{- if eq .Values.secrets.backend "sealed-secrets" }} +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: {{ include "agentkit.secretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + encryptedData: + # 1) JWT 签名密钥 + {{ .Values.secrets.jwtSigningKey.key }}: {{ .Values.secrets.jwtSigningKey.encryptedData | default "" | quote }} + # 2) API Key + {{ .Values.secrets.apiKey.key }}: {{ .Values.secrets.apiKey.encryptedData | default "" | quote }} + # 3) DATABASE_URL + {{ .Values.secrets.dbCredentials.key }}: {{ .Values.secrets.dbCredentials.encryptedData | default "" | quote }} + # 4) IDP 签名证书 + {{ .Values.secrets.idpSigningCerts.key }}: {{ .Values.secrets.idpSigningCerts.encryptedData | default "" | quote }} + # 5) 第三方 API Key + {{ .Values.secrets.thirdPartyApiKeys.key }}: {{ .Values.secrets.thirdPartyApiKeys.encryptedData | default "" | quote }} + # 6) TLS 服务器证书（tls.crt + tls.key — ponytail: 单独 Secret 由 cert-manager 或运维创建，此处仅登记） + {{ .Values.secrets.tlsServerCert.key }}: {{ .Values.secrets.tlsServerCert.encryptedData | default "" | quote }} + # 7) mTLS 客户端证书 + {{ .Values.secrets.mtlsClientCert.key }}: {{ .Values.secrets.mtlsClientCert.encryptedData | default "" | quote }} + # 8) KMS/HSM PKCS#11 PIN + {{ .Values.secrets.kmsHsmPin.key }}: {{ .Values.secrets.kmsHsmPin.encryptedData | default "" | quote }} + # 9) OTel exporter token + {{ .Values.secrets.otelExporterToken.key }}: {{ .Values.secrets.otelExporterToken.encryptedData | default "" | quote }} + # 10) Redis 密码 + {{ .Values.secrets.redisPassword.key }}: {{ .Values.secrets.redisPassword.encryptedData | default "" | quote }} + # 11) PostgreSQL 密码 + {{ .Values.secrets.postgresPassword.key }}: {{ .Values.secrets.postgresPassword.encryptedData | default "" | quote }} + template: + metadata: + name: {{ include "agentkit.secretName" . }} + labels: + {{- include "agentkit.labels" . | nindent 8 }} +{{- end }} diff --git a/deploy/helm/agentkit/templates/service.yaml b/deploy/helm/agentkit/templates/service.yaml new file mode 100644 index 0000000..7cb8110 --- /dev/null +++ b/deploy/helm/agentkit/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentkit.fullname" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: api + port: {{ .Values.service.apiPort }} + targetPort: api + protocol: TCP + - name: gui + port: {{ .Values.service.guiPort }} + targetPort: gui + protocol: TCP + selector: + {{- include "agentkit.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/agentkit/templates/serviceaccount.yaml b/deploy/helm/agentkit/templates/serviceaccount.yaml new file mode 100644 index 0000000..ac96f10 --- /dev/null +++ b/deploy/helm/agentkit/templates/serviceaccount.yaml @@ -0,0 +1,36 @@ +{{- /* +ServiceAccount — 当 serviceAccount.create=true 时创建。 +workloadIdentity.enabled=true 时： + 1) automountServiceAccountToken=false（禁用默认 token 投递） + 2) 额外创建 type=service-account-token 的 Secret（projected token），供 KMS/HSM 使用 +*/ -}} +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "agentkit.serviceAccountName" . }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.workloadIdentity.enabled }} +automountServiceAccountToken: false +{{- end }} +{{- end }} +{{- if and .Values.serviceAccount.create .Values.workloadIdentity.enabled }} +--- +# ponytail: projected SA token — 限时令牌，供 KMS/HSM workload identity 换取访问凭据。 +# 升级路径：云厂商原生 workload identity 注解（P1，避免手动 token 投递）。 +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "agentkit.serviceAccountName" . }}-token + namespace: {{ .Release.Namespace }} + labels: + {{- include "agentkit.labels" . | nindent 4 }} + annotations: + kubernetes.io/service-account.name: {{ include "agentkit.serviceAccountName" . }} +type: kubernetes.io/service-account-token +{{- end }} diff --git a/deploy/helm/agentkit/values.yaml b/deploy/helm/agentkit/values.yaml new file mode 100644 index 0000000..4caed7a --- /dev/null +++ b/deploy/helm/agentkit/values.yaml @@ -0,0 +1,243 @@ +# Fischer AgentKit — Helm values +#政企生产级配置；所有密钥通过 Sealed Secrets 或 External Secrets Operator 注入， +# 禁止明文 Kubernetes Secret + Git 提交（KTD-4 / R2a）。 + +# --------------------------------------------------------------------------- +# 镜像配置 +# --------------------------------------------------------------------------- +image: + repository: fischer-agentkit + tag: 0.1.0 + pullPolicy: IfNotPresent + # 私有镜像仓库时配置 pullSecret（secret 引用名，由外部创建） + pullSecret: "" + +# --------------------------------------------------------------------------- +# 副本与调度 +# --------------------------------------------------------------------------- +replicaCount: 1 +# ponytail: 暂不支持 HPA，单副本起步；扩容由运维手动 rollout。 +# 升级路径： values.replicaCount > 1 + PodDisruptionBudget（P1） + +# --------------------------------------------------------------------------- +# ServiceAccount +# --------------------------------------------------------------------------- +serviceAccount: + create: true + name: "" # 留空则用 fullname + annotations: {} # 云厂商 workload identity 注解（如 EKS/GKE） + +# --------------------------------------------------------------------------- +# Service +# --------------------------------------------------------------------------- +service: + type: ClusterIP + apiPort: 8001 # API 服务（agentkit serve） + guiPort: 8002 # Web GUI（agentkit gui） + +# --------------------------------------------------------------------------- +# Ingress（TLS 终止） +# --------------------------------------------------------------------------- +ingress: + enabled: true + className: nginx + annotations: {} + host: agentkit.example.com + tls: + enabled: true + # 引用 secrets.tlsServerCert 对应的 TLS Secret + secretName: agentkit-tls + +# --------------------------------------------------------------------------- +# 资源 +# --------------------------------------------------------------------------- +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1000m" + memory: 2Gi + +# --------------------------------------------------------------------------- +# Pod 安全上下文（非 root 用户，与 Dockerfile appuser 对齐） +# --------------------------------------------------------------------------- +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +securityContext: + readOnlyRootFilesystem: false # agentkit 需写临时文件（缓存/日志） + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + +# --------------------------------------------------------------------------- +# 探针 +# --------------------------------------------------------------------------- +probes: + liveness: + httpGet: + path: /api/v1/health + port: api + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + readiness: + httpGet: + path: /api/v1/health + port: api + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# --------------------------------------------------------------------------- +# OTel 可观测性 +# --------------------------------------------------------------------------- +otel: + enabled: true + endpoint: http://otel-collector:4317 # OTLP gRPC + serviceName: fischer-agentkit + # exporter token 从 secrets.otelExporterToken 引用（OTel collector 启用鉴权时） + headers: {} + +# --------------------------------------------------------------------------- +# Redis / PostgreSQL 连接（密码从 secrets 引用） +# --------------------------------------------------------------------------- +redis: + url: redis://redis:6379/0 + # 实际密码通过 env REDIS_PASSWORD 注入（来自 secrets.redisPassword） + +postgres: + # DATABASE_URL（不含密码，密码通过 ${POSTGRES_PASSWORD} 占位由 env 注入） + url: postgresql+asyncpg://agentkit@postgres:5432/agentkit + +# --------------------------------------------------------------------------- +# 认证 / IDP（引用 secrets） +# --------------------------------------------------------------------------- +auth: + scimToken: "" # 从 secrets.apiKey 引用（SCIM bearer） + idps: [] # IDP 配置列表（非密部分），密钥引用 secrets.idpSigningCerts + +# --------------------------------------------------------------------------- +# 密钥清单 — 11 个 secret slot（KTD-4 / R2a） +# --------------------------------------------------------------------------- +# backend 选择： +# sealed-secrets — 使用 Bitnami Sealed Secrets（模板 templates/sealed-secret.yaml） +# external-secrets — 使用 External Secrets Operator（模板 templates/external-secret.yaml） +# 客户按环境二选一；模板按 backend 条件渲染。禁止明文 Kubernetes Secret + Git 提交。 +secrets: + backend: sealed-secrets + + # External Secrets Operator 配置（仅当 backend: external-secrets 时生效） + externalSecret: + # SecretStore 名称（运维预创建，指向 Vault / AWS SM / GCP SM / Azure KV） + secretStoreName: agentkit-vault-store + # 后端类型：vault | aws | gcp | azure + backend: vault + + # --- 11 个 secret slot --- + # 每个 slot：name=Kubernetes Secret key；description=用途；rotation=轮换周期 + + # 1) JWT 签名密钥（HS256）— access(15m) + refresh(7d) token 签名 + jwtSigningKey: + key: jwt-signing-key + description: "JWT access/refresh token HS256 签名密钥" + rotation: 90d + + # 2) API Key — 服务间 / SCIM / 客户端鉴权（恒定时间比较） + apiKey: + key: api-key + description: "服务间调用 / SCIM bearer / agentkit pair 生成的 API Key" + rotation: 180d + + # 3) DB 凭据 — 应用层 DATABASE_URL（含用户名，密码占位由 env 注入） + dbCredentials: + key: database-url + description: "应用层 DATABASE_URL DSN（postgresql+asyncpg://agentkit:***@postgres:5432/agentkit）" + rotation: 90d + + # 4) IDP 签名证书 — OIDC/SAML IdP 签名证书 PEM（多 IDP 信任边界） + idpSigningCerts: + key: idp-signing-certs + description: "OIDC/SAML IdP 签名证书 PEM（JSON map: {idp_name: pem}，支持多 IDP 热轮换）" + rotation: 365d + + # 5) 第三方 API Key（LLM providers）— DashScope / DeepSeek / OpenAI / Anthropic 等 + thirdPartyApiKeys: + key: third-party-api-keys + description: "LLM provider API Key（JSON map: {provider: key}，DashScope/DeepSeek/OpenAI/Anthropic）" + rotation: 90d + + # 6) TLS 服务器证书私钥 — Ingress TLS 终止用的 cert + key + tlsServerCert: + key: tls-server-cert + description: "Ingress TLS 服务器证书 + 私钥（PEM，tls.crt + tls.key）" + rotation: 365d + + # 7) mTLS 客户端证书私钥 — 与下游服务（KMS/HSM、内部 API）双向 TLS + mtlsClientCert: + key: mtls-client-cert + description: "mTLS 客户端证书 + 私钥（PEM，client.crt + client.key + ca.crt）" + rotation: 180d + + # 8) KMS/HSM PKCS#11 PIN — 硬件安全模块访问 PIN + kmsHsmPin: + key: kms-hsm-pin + description: "KMS/HSM PKCS#11 访问 PIN（用于密钥托管 / 签名 / 解密）" + rotation: 365d + + # 9) OTel exporter token — OTLP collector 鉴权 token + otelExporterToken: + key: otel-exporter-token + description: "OTLP exporter 鉴权 token（DEPLOY-5: 值格式必须为 'Authorization=Bearer '，符合 OTEL_EXPORTER_OTLP_HEADERS key=value 规范）" + rotation: 90d + + # 10) Redis 密码 — Redis requirepass（基础设施层） + redisPassword: + key: redis-password + description: "Redis requirepass（基础设施层，env REDIS_PASSWORD 注入应用）" + rotation: 90d + + # 11) PostgreSQL 密码 — POSTGRES_PASSWORD（基础设施层，PG StatefulSet 初始化用） + postgresPassword: + key: postgres-password + description: "PostgreSQL POSTGRES_PASSWORD（DEPLOY-4: 仅 PG StatefulSet 初始化消费，应用通过 DATABASE_URL Slot 3 内嵌密码连接，不读此 env）" + rotation: 90d + +# --------------------------------------------------------------------------- +# workload identity — KMS/HSM projected service account token +# --------------------------------------------------------------------------- +workloadIdentity: + enabled: false # 启用后 serviceaccount.yaml 注入 projected SA token + audience: sts.example.com # 受众（云厂商 STS / 内部 KMS） + expirationSeconds: 3600 + +# --------------------------------------------------------------------------- +# agentkit.yaml 非密配置（注入 ConfigMap） +# --------------------------------------------------------------------------- +config: + server: + host: 0.0.0.0 + port: 8001 + workers: 1 + rate_limit: 60 + session: + backend: redis + bus: + backend: redis + task_store: + backend: redis + skills: + auto_discover: true + paths: ["./configs/skills"] + experts: + paths: ["./configs/experts"] + logging: + level: INFO + format: text + # 注意：llm / auth / telemetry 等含密钥的段由 env 注入，不进 ConfigMap diff --git a/deploy/offline/Makefile b/deploy/offline/Makefile new file mode 100644 index 0000000..67bb8fc --- /dev/null +++ b/deploy/offline/Makefile @@ -0,0 +1,37 @@ +# Fischer AgentKit — 离线安装包构建编排 +# 目标：在联网环境打包镜像 + Chart，拷贝到离线集群安装。 +# +# 用法： +# make all # 构建镜像 + 打包 chart（联网环境） +# make install # 离线集群安装（需先拷贝 dist/ 过去） +# make clean # 清理 dist/ + +SHELL := /bin/bash +.DEFAULT_GOAL := all + +# 可覆盖变量 +IMAGE_REPO ?= fischer-agentkit +IMAGE_TAG ?= 0.1.0 +CHART_DIR ?= ../helm/agentkit +DIST_DIR ?= dist +NAMESPACE ?= agentkit +RELEASE ?= agentkit + +.PHONY: all build-images package-chart install clean + +all: build-images package-chart + +# 构建镜像并保存为 tarball +build-images: + @bash scripts/build-images.sh $(IMAGE_REPO) $(IMAGE_TAG) $(DIST_DIR) + +# 打包 Helm chart 为 tgz +package-chart: + @bash scripts/package-chart.sh $(CHART_DIR) $(DIST_DIR) + +# 离线安装：加载镜像 + helm install +install: + @bash scripts/install.sh $(DIST_DIR) $(NAMESPACE) $(RELEASE) + +clean: + rm -rf $(DIST_DIR) diff --git a/deploy/offline/scripts/build-images.sh b/deploy/offline/scripts/build-images.sh new file mode 100755 index 0000000..62776c9 --- /dev/null +++ b/deploy/offline/scripts/build-images.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# build-images.sh — 构建 AgentKit Docker 镜像并保存为 tarball（离线安装用） +# 用法：bash build-images.sh [IMAGE_REPO] [IMAGE_TAG] [DIST_DIR] +set -euo pipefail + +IMAGE_REPO="${1:-fischer-agentkit}" +IMAGE_TAG="${2:-0.1.0}" +DIST_DIR="${3:-dist}" +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + +mkdir -p "$DIST_DIR" +cd "$REPO_ROOT" + +IMAGE_REF="${IMAGE_REPO}:${IMAGE_TAG}" +echo "==> 构建 Docker 镜像：${IMAGE_REF}" +docker build -t "${IMAGE_REF}" -f Dockerfile . + +TARBALL="${DIST_DIR}/agentkit-image-${IMAGE_TAG}.tar.gz" +echo "==> 保存镜像到 tarball：${TARBALL}" +# ponytail: gzip 压缩镜像，离线传输体积更小；docker save 默认未压缩。 +docker save "${IMAGE_REF}" | gzip > "${TARBALL}" + +echo "==> 完成。产物：${TARBALL}" +echo " 大小：$(du -h "${TARBALL}" | cut -f1)" diff --git a/deploy/offline/scripts/install.sh b/deploy/offline/scripts/install.sh new file mode 100755 index 0000000..6b8793b --- /dev/null +++ b/deploy/offline/scripts/install.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# install.sh — 离线安装 AgentKit（加载镜像 + helm install） +# 用法：bash install.sh [DIST_DIR] [NAMESPACE] [RELEASE_NAME] +# 前置：已将 dist/ 拷贝到离线集群节点，节点已安装 docker + helm + kubectl。 +set -euo pipefail + +DIST_DIR="${1:-dist}" +NAMESPACE="${2:-agentkit}" +RELEASE="${3:-agentkit}" +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +cd "${SCRIPT_DIR}" + +echo "==> 检查依赖" +for cmd in docker helm kubectl; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: ${cmd} 未安装" >&2 + exit 1 + fi +done + +if [ ! -d "${DIST_DIR}" ]; then + echo "ERROR: 产物目录不存在：${DIST_DIR}" >&2 + echo " 请先在联网环境执行 make all，并将 dist/ 拷贝到本节点。" >&2 + exit 1 +fi + +# 1) 加载镜像 tarball +IMAGE_TARBALL="$(ls "${DIST_DIR}"/agentkit-image-*.tar.gz 2>/dev/null | head -1 || true)" +if [ -z "${IMAGE_TARBALL}" ]; then + echo "ERROR: 镜像 tarball 未找到（${DIST_DIR}/agentkit-image-*.tar.gz）" >&2 + exit 1 +fi +echo "==> 加载镜像：${IMAGE_TARBALL}" +gunzip -c "${IMAGE_TARBALL}" | docker load +# 列出刚加载的镜像（便于校验 tag） +LOADED_IMAGE="$(docker images --format '{{.Repository}}:{{.Tag}}' | grep agentkit | head -1)" +echo " 已加载：${LOADED_IMAGE}" + +# 2) helm install +CHART_TGZ="$(ls "${DIST_DIR}"/agentkit-*.tgz 2>/dev/null | grep -v agentkit-image | head -1 || true)" +if [ -z "${CHART_TGZ}" ]; then + echo "ERROR: chart tgz 未找到（${DIST_DIR}/agentkit-*.tgz）" >&2 + exit 1 +fi +echo "==> 创建 namespace：${NAMESPACE}" +kubectl get namespace "${NAMESPACE}" >/dev/null 2>&1 || \ + kubectl create namespace "${NAMESPACE}" + +echo "==> helm install：${RELEASE} (namespace=${NAMESPACE})" +helm upgrade --install "${RELEASE}" "${CHART_TGZ}" \ + --namespace "${NAMESPACE}" \ + --set image.repository="$(echo "${LOADED_IMAGE}" | cut -d: -f1)" \ + --set image.tag="$(echo "${LOADED_IMAGE}" | cut -d: -f2)" \ + --set image.pullPolicy=Never # 离线本地镜像，Never 强制使用已加载的本地镜像 + +echo "==> 等待 rollout" +# DEPLOY-1: 用标签选择器定位 Deployment — fullname 在 release 名包含 chart 名时 +# 仅返回 release 名（如 release=agentkit → Deployment=agentkit），其他情况返回 +# release-chart（如 release=myprod → myprod-agentkit）。硬编码 "${RELEASE}-agentkit" +# 在默认 release=agentkit 时拼出不存在的 agentkit-agentkit。改用 instance 标签。 +kubectl rollout status deployment -l app.kubernetes.io/instance="${RELEASE}" -n "${NAMESPACE}" --timeout=300s || \ + echo "WARN: rollout 未在 300s 内完成，请检查 pod 状态：kubectl get pods -n ${NAMESPACE}" + +echo "==> 完成。验证：" +echo " kubectl get pods -n ${NAMESPACE}" +echo " kubectl get ingress -n ${NAMESPACE}" +echo " helm status ${RELEASE} -n ${NAMESPACE}" diff --git a/deploy/offline/scripts/package-chart.sh b/deploy/offline/scripts/package-chart.sh new file mode 100755 index 0000000..839f86d --- /dev/null +++ b/deploy/offline/scripts/package-chart.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# package-chart.sh — 打包 Helm chart 为 tgz（离线安装用） +# 用法：bash package-chart.sh [CHART_DIR] [DIST_DIR] +set -euo pipefail + +CHART_DIR="${1:-../helm/agentkit}" +DIST_DIR="${2:-dist}" +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +# 解析 CHART_DIR 相对路径（相对于 offline/ 目录） +case "$CHART_DIR" in + /*) ABS_CHART_DIR="$CHART_DIR" ;; + *) ABS_CHART_DIR="${SCRIPT_DIR}/$CHART_DIR" ;; +esac + +if [ ! -f "${ABS_CHART_DIR}/Chart.yaml" ]; then + echo "ERROR: Chart.yaml 未找到：${ABS_CHART_DIR}" >&2 + exit 1 +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "ERROR: helm 未安装。请先安装 Helm 3.x（https://helm.sh/docs/intro/install/）" >&2 + exit 1 +fi + +mkdir -p "${SCRIPT_DIR}/${DIST_DIR}" + +echo "==> 打包 Helm chart：${ABS_CHART_DIR}" +helm dependency build "${ABS_CHART_DIR}" 2>/dev/null || true # ponytail: 无依赖时静默跳过 +helm package "${ABS_CHART_DIR}" -d "${SCRIPT_DIR}/${DIST_DIR}" + +echo "==> 完成。产物目录：${SCRIPT_DIR}/${DIST_DIR}" +ls -lh "${SCRIPT_DIR}/${DIST_DIR}"/agentkit-*.tgz 2>/dev/null || true diff --git a/docs/compliance/dengbao-level3/00-overview.md b/docs/compliance/dengbao-level3/00-overview.md new file mode 100644 index 0000000..484430a --- /dev/null +++ b/docs/compliance/dengbao-level3/00-overview.md @@ -0,0 +1,143 @@ +# 等保三级认证文档 — 总览 + +## 1. 认证目标与范围 + +### 1.1 认证目标 + +本套文档用于 Fischer AgentKit（以下简称 "AgentKit"）通过 GB/T 22239-2019 +《信息安全技术 网络安全等级保护基本要求》**第三级**（等保三级）测评准备。 + +目标：在政企 POC 采购关闭前完成认证 readiness，使 AgentKit 作为应用系统 +具备部署到等保三级信息系统的合规条件。 + +### 1.2 认证范围 + +AgentKit 作为一个**应用系统**纳入等保三级测评，覆盖： + +- 应用软件本身（FastAPI 后端 + Vue 前端 + Tauri 桌面客户端） +- 部署形态：Kubernetes（客户内网）+ Redis + PostgreSQL（含 pgvector） +- 数据处理：用户输入、LLM 调用、记忆存储、终端命令执行 + +**不在范围内**（由客户/平台侧负责）： + +- 物理机房环境（客户机房） +- K8s 宿主机操作系统内核加固 +- 网络骨干设施（防火墙、入侵检测系统、堡垒机） +- PKI 证书签发机构（CA） +- HSM/KMS 硬件设备 + +AgentKit 文档中明确标注「依赖客户侧」与「AgentKit 实现」的边界，避免越界承诺。 + +### 1.3 文档结构 + +本套文档按 GB/T 22239-2019 三级要求 6 个维度组织： + +| 文件 | 维度 | 主要内容 | +|------|------|----------| +| `00-overview.md` | — | 总览（本文档） | +| `01-physical.md` | 物理安全 | 部署假设（客户机房）、物理访问、环境监控、电力 | +| `02-network.md` | 网络安全 | 网络架构、边界防护（Ingress TLS）、访问控制（NetworkPolicy）、入侵防范、安全审计（OTel） | +| `03-host.md` | 主机安全 | 容器镜像安全、K8s node 基线、身份鉴别（SA token）、访问控制（RBAC）、安全审计、入侵防范 | +| `04-application.md` | 应用安全 | 身份鉴别（SSO 多 IDP）、访问控制（RBAC 3 级 + 权限位）、安全审计（OTel + audit.py）、通信完整性（TLS + JWT）、软件容错、资源控制 | +| `05-data.md` | 数据安全 | 数据完整性（PG + pgvector）、数据保密性（PII 脱敏 + 国密 P1）、备份恢复、剩余信息保护（session revoke） | +| `06-management.md` | 管理安全 | 密钥轮换 runbook、break-glass 告警、变更管理、应急响应、安全评估 | +| `control-points.yaml` | — | 控制点清单 + AgentKit 实现映射（已实现 / P0 / P1 / 待评估） | + +## 2. AgentKit 架构摘要 + +### 2.1 技术栈 + +- **后端**：Python 3.11+、FastAPI、Uvicorn、Pydantic v2、SQLAlchemy 2（async） +- **前端**：Vue 3、TypeScript、Vite 5、Ant Design Vue 4、Pinia +- **桌面端**：Tauri 2.x（Rust 外壳 + Python sidecar） +- **基础设施**：Redis（总线/缓存/状态）、PostgreSQL + pgvector（情景记忆） +- **可观测性**：OpenTelemetry（OTLP gRPC 导出，U1 已强制依赖） +- **部署**：Helm Chart（K8s 内网部署，U5） + +### 2.2 部署形态 + +``` +客户内网 K8s 集群 + ├── Ingress (nginx, TLS 终止) ← 唯一对外入口 + │ └── TLS 证书（cert-manager 或手动） + ├── AgentKit Pod (非 root, UID 1001) + │ ├── FastAPI serve (:8001) — API + │ └── FastAPI gui (:8002) — Web GUI + ├── Redis (内部, requirepass) + ├── PostgreSQL + pgvector (内部, 密码) + └── OTel Collector (内部, bearer token) + ↑ OTLP gRPC (mTLS: 待评估) +``` + +密钥通过 Sealed Secrets 或 External Secrets Operator 注入（禁止明文 +Kubernetes Secret + Git 提交，KTD-4 / R2a）。共 10 个 secret slot， +每个含轮换周期（90~365 天）。 + +### 2.3 关键安全子系统映射 + +| 子系统 | 模块 | 等保维度 | +|--------|------|----------| +| OTel 可观测性 | `src/agentkit/telemetry/` | 网络/主机/应用审计 | +| PII 脱敏 | `src/agentkit/llm/pii_filter.py` | 数据保密性 | +| SSO 多 IDP | `src/agentkit/server/auth/providers/`、`idp_registry.py` | 应用身份鉴别 | +| SCIM 2.0 | `src/agentkit/server/auth/scim/router.py` | 应用账户生命周期 | +| RBAC 审计 | `src/agentkit/server/auth/audit.py` | 应用安全审计 | +| 终端安全 6 层 | `src/agentkit/server/auth/terminal_security.py` | 应用访问控制 | +| RBAC 权限 | `src/agentkit/server/auth/permissions.py` | 应用访问控制 | +| JWT 会话 | `src/agentkit/server/auth/jwt_utils.py`、`denylist.py` | 应用身份鉴别 + 剩余信息保护 | +| 密码哈希 | `src/agentkit/server/auth/password.py` | 应用身份鉴别 | +| Helm 部署 | `deploy/helm/agentkit/` | 网络/主机/管理 | +| 密钥轮换 | `docs/deployment/key-rotation-runbook.md` | 管理安全 | + +## 3. 各维度实现状态概览 + +下表汇总各维度控制点的实现状态。状态定义： + +- **已实现**：代码/配置已落地，有对应文件可查 +- **P0**：认证 readiness 必需，本次 POC 采购关闭前需补齐 +- **P1**：下一阶段补齐，文档中标注为参考 +- **待评估**：依赖客户侧或需进一步评估方案 + +| 维度 | 控制点数 | 已实现 | P0 | P1 | 待评估 | +|------|----------|--------|----|----|--------| +| 物理安全 | 6 | 0 | 0 | 0 | 6 | +| 网络安全 | 8 | 4 | 2 | 1 | 1 | +| 主机安全 | 7 | 4 | 2 | 0 | 1 | +| 应用安全 | 8 | 8 | 0 | 0 | 0 | +| 数据安全 | 8 | 6 | 1 | 1 | 0 | +| 管理安全 | 7 | 4 | 1 | 1 | 1 | +| **合计** | **44** | **26** | **6** | **3** | **9** | + +> 应用安全是 AgentKit 自身责任范围，已全部实现。物理安全完全依赖客户机房， +> 故全部为「待评估」。网络/主机/数据/管理维度的 P0 项（共 6 个）是认证 +> readiness 的关键补齐项，详见 `control-points.yaml`。 +> +> **P0 关键补齐项清单**： +> - NET-03 NetworkPolicy 模板 +> - NET-07 审计日志保留 180 天（KTD-9） +> - HST-03 容器日志标准化采集 +> - HST-07 镜像漏洞扫描（Trivy/CI 集成） +> - DAT-03 PostgreSQL 备份恢复 runbook +> - MGT-02 CI/CD 规范文档化 + +## 4. 与既有交付的引用关系 + +本套文档不重复实现细节，只映射到既有交付物： + +| 既有交付 | 等保映射 | 引用文档 | +|----------|----------|----------| +| U1 OTel（telemetry/） | 应用/网络审计 | `02-network.md` §5、`04-application.md` §4 | +| U2 PII 脱敏（pii_filter.py） | 数据保密性 | `05-data.md` §2 | +| U3 DR-fixes（bitable） | 应用容错 | `04-application.md` §5 | +| U4 SSO/SCIM/Audit | 应用身份鉴别/审计 | `04-application.md` §1/§4、`06-management.md` §2 | +| U5 K8s Helm Chart | 网络/主机/管理 | `02-network.md`、`03-host.md`、`06-management.md` | +| R7a 终端安全 6 层 | 应用访问控制 | `04-application.md` §2 | + +## 5. 文档使用说明 + +1. 测评机构按 `control-points.yaml` 逐条核对，每条含 `id` / `dimension` / + `title` / `requirement` / `implementation` / `status` / `references`。 +2. `references` 字段给出 AgentKit 代码文件绝对路径，使用 `file:///` 链接格式。 +3. 状态为 P0 的控制点需在认证前补齐，对应文件中标有「P0 补齐项」段落。 +4. 状态为 P1 的控制点在文档中标注为「P1 参考」，不阻碍三级认证但需写入整改计划。 +5. 状态为「待评估」的控制点依赖客户侧环境，需在客户机房测评时单独核对。 diff --git a/docs/compliance/dengbao-level3/01-physical.md b/docs/compliance/dengbao-level3/01-physical.md new file mode 100644 index 0000000..0611843 --- /dev/null +++ b/docs/compliance/dengbao-level3/01-physical.md @@ -0,0 +1,99 @@ +# 01 — 物理安全 + +> GB/T 22239-2019 第三级「物理和环境安全」控制点映射。 + +## 1. 维度说明 + +AgentKit 作为应用系统以容器形态部署在客户内网 Kubernetes 集群中， +**不拥有也不运维物理机房**。物理安全责任由客户机房运营方承担。 + +本维度文档的目的是： + +1. 明确 AgentKit 部署对物理环境的**假设**（前置条件）。 +2. 列出 AgentKit 自身在物理安全维度的**实现空缺**（全部为「待评估」）。 +3. 在客户机房测评时，由客户方提供物理安全控制点证据， + AgentKit 侧仅提供「部署形态证明」（Helm Chart / Pod 清单）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| PHY-01 | 物理访问控制：机房出入口配置电子门禁，记录进入人员和时间 | +| PHY-02 | 防盗窃和防破坏：主要设备设置防盗标识，机房设置监控系统 | +| PHY-03 | 防雷击：机房设置防雷保安器，接地电阻符合要求 | +| PHY-04 | 防火：机房设置火灾自动消防系统，检测报警器 | +| PHY-05 | 防水和防潮：机房安装水敏检测装置，防止水管安装 | +| PHY-06 | 防静电：机房采用防静电地板，设置静电消除器 | + +## 3. AgentKit 实现映射 + +### 3.1 部署假设（前置条件） + +AgentKit 容器化部署对物理环境的假设： + +1. 客户机房满足 GB/T 22239-2019 三级物理安全全部要求。 +2. K8s 宿主机位于受控机房内，物理访问由客户运营方管理。 +3. 电力供应由机房提供（含 UPS / 后备发电机）。 +4. 网络骨干（防火墙、IDS、堡垒机）由客户网络团队运维。 + +### 3.2 控制点状态 + +全部 6 个物理安全控制点状态均为 **待评估** —— 依赖客户机房测评时由客户方 +提供证据。AgentKit 侧仅提供以下证明材料： + +- 部署形态证明：[deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + （`podSecurityContext.runAsNonRoot: true`，`runAsUser: 1000`） +- Pod 资源清单：[deploy/helm/agentkit/templates/deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) +- 容器镜像基线：[Dockerfile](file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile) + （多阶段构建 + `USER appuser` 非 root 运行） + +### 3.3 控制点详情 + +#### PHY-01 物理访问控制 — 待评估 + +- **要求**：机房出入口配置电子门禁，记录进入人员和时间。 +- **AgentKit 实现**：无物理访问控制责任。AgentKit 软件资产以镜像形式 + 存储在客户镜像仓库，物理访问由客户机房运营方控制。 +- **客户侧需提供**：机房门禁系统记录、人员进出日志、访问审批流程。 + +#### PHY-02 防盗窃和防破坏 — 待评估 + +- **要求**：主要设备设置防盗标识，机房设置监控系统。 +- **AgentKit 实现**：无物理设备责任。K8s worker node 由客户运维。 +- **客户侧需提供**：机房监控系统、设备资产清单。 + +#### PHY-03 防雷击 — 待评估 + +- **要求**：机房设置防雷保安器，接地电阻符合要求。 +- **AgentKit 实现**：无防雷责任。 +- **客户侧需提供**：防雷检测报告、接地电阻测试记录。 + +#### PHY-04 防火 — 待评估 + +- **要求**：机房设置火灾自动消防系统，检测报警器。 +- **AgentKit 实现**：无消防责任。 +- **客户侧需提供**：消防系统验收报告、定期检测记录。 + +#### PHY-05 防水和防潮 — 待评估 + +- **要求**：机房安装水敏检测装置，防止水管安装。 +- **AgentKit 实现**：无防水责任。 +- **客户侧需提供**：水敏检测装置部署记录、机房湿度监控。 + +#### PHY-06 防静电 — 待评估 + +- **要求**：机房采用防静电地板，设置静电消除器。 +- **AgentKit 实现**：无防静电责任。 +- **客户侧需提供**：防静电地板验收报告、静电消除器检测记录。 + +## 4. 测评建议 + +物理安全维度在 AgentKit 侧**无代码/配置可核对**，全部依赖客户机房测评。 +建议在测评前与客户安全团队确认： + +1. 客户机房是否已通过等保三级（或更高）测评。 +2. 客户机房测评报告是否覆盖 AgentKit 部署区域。 +3. 若客户机房未单独测评，需将 AgentKit 部署区域纳入本次测评范围。 + +AgentKit 侧提供的证明材料仅证明「软件部署形态合规」（非 root 容器、 +资源受限、Helm Chart 标准化部署），不替代物理环境测评。 diff --git a/docs/compliance/dengbao-level3/02-network.md b/docs/compliance/dengbao-level3/02-network.md new file mode 100644 index 0000000..18cdc6d --- /dev/null +++ b/docs/compliance/dengbao-level3/02-network.md @@ -0,0 +1,197 @@ +# 02 — 网络安全 + +> GB/T 22239-2019 第三级「网络和通信安全」控制点映射。 + +## 1. 维度说明 + +AgentKit 部署在客户内网 K8s 集群，网络安全的边界由 Ingress 控制， +内部东西向流量由 K8s 网络模型与（待补齐的）NetworkPolicy 控制。 +AgentKit 自身实现： + +- **已实现**：Ingress TLS 终止、OTel 安全审计（日志/trace）、 + Redis/PG 密码鉴权（基础设施层）、API 限流。 +- **P0 补齐项**：NetworkPolicy 模板（东西向流量隔离）、 + 审计日志保留策略（180 天，KTD-9）。 +- **P1 参考**：OTel exporter mTLS、网络入侵检测接入。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| NET-01 | 网络架构：划分网络区域，避免单点故障 | +| NET-02 | 边界防护：边界实施访问控制和安全过滤 | +| NET-03 | 访问控制：网络层访问控制策略，最小权限 | +| NET-04 | 入侵防范：网络入侵检测，恶意流量检测 | +| NET-05 | 安全审计：网络审计日志，覆盖关键网络活动 | +| NET-06 | 恶意代码防范：网络边界恶意代码检测 | +| NET-07 | 安全审计日志保留：日志保留期满足合规要求 | +| NET-08 | 资源控制：会话与带宽限制，防止资源耗尽 | + +## 3. AgentKit 实现映射 + +### 3.1 网络架构（NET-01）— 已实现 + +AgentKit K8s 部署采用单 Ingress 入口 + 内部 Service 架构： + +``` +外网/办公网 ── HTTPS ──> Ingress (nginx, TLS 终止) + │ + ▼ + AgentKit Service (ClusterIP) + │ + ▼ + AgentKit Pod (:8001 API, :8002 GUI) + │ + ┌─────────────┼─────────────┐ + ▼ ▼ ▼ + Redis PostgreSQL OTel Collector + (ClusterIP) (ClusterIP) (ClusterIP) +``` + +- **Ingress 是唯一对外入口**：[deploy/helm/agentkit/templates/ingress.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml) +- **Service 类型为 ClusterIP**（不直接对外暴露）： + [deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + （`service.type: ClusterIP`） +- **API 与 GUI 走分离路径**：`/api` → API Service，`/` → GUI Service + +### 3.2 边界防护（NET-02）— 已实现 + +Ingress 启用 TLS 终止，所有外网流量强制 HTTPS： + +```yaml +ingress: + enabled: true + className: nginx + tls: + enabled: true + secretName: agentkit-tls # 引用 secrets.tlsServerCert +``` + +- TLS 证书通过 cert-manager 自动续期或手动签发（365 天轮换）。 +- HTTP→HTTPS 跳转由 nginx ingress controller 默认配置提供 + （客户侧 ingress controller 责任）。 +- 证书私钥通过 Sealed Secret / External Secret 注入（KTD-4 / R2a）。 + +**参考文件**： +[deploy/helm/agentkit/templates/ingress.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml) +[deploy/helm/agentkit/values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml)（`ingress` 段、`secrets.tlsServerCert` slot） + +### 3.3 访问控制 / NetworkPolicy（NET-03）— P0 补齐项 + +**当前状态：P0**。Helm Chart 暂无 NetworkPolicy 模板， +Pod 间东西向流量默认全通（K8s 默认网络模型）。 + +**P0 补齐计划**： + +1. 新增 `deploy/helm/agentkit/templates/networkpolicy.yaml`。 +2. 默认 deny-all，按白名单放行： + - Ingress → AgentKit Pod（8001/8002） + - AgentKit Pod → Redis（6379） + - AgentKit Pod → PostgreSQL（5432） + - AgentKit Pod → OTel Collector（4317） +3. 命名空间隔离：AgentKit 独立 namespace，跨 namespace 流量显式声明。 + +**当前缓解**：Service 全部 ClusterIP（不对外），Redis/PG 由客户 K8s 集群 +内部网络策略保护（客户侧责任）。 + +### 3.4 入侵防范（NET-04）— P1 参考 + +**当前状态：P1**。AgentKit 自身不部署网络 IDS， +依赖客户侧网络骨干的 IDS / IPS。 + +- API 层面：[RateLimitMiddleware](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 实现单进程滑动窗口限流，Webhook 入站端点单独限流 + （[src/agentkit/server/routes/channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- 异常登录：JWT 验证失败、SCIM token 校验失败返回 401 + （[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) + 使用 `hmac.compare_digest` 恒定时间比较）。 + +**P1 补齐计划**：接入客户 SIEM，将 AgentKit OTel 日志投递到客户 +入侵检测平台，由 SIEM 做 abnormal pattern 检测。 + +### 3.5 安全审计（NET-05）— 已实现 + +AgentKit 通过 OpenTelemetry（U1，强制依赖）实现网络活动审计： + +- **HTTP 请求 trace**：FastAPI Instrumentor 自动埋点，每个请求生成 span， + 含 HTTP method/path/status/duration。 + [src/agentkit/telemetry/setup.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py) + （`FastAPIInstrumentor`） +- **审计事件 span**：RBAC 角色变更、deprovisioning 写入 OTel span event + （[src/agentkit/server/auth/audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 +- **SCIM 推送失败告警**：span event `scim.push.failure` + （[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py)）。 +- **终端命令审计**：每条终端命令写入 `terminal_audit_logs` 表 + （[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 + +OTLP gRPC 导出到内部 collector +（[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`otel.endpoint: http://otel-collector:4317`）。 + +### 3.6 恶意代码防范（NET-06）— 待评估 + +**当前状态：待评估**。网络边界恶意代码检测由客户网络骨干的 +防病毒网关 / 沙箱负责。AgentKit 不在网络边界位置。 + +AgentKit 侧的缓解： + +- 容器镜像来自可信构建（CI/CD），不在运行时下载可执行文件。 +- 终端命令执行受 6 层白名单 + 危险检测约束（R7a）， + 限制任意命令执行能力（详见 `04-application.md` §2）。 + +### 3.7 安全审计日志保留（NET-07）— P0 补齐项 + +**当前状态：P0**。审计日志保留策略未在代码中强制实施： + +- `auth_audit_log` 表（SQLite）和 `terminal_audit_logs` 表无自动清理 / + 归档策略，依赖运维手动管理。 +- OTel collector 侧的日志保留由客户可观测性平台配置，AgentKit 不控制。 + +**P0 补齐计划**： + +1. 在 Helm values 增加 `audit.retentionDays: 180` 配置（KTD-9）。 +2. 实现审计日志归档 CronJob：超过 180 天的记录导出到对象存储（客户侧） + 并从主表清理。 +3. OTel collector 配置 180 天保留策略（与客户可观测性团队协同）。 + +### 3.8 资源控制（NET-08）— 已实现 + +AgentKit 在应用层与容器层均实现资源控制： + +- **容器层**：Pod resources requests/limits + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `resources` 段，CPU 250m~1000m / Memory 512Mi~2Gi）。 +- **应用层**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 实现 IP 维度滑动窗口限流，Webhook 入站端点独立限流 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- **会话超时**：终端 session 闲置 1800s 自动断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py) + `SESSION_IDLE_TIMEOUT = 1800`）。 +- **JWT 短时效**：access token 15min + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + `ACCESS_TOKEN_TTL`）。 + +## 4. OTel exporter mTLS（P1 参考） + +**当前状态：P1**。OTel exporter 通过 bearer token 鉴权 +（[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`secrets.otelExporterToken` slot），未启用 mTLS。 + +Helm Chart 已预留 `secrets.mtlsClientCert` slot（用于下游服务 mTLS）， +但 OTel exporter 侧的 mTLS 配置待补齐。 + +**P1 补齐计划**： + +1. 在 OTel SDK 初始化时配置 `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE` / + `OTEL_EXPORTER_OTLP_CLIENT_KEY` / `OTEL_EXPORTER_OTLP_CA_CERT` 环境变量。 +2. OTel collector 侧启用 mTLS 服务端证书验证。 +3. 复用 `secrets.mtlsClientCert` slot 注入客户端证书。 + +## 5. 测评建议 + +1. **NET-01 / NET-02 / NET-05 / NET-08**：直接展示 Helm Ingress + OTel + + 限流配置即可。 +2. **NET-03 / NET-07**：P0 补齐项，需在测评前完成 NetworkPolicy 模板和 + 审计日志保留 CronJob。 +3. **NET-04 / NET-06**：依赖客户侧网络骨干，由客户网络团队提供 IDS / + 恶意代码检测证据。 diff --git a/docs/compliance/dengbao-level3/03-host.md b/docs/compliance/dengbao-level3/03-host.md new file mode 100644 index 0000000..9fec7ea --- /dev/null +++ b/docs/compliance/dengbao-level3/03-host.md @@ -0,0 +1,197 @@ +# 03 — 主机安全 + +> GB/T 22239-2019 第三级「设备和计算安全」控制点映射。 +> 本维度以**容器与 Pod**为「主机」单元（AgentKit 不直接运维 K8s node）。 + +## 1. 维度说明 + +AgentKit 以容器形态运行在客户 K8s 集群。「主机安全」分为两层： + +- **容器/Pod 层**（AgentKit 责任）：镜像安全、运行时安全上下文、 + ServiceAccount、Pod 级 RBAC。 +- **K8s node 层**（客户责任）：宿主机操作系统内核加固、node 基线、 + 主机入侵检测（HIDS）。 + +AgentKit 自身实现： + +- **已实现**：非 root 容器、最小权限安全上下文（capabilities drop ALL）、 + ServiceAccount 隔离、容器级资源限制。 +- **P0 补齐项**：容器镜像漏洞扫描（CI 集成 Trivy/Grype）、 + Pod 安全审计日志采集标准化。 +- **P1 参考**：distroless 基础镜像。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| HST-01 | 身份鉴别：设备/系统身份标识与鉴别 | +| HST-02 | 访问控制：操作系统/账户权限最小化，远程管理受控 | +| HST-03 | 安全审计：设备审计日志，覆盖系统管理活动 | +| HST-04 | 入侵防范：主机入侵检测，系统加固 | +| HST-05 | 恶意代码防范：主机防病毒，恶意代码检测 | +| HST-06 | 资源控制：CPU/内存/磁盘限制，会话超时 | +| HST-07 | 镜像与补丁管理：系统补丁及时更新，镜像来源可信 | + +## 3. AgentKit 实现映射 + +### 3.1 容器镜像安全（HST-07）— 部分已实现 / P0 补齐 + +#### 已实现 + +- **多阶段构建**：builder + runner 两阶段，最终镜像不含构建工具链。 + [Dockerfile](file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile) +- **非 root 用户**：runner 阶段创建 `appuser`（UID 1001 / GID 1001）， + `USER appuser` 强制非 root 运行。 +- **基础镜像**：`python:3.11-slim`（精简版，比 full 镜像减少攻击面）。 +- **HEALTHCHECK**：内置健康检查 + （`/api/v1/health`，30s 间隔）。 +- **不写字节码**：`PYTHONDONTWRITEBYTECODE=1`，减少磁盘 .pyc 残留。 + +#### P0 补齐项：镜像漏洞扫描 + +**当前状态：P0**。CI 流水线未集成容器镜像漏洞扫描。 + +**P0 补齐计划**： + +1. CI 流水线增加 Trivy 扫描步骤： + ```yaml + - name: Trivy scan + run: trivy image --severity HIGH,CRITICAL --exit-code 1 fischer-agentkit:${{ github.sha }} + ``` +2. Helm Chart 增加 `image.digest` 字段，部署时使用不可变 digest 而非 tag。 +3. 镜像签名（cosign）— P1，可选。 + +### 3.2 K8s node 安全基线（HST-04 / HST-05）— 待评估 + +**当前状态：待评估**。K8s 宿主机由客户运维，AgentKit 不控制 node 操作系统。 + +客户侧需提供： + +- node 操作系统加固基线（CIS Benchmark for Kubernetes / 操作系统）。 +- 主机入侵检测（HIDS，如 Falco / OSSEC）。 +- 主机防病毒（如客户机房要求）。 + +AgentKit 侧的缓解： + +- Pod `securityContext` 强制非 root + drop ALL capabilities + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `podSecurityContext` / `securityContext` 段），即使 node 被攻破， + 容器逃逸到 root 的难度提升。 + +### 3.3 身份鉴别 / ServiceAccount（HST-01）— 已实现 + +AgentKit Pod 使用专用 ServiceAccount，不共享 default SA： + +- **ServiceAccount 创建**： + [deploy/helm/agentkit/templates/serviceaccount.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/serviceaccount.yaml) +- **values 配置**： + ```yaml + serviceAccount: + create: true + name: "" # 留空则用 fullname（专用 SA） + annotations: {} # 云厂商 workload identity 注解 + ``` +- **Workload Identity**（可选）：`workloadIdentity.enabled: true` 时 + 注入 projected SA token，用于访问云 KMS / STS + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `workloadIdentity` 段）。 + +Pod 内应用层身份鉴别见 `04-application.md` §1（SSO + JWT）。 + +### 3.4 访问控制 / Pod 级 RBAC（HST-02）— 已实现 + +- **capabilities drop ALL**： + ```yaml + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + ``` + ([values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) `securityContext` 段) +- **runAsNonRoot 强制**：`podSecurityContext.runAsNonRoot: true`， + K8s admission 拒绝非 root Pod。 +- **readOnlyRootFilesystem**：当前为 `false`（agentkit 需写临时文件用于缓存/日志）， + 通过 `emptyDir` 挂载 `/tmp` 实现写入隔离 + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `volumes.tmp`）。 +- **privileged: false**（默认，未启用特权模式）。 + +应用层 RBAC（member/operator/admin + 9 权限位）见 +`04-application.md` §2。 + +### 3.5 安全审计 / 容器日志（HST-03）— 已实现 + P0 补齐 + +#### 已实现 + +- **stdout/stderr 标准输出**：AgentKit 日志全部输出到 stdout， + 由 K8s 日志采集（如 Loki / Fluent Bit）收集。 +- **OTel trace**：每个 Pod 的请求 trace 通过 OTLP 导出到 collector + （U1，强制依赖）。 +- **审计事件**：RBAC 变更、deprovisioning、SCIM 操作写入 `auth_audit_log` 表 + + OTel span event（[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 +- **终端命令审计**：每条终端命令 + 决策（executed/confirmed/blocked/denied） + 写入 `terminal_audit_logs` 表 + （[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 + +#### P0 补齐项：审计日志标准化采集 + +**当前状态：P0**。容器日志采集到客户日志平台的链路未标准化。 + +**P0 补齐计划**： + +1. Helm Chart 增加 `logging.fluentBitSidecar` 选项（可选 sidecar 采集）。 +2. 审计日志 JSON 化（structured logging），便于 SIEM 解析。 +3. 与 `02-network.md` §3.7 的 180 天保留策略协同落地。 + +### 3.6 入侵防范（HST-04）— 已实现（容器层） + +容器层入侵防范已实现： + +- **非 root + drop ALL capabilities**：限制容器内可执行的系统调用。 +- **allowPrivilegeEscalation: false**：阻止子进程提权。 +- **资源 limits**：CPU 1000m / Memory 2Gi 上限，防止容器失控耗尽 node 资源 + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `resources.limits`）。 +- **Liveness/Readiness 探针**：故障 Pod 自动重启 + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `livenessProbe` / `readinessProbe`）。 + +node 层入侵检测（HIDS）见 §3.2，依赖客户侧。 + +### 3.7 资源控制（HST-06）— 已实现 + +- **Pod resources**： + ```yaml + resources: + requests: { cpu: 250m, memory: 512Mi } + limits: { cpu: 1000m, memory: 2Gi } + ``` +- **应用层限流**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + IP 维度滑动窗口。 +- **会话超时**：终端 session 1800s 闲置断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py)）。 +- **JWT 短时效**：access 15min / refresh 7d + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py)）。 + +## 4. distroless 基础镜像（P1 参考） + +**当前状态：P1**。当前基础镜像为 `python:3.11-slim`，含 shell + 包管理器， +攻击面大于 distroless。 + +**P1 升级路径**： + +1. 切换到 `gcr.io/distroless/python3-debian12` 作为 runner 基础镜像。 +2. 需调整：distroless 无 shell，HEALTHCHECK 的 `python -c` 命令需改为 + HTTP 探针（K8s livenessProbe httpGet，已支持）。 +3. 多阶段构建的 builder 阶段保持 `python:3.11-slim`（构建工具链需要）。 + +distroless 不阻碍等保三级认证，但作为安全增强项写入整改计划。 + +## 5. 测评建议 + +1. **HST-01 / HST-02 / HST-06**：直接展示 ServiceAccount + securityContext + + resources 配置即可。 +2. **HST-03**：已实现部分展示 audit.py + terminal_security.py， + P0 补齐日志标准化采集。 +3. **HST-07**：P0 补齐镜像漏洞扫描后展示 Trivy 扫描报告。 +4. **HST-04 / HST-05**：容器层已实现，node 层由客户提供 HIDS 证据。 diff --git a/docs/compliance/dengbao-level3/04-application.md b/docs/compliance/dengbao-level3/04-application.md new file mode 100644 index 0000000..2b05041 --- /dev/null +++ b/docs/compliance/dengbao-level3/04-application.md @@ -0,0 +1,257 @@ +# 04 — 应用安全 + +> GB/T 22239-2019 第三级「应用和数据安全 — 应用部分」控制点映射。 + +## 1. 维度说明 + +应用安全是 AgentKit **自身核心责任范围**，全部 8 个控制点已实现。 +本维度文档引用 U4 SSO 多 IDP / SCIM、U1 OTel 审计、R7a 终端安全 6 层、 +RBAC 3 级权限、JWT 会话等既有实现，不重复代码细节。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| APP-01 | 身份鉴别：用户身份标识与鉴别，口令复杂度，登录失败处理 | +| APP-02 | 访问控制：基于角色的访问控制，最小权限，默认拒绝 | +| APP-03 | 安全审计：应用审计日志，覆盖用户操作关键事件 | +| APP-04 | 通信完整性：传输完整性保护，防篡改 | +| APP-05 | 软件容错：错误处理，故障隔离，降级机制 | +| APP-06 | 资源控制：会话并发控制，超时自动断开 | +| APP-07 | 应用账户生命周期：开户/变更/销户审计，SCIM 自动化 | +| APP-08 | 代码安全：输入验证，输出编码，依赖漏洞管理 | + +## 3. AgentKit 实现映射 + +### 3.1 身份鉴别（APP-01）— 已实现 + +#### 多 IDP 单点登录（U4 SSO） + +AgentKit 支持 OIDC + SAML 双协议并存的多 IDP 信任边界： + +- **OIDC 适配器**（authlib redirect flow + JIT 用户创建）： + [src/agentkit/server/auth/providers/oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +- **SAML 2.0 适配器**（python3-saml / OneLogin ACS）： + [src/agentkit/server/auth/providers/saml.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/saml.py) +- **多 IDP 注册表**（热证书轮换 + 单 IDP 故障隔离，R1a 按 sub 关联 + 禁止邮箱合并）： + [src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + +支持的 IDP 类型：飞书 / Azure AD / 阿里云 IDaaS 等任意 OIDC/SAML 合规 IDP。 + +#### 本地密码认证 + +- **bcrypt 哈希**：rounds=12（OWASP 推荐值） + [src/agentkit/server/auth/password.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py) +- **JWT 会话**：HS256 签名，access 15min + refresh 7d（30d 记住我） + [src/agentkit/server/auth/jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + - V2 claims 含 `sid`（session id）+ `jti`（per-token unique id）， + 支持服务端 session 撤销。 +- **Refresh token 轮换 + reuse 检测**：旧 token 进入 denylist 30s 窗口， + reuse 触发撤销该用户全部 session + [src/agentkit/server/auth/denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py) +- **登录失败处理**：JWT 验证失败返回 401，refresh token reuse 触发 + 全 session 撤销（强制重新登录）。 +- **OAuth state CSRF 防护**：OIDC state 缓存 TTL 5min，一次性消费 + （[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) `_StateCache`）。 + +### 3.2 访问控制 / RBAC 3 级（APP-02）— 已实现 + +#### 3 级 RBAC + 9 权限位 + +[src/agentkit/server/auth/permissions.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/permissions.py) + +| 角色 | 权限 | +|------|------| +| `member` | CHAT / KB_QUERY / WORKFLOW_EXECUTE | +| `operator` | member 全部 + KB_WRITE + TERMINAL_LOCAL_USE + TERMINAL_WHITELIST_MANAGE | +| `admin` | operator 全部 + TERMINAL_SERVER_USE + USER_MANAGE + SYSTEM_CONFIG | + +权限位作为稳定字符串标识符用于 JWT payload + 前端 store + 审计日志， +不重命名既有值（向前兼容）。 + +#### 终端安全 6 层白名单（R7a） + +[src/agentkit/server/auth/terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py) + +评估优先级（最高→最低）： + +1. **Blocklist**（admin 管理，永远拒绝） +2. **Built-in safe whitelist**（`_SAFE_COMMAND_PREFIXES`） +3. **Global whitelist**（admin 管理，DB 持久化） +4. **User whitelist**（per-user，DB 持久化 + 内存缓存） +5. **Session whitelist**（per-session，仅内存） +6. **Danger detection**（`_is_dangerous`，需确认） + +**关键安全特性**：shell 操作符（`&&`、`;`、`|`、`$()`、backticks、`>`、`<`、 +换行）**永远绕过所有 whitelist 层**，强制走 danger detection — +即使 `git push` 在白名单中，`git push && rm -rf /` 仍需确认。 + +#### 终端双层授权 + +终端端点要求 RBAC 角色 + 个人授权标志双重校验： + +- `is_terminal_authorized` — 本地终端（client sidecar） +- `is_server_terminal_authorized` — 服务端终端（server PTY） + +允许 admin 按用户授权终端而不改变其角色。 + +#### SCIM token 恒定时间比较 + +SCIM bearer token 使用 `hmac.compare_digest` 恒定时间比较防时序攻击： +[src/agentkit/server/auth/scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +（`_verify_scim_token`）。 + +### 3.3 安全审计（APP-03）— 已实现 + +#### OTel 审计通道（U1 强制依赖） + +[src/agentkit/server/auth/audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) + +审计事件类型： + +- `role_change` — RBAC 角色变更（actor/target/role_before/role_after） +- `deprovision` — 手动 deprovisioning（含 `break_glass` 标记） + +每条记录含 actor/target/reason/source/timestamp，写入 `auth_audit_log` 表 ++ 发 OTel span event（OTel 不可用时降级为 SQLite + 日志）。 + +审计来源（`source` 字段）： + +- `manual` — 手动操作 +- `scim` — SCIM 自动化 +- `cron_reconciliation` — 定时对账 +- `break_glass` — 应急通道（高权限账户） + +#### SCIM 推送失败告警 + +SCIM push 失败实时告警：日志 error + OTel span event `scim.push.failure` +（[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +`_alert_scim_failure`）。 + +#### 终端命令审计 + +每条终端命令 + 决策结果（executed/confirmed/rejected/blocked/denied） ++ cwd + exit_code 写入 `terminal_audit_logs` 表 +（[terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py) +`write_audit_log`）。 + +#### OTel 指标 + +U1 OTel 暴露以下审计相关指标 +（[src/agentkit/telemetry/metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)）： + +- `agent.request.total` / `agent.execution.duration` +- `gen_ai.usage.tokens` / `tool.call.duration` +- `prompt_cache.hit` / `prompt_cache.miss` +- `pii_filter.coverage` / `pii_filter.redacted`（U2 PII 脱敏覆盖率） + +### 3.4 通信完整性（APP-04）— 已实现 + +- **传输层 TLS**：Ingress TLS 终止（详见 `02-network.md` §3.2）， + 客户端→Ingress 全程 HTTPS。 +- **JWT 签名完整性**：HS256 签名，任何 payload 字段篡改导致签名校验失败 + （[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py) + `verify_token`）。 +- **mTLS 客户端证书**（下游服务）：Helm Chart 预留 `secrets.mtlsClientCert` + slot，用于与 KMS/HSM、内部 API 双向 TLS（详见 `06-management.md`）。 +- **Webhook 签名校验**：入站 Webhook fail-closed，签名校验失败返回 401 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 + +### 3.5 软件容错（APP-05）— 已实现 + +- **LLM 网关 fallback**：多 provider fallback 链 + （[src/agentkit/llm/gateway.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/gateway.py)）。 +- **OTel 运行时容错**：OTel exporter 连接失败时降级为 NoOpTracer， + 应用启动不崩溃（[tracer.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/tracer.py) + `init_telemetry`）。 +- **PII 脱敏 fail-closed**：脱敏异常时拒绝发送至 LLM，不降级到原文 + （[pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py) + `fail_closed=True` 默认）。 +- **专家团队失败回退**：所有阶段失败时回退到单 agent 模式 + （详见 AGENTS.md「专家团队模式」）。 +- **DR-fixes（U3）**：bitable 字段校验（DR-1）、LastViewDeletionError（DR-4）、 + 工具调用容错（DR-5）。 + - [src/agentkit/bitable/repository.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/repository.py) + - [src/agentkit/bitable/service.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/service.py) + - [src/agentkit/tools/bitable_tool.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/tools/bitable_tool.py) + +### 3.6 资源控制（APP-06）— 已实现 + +- **IP 限流**：[RateLimiter](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py) + 滑动窗口（`max_requests` / `window_seconds` 可配置）。 +- **Webhook 独立限流**：[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py) + 入站端点单独限流 + nonce dedup。 +- **会话超时**： + - 终端 session 闲置 1800s 自动断开 + （[terminal_server.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py)）。 + - JWT access token 15min 过期强制重新认证。 +- **专家名称正则校验**：`_EXPERT_NAME_RE = re.compile(r"^[a-zA-Z0-9_-]{1,64}$")` + 防止注入（项目规则）。 +- **HandoffTransport 有界队列**：`maxsize=1024`，关闭使用 sentinel 模式 + 防止无界内存增长（项目规则）。 + +### 3.7 应用账户生命周期 / SCIM（APP-07）— 已实现 + +#### SCIM 2.0 自动化 + +[src/agentkit/server/auth/scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) + +端点： + +- `POST /scim/v2/Users` — 创建用户（随机密码，不可本地登录） +- `PATCH /scim/v2/Users/{id}` — 禁用（active=false）/ 更新字段 +- `GET /scim/v2/Users` — 列表 / 过滤（`userName eq` / `active eq`） + +Bearer token 认证（独立于 JWT，恒定时间比较）。 + +#### Deprovisioning（账户销户） + +管理员通过 `/admin/users/{id}/deprovision` 强制销户： + +1. 禁用本地用户账户（`is_active = 0`） +2. 撤销该用户所有活跃 session +3. 记录审计（actor/target/reason/source/timestamp）+ OTel +4. `break_glass=True` 时记录 `source=break_glass`（应急通道标记） + +详见 `06-management.md` §2 break-glass 告警流程。 + +#### JIT 用户创建（R1a） + +OIDC/SAML 首次登录 JIT 创建本地用户 + `user_idp_links` 行， +按 `(idp_name, sub)` 严格关联，**禁止邮箱合并**已存在账户 +（[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +`_find_or_create_user`）。 + +### 3.8 代码安全（APP-08）— 已实现 + +- **输入验证**：所有 API 入口使用 Pydantic v2 模型校验 + （`model_config = ConfigDict(extra="forbid")` 拒绝额外字段）。 + - SCIM 模型：[src/agentkit/server/auth/scim/models.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/models.py) + - IDP 配置：[src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + （`OIDCConfig` / `SAMLConfig` 均 `extra="forbid"`） +- **禁止 `any` 类型**：项目规则强制使用 Pydantic 模型或 `Unknown` + （AGENTS.md）。 +- **专家名称正则校验**：`_EXPERT_NAME_RE` 防 SQL/命令注入。 +- **API Key 恒定时间比较**：`hmac.compare_digest`（项目规则）。 +- **Ruff lint + format**：`ruff check src/ && ruff format src/`（目标 py311） + 作为代码质量基线。 +- **依赖管理**：`pyproject.toml` 锁定精确版本，`pip install -e ".[dev]"` + 可复现环境。 +- **终端命令安全**：6 层白名单 + shell 操作符检测 + （详见 §3.2），防止命令注入链。 + +## 4. 测评建议 + +应用安全维度全部控制点已实现，测评时直接展示对应代码文件即可： + +1. **APP-01**：展示 oidc.py / saml.py / jwt_utils.py / password.py。 +2. **APP-02**：展示 permissions.py / terminal_security.py + RBAC 矩阵表。 +3. **APP-03**：展示 audit.py + 终端审计日志 + OTel 指标清单。 +4. **APP-04**：展示 ingress.yaml TLS + jwt_utils.py 签名校验 + channels.py + Webhook 签名。 +5. **APP-05**：展示 gateway.py fallback + pii_filter.py fail-closed + + DR-fixes 引用。 +6. **APP-06**：展示 middleware.py 限流 + terminal_server.py 超时。 +7. **APP-07**：展示 scim/router.py + deprovision 端点 + JIT 创建逻辑。 +8. **APP-08**：展示 Pydantic 模型校验 + Ruff 配置 + 终端命令安全。 diff --git a/docs/compliance/dengbao-level3/05-data.md b/docs/compliance/dengbao-level3/05-data.md new file mode 100644 index 0000000..ab2c3c5 --- /dev/null +++ b/docs/compliance/dengbao-level3/05-data.md @@ -0,0 +1,217 @@ +# 05 — 数据安全 + +> GB/T 22239-2019 第三级「应用和数据安全 — 数据部分」控制点映射。 + +## 1. 维度说明 + +AgentKit 处理的数据类型： + +- **用户输入**：聊天消息、终端命令、工作流配置 +- **LLM 调用**：prompt + completion（含潜在 PII） +- **持久化数据**：用户账户、会话、记忆（情景/语义/工作） +- **审计数据**：RBAC 变更、终端命令、SCIM 操作 + +AgentKit 自身实现： + +- **已实现**：PII 脱敏（U2 fail-closed + hash 审计）、 + session 撤销（剩余信息保护）、PG + pgvector 完整性、 + 审计数据 hash 化。 +- **P0 补齐项**：PostgreSQL 备份恢复 runbook。 +- **P1 参考**：国密加密（R11a，SM4/SM2 替代 AES/RSA）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| DAT-01 | 数据完整性：数据传输/存储完整性，校验机制 | +| DAT-02 | 数据保密性：敏感数据加密存储/传输，密钥管理 | +| DAT-03 | 数据备份与恢复：重要数据定期备份，恢复演练 | +| DAT-04 | 剩余信息保护：鉴别信息/文件/内存空间释放前清理 | +| DAT-05 | 个人信息保护：PII 采集/处理/传输最小化与脱敏 | +| DAT-06 | 数据采集与分类分级：按敏感度分级保护 | +| DAT-07 | 数据销毁：数据销毁审计，不可恢复 | +| DAT-08 | 国密算法合规（R11a，政企客户要求）：敏感数据加密使用国密算法 | + +## 3. AgentKit 实现映射 + +### 3.1 数据完整性（DAT-01）— 已实现 + +- **PostgreSQL + pgvector**：主数据库使用 PostgreSQL（事务 ACID 保证）， + pgvector 扩展存储向量记忆数据。 + - 连接串：[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `postgres.url: postgresql+asyncpg://agentkit@postgres:5432/agentkit` + - SQLAlchemy 2 async ORM 提供应用层事务管理。 +- **JWT 签名完整性**：HS256 签名防篡改（详见 `04-application.md` §3.4）。 +- **Webhook 签名校验**：fail-closed 防篡改 + （[channels.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py)）。 +- **Pydantic 模型校验**：所有 API 入口 `extra="forbid"` 拒绝未知字段， + 防止数据注入。 + +### 3.2 数据保密性 / PII 脱敏（DAT-02 / DAT-05）— 已实现（U2） + +#### PII 脱敏 hook + +[src/agentkit/llm/pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py) + +设计要点： + +- **正则版（默认）**：识别手机号 / 邮箱 / 身份证 / 银行卡， + 替换为 `[REDACTED_TYPE:hash8]`。 +- **ner_hook（可选）**：客户内网 NER 模型（LAC/HanLP）， + `module:func` 路径动态 import，优先于正则版。 +- **fail-closed**：任何异常时拒绝发送至 LLM（`raise PIIFilterError`）， + **不降级到原文发送**。 +- **hash 审计**：脱敏前后记录 sha256 前 8 位（仅 hash，不含原文）， + 用于审计追溯。 + +#### PII 类型与正则 + +| PII 类型 | 正则模式 | 替换格式 | +|----------|----------|----------| +| 身份证 | 18 位（含校验位 X） | `[REDACTED_ID_CARD:hash8]` | +| 手机号 | `1[3-9]\d{9}` | `[REDACTED_PHONE:hash8]` | +| 邮箱 | 标准邮箱正则 | `[REDACTED_EMAIL:hash8]` | +| 银行卡 | 16-19 位数字 | `[REDACTED_BANK_CARD:hash8]` | + +> 身份证正则优先于手机号正则匹配，避免身份证号内 11 位数字子串被 +> 手机号正则误匹配。 + +#### OTel 脱敏指标 + +- `pii_filter.coverage` — 脱敏扫描覆盖率（status: success/failed_closed/fallback_regex） +- `pii_filter.redacted` — 脱敏实体计数 + +（[metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)） + +#### fail-closed 策略 + +```python +try: + result = _redact(text, ner_hook=ner_hook) + ... +except Exception as e: + if fail_closed: + raise PIIFilterError(f"PII filter failed (fail-closed): {e}") from e + # fail-open（非默认）：降级到正则版重试 +``` + +生产环境默认 `fail_closed=True`，宁可拒绝服务也不泄露 PII。 + +### 3.3 国密加密（R11a）— P1 参考 + +**当前状态：P1**。AgentKit 当前使用： + +- **传输加密**：TLS（Ingress 终止），算法套件由 nginx ingress controller + 默认配置（含 AES-GCM 等国际算法）。 +- **密码哈希**：bcrypt（国际算法）。 +- **JWT 签名**：HS256（HMAC-SHA256，国际算法）。 +- **PII hash**：SHA-256（国际算法）。 + +**未使用国密算法**（SM2/SM3/SM4/SM9）。 + +**P1 补齐计划**（R11a）： + +1. **JWT 签名**：替换 HS256 → SM2-HMAC 或国密 JWT 库（如 `gmssl`）。 +2. **密码哈希**：评估 SM3 替代 bcrypt（需兼顾兼容性，可能双算法并行）。 +3. **PII hash**：SHA-256 → SM3。 +4. **TLS**：启用国密 TLS 套件（需 nginx ingress controller + 国密证书）。 +5. **数据加密**：PG 字段级加密（透明数据加密 TDE 或应用层 SM4）。 + +**注**：国密算法**不阻碍等保三级认证**（三级要求「加密」而非「国密」）， +但政企客户常要求国密合规，故列入 P1 整改计划。 + +### 3.4 数据备份与恢复（DAT-03）— P0 补齐项 + +**当前状态：P0**。AgentKit 自身未提供 PG 备份 runbook， +依赖客户侧数据库运维。 + +**AgentKit 侧已实现**： + +- PostgreSQL 持久化（pgvector 扩展，事务日志 WAL 默认开启）。 +- Redis 配置 AOF/RDB（由客户 Redis 运维决定）。 + +**P0 补齐计划**： + +1. 新增 `docs/deployment/pg-backup-runbook.md`，包含： + - 全量备份策略（每日 `pg_dump`，保留 7 天） + - WAL 归档（连续归档，PITR 恢复） + - 恢复演练流程（每季度执行一次恢复测试） + - 备份加密（备份文件 SM4/AES-256 加密，密钥由 KMS 托管） +2. Helm Chart 增加 `backup` 段，可选部署 pg-backup sidecar / CronJob。 +3. pgvector 向量数据备份纳入 `pg_dump`（已原生支持）。 + +### 3.5 剩余信息保护（DAT-04）— 已实现 + +#### Session 撤销 + +JWT V2 token 携带 `sid`（session id）+ `jti`（access token 唯一 id）， +中间件校验 session 是否被服务端撤销 +（[jwt_utils.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py)）： + +- 销户时撤销该用户全部 session + （[auth.py deprovision](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py)）。 +- Refresh token 轮换：旧 token 进入 denylist 30s 窗口 + （[denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py)）。 +- Token reuse 检测：reuse 触发撤销该用户全部 session（强制重新登录）。 + +#### Refresh token hash 化 + +denylist 不存储明文 refresh token，存储其 SHA-256 hash +（[denylist.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py) +`hash_token`），减少敏感信息驻留内存。 + +#### JIT 用户随机密码 + +OIDC/SAML JIT 创建的本地用户使用随机密码 +（`hash_password(secrets.token_urlsafe(32))`），不可用于本地登录， +避免 SSO 用户的本地密码驻留 +（[oidc.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py) +`_find_or_create_user`）。 + +#### SCIM 用户随机密码 + +SCIM 创建的用户同样使用随机密码 +（[scim/router.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py) +`create_user`）。 + +### 3.6 数据采集与分类分级（DAT-06）— 已实现 + +- **PII 分类**：pii_filter.py 按 PII 类型分级 + （id_card / phone / email / bank_card），不同类型采用相同脱敏策略 + 但审计 hash 独立记录。 +- **记忆分层**：4 层记忆架构（SOUL/USER/MEMORY/DAILY）按敏感度存储 + （SOUL 最敏感，DAILY 最不敏感），详见 AGENTS.md。 +- **数据最小化**：PII 脱敏后原文不落盘（仅 hash 用于审计）， + LLM prompt 中不含原文 PII + （[pii_filter.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py)）。 + +### 3.7 数据销毁（DAT-07）— 已实现 + +- **账户销户**：deprovision 禁用账户（`is_active = 0`）+ 撤销全部 session。 +- **审计保留**：销户操作本身记录审计日志，不可删除（admin 不可删除审计记录， + 审计表无 delete 接口）。 +- **PII 销毁**：脱敏后 PII 原文不落盘（内存中处理后即替换）， + 无需专门销毁流程。 +- **PG 数据销毁**：客户侧 PG 运维负责（如需彻底删除用户数据， + 由 admin 发起 SQL 操作，记录审计）。 + +## 4. 测评建议 + +1. **DAT-01 / DAT-04 / DAT-06 / DAT-07**：直接展示 PG 配置 + jwt_utils.py + + denylist.py + pii_filter.py 即可。 +2. **DAT-02 / DAT-05**：展示 pii_filter.py fail-closed 流程 + OTel 脱敏指标。 +3. **DAT-03**：P0 补齐项，需在测评前完成 pg-backup-runbook.md。 +4. **国密**：P1 参考，不阻碍三级认证，但写入整改计划。 + +## 5. PII 脱敏边界（重要说明） + +AgentKit PII 脱敏的**边界**： + +1. **覆盖范围**：仅脱敏发送至 LLM 的 prompt，**不脱敏**： + - 用户本地终端命令（终端命令有自己的审计） + - 用户上传的文档原文（文档处理模块不经过 pii_filter） + - bitable 数据（结构化数据，由客户自行控制） +2. **ner_hook 依赖客户**：正则版覆盖中国常见 PII（手机/邮箱/身份证/银行卡）， + 复杂 PII（地址/姓名/组织）需客户接入内网 NER 模型。 +3. **不替代 DLP**：AgentKit PII 脱敏是应用层防护，不替代客户侧 + DLP（数据防泄漏）系统。 diff --git a/docs/compliance/dengbao-level3/06-management.md b/docs/compliance/dengbao-level3/06-management.md new file mode 100644 index 0000000..b52da74 --- /dev/null +++ b/docs/compliance/dengbao-level3/06-management.md @@ -0,0 +1,215 @@ +# 06 — 管理安全 + +> GB/T 22239-2019 第三级「安全管理」控制点映射。 + +## 1. 维度说明 + +管理安全覆盖「安全管理制度的运行」，AgentKit 自身已实现： + +- **已实现**：密钥轮换 runbook（U5，10 个 secret slot）、 + break-glass 告警流程（U4 R1b）、审计日志记录、应急隔离（销户 + session 撤销）。 +- **P0 补齐项**：变更管理流程文档化（CI/CD 流水线规范）。 +- **P1 参考**：定期安全评估流程。 +- **待评估**：应急响应预案演练（依赖客户安全团队协同）。 + +## 2. GB/T 22239-2019 三级要求（简要） + +| 控制点 ID | 要求摘要 | +|-----------|----------| +| MGT-01 | 密钥管理：密钥生成/存储/分发/轮换/销毁 | +| MGT-02 | 变更管理：变更审批/测试/回滚/审计 | +| MGT-03 | 应急响应：应急事件处置流程，隔离/恢复 | +| MGT-04 | 安全审计与评估：定期安全评估，漏洞管理 | +| MGT-05 | 配置管理：基线配置，配置变更审计 | +| MGT-06 | 应急通道（break-glass）：高权限应急通道，审计追溯 | +| MGT-07 | 密钥泄露响应：泄露检测/隔离/轮换/通报 | + +## 3. AgentKit 实现映射 + +### 3.1 密钥管理（MGT-01）— 已实现 + +#### 10 个 secret slot + 轮换 runbook + +[docs/deployment/key-rotation-runbook.md](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md) ++ [docs/deployment/secret-inventory.md](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/secret-inventory.md) + +| # | Slot 名称 | 用途 | 轮换周期 | 归属 | +|---|-----------|------|----------|------| +| 1 | JWT 签名密钥 | access(15m)/refresh(7d) token HS256 签名 | 90 天 | 安全团队 | +| 2 | API Key | 服务间 / SCIM / `agentkit pair` 鉴权 | 180 天 | 平台运维 | +| 3 | DB 凭据 | 应用层 PG 连接 DSN | 90 天 | DBA | +| 4 | IDP 签名证书 | OIDC/SAML IdP 签名证书 PEM（多 IDP 热轮换） | 365 天 | 身份团队 | +| 5 | 第三方 API Key | LLM provider API Key（JSON map） | 90 天 | LLM 运维 | +| 6 | TLS 服务器证书 | Ingress TLS 终止证书 + 私钥 | 365 天 | 平台运维 | +| 7 | mTLS 客户端证书 | 下游服务（KMS/HSM）mTLS | 180 天 | 安全团队 | +| 8 | KMS/HSM PKCS#11 PIN | 硬件安全模块访问 PIN | 365 天 | HSM 运维 | +| 9 | OTel exporter token | OTLP collector 鉴权 bearer | 90 天 | 可观测性团队 | +| 10 | Redis-PG 密码 | Redis requirepass + PG 服务密码 | 90 天 | DBA | + +#### 通用轮换原则 + +1. **零停机**：新密钥生效后再淘汰旧密钥，避免单点切换失败。 +2. **审计先行**：轮换前记录当前密钥指纹（hash8），轮换后校验变更。 +3. **回滚预案**：保留旧密钥值至少一个轮换周期，便于快速回滚。 +4. **权限最小化**：轮换操作仅限归属角色，操作全程审计日志。 +5. **后端无关**：适用于 Sealed Secrets 与 External Secrets 两种后端。 + +#### 密钥存储 + +- **禁止明文 Kubernetes Secret + Git 提交**（KTD-4 / R2a）。 +- 后端二选一：Sealed Secrets（Bitnami）或 External Secrets Operator + （指向 Vault / AWS SM / GCP SM / Azure KV）。 +- 渲染到统一 `-secrets` Secret，通过 `env.valueFrom.secretKeyRef` + 注入 Pod。 + +详见 [values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) +`secrets` 段。 + +#### 热证书轮换（SAML） + +SAML IDP 签名证书支持**热轮换**（不重启即时生效）： +[src/agentkit/server/auth/idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) +`update_certificate` 方法。OIDC 通过 `server_metadata_url` 自动发现 +（authlib 缓存 TTL），无需显式热轮换。 + +### 3.2 break-glass 应急通道（MGT-06 / MGT-07）— 已实现 + +#### break-glass deprovisioning + +[src/agentkit/server/routes/auth.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py) +`deprovision_user` 端点支持 `break_glass: true` 标记： + +```python +class DeprovisionRequest: + reason: str | None = None + break_glass: bool = False # 高权限账户 break-glass 标记 +``` + +`break_glass=True` 时审计记录 `source=break_glass` +（[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) +`SOURCE_BREAKGLASS`），用于事后追溯应急操作。 + +#### 应急流程 + +1. **隔离**：admin 通过 deprovision 端点强制销户 + （`break_glass=True` + `reason` 说明） +2. **撤销**：自动撤销该用户全部活跃 session +3. **审计**：记录 actor/target/reason/source=break_glass/timestamp +4. **告警**：OTel span event `audit.deprovision` 接入审计通道 +5. **通报**：由安全团队人工通报（AgentKit 不自动通报，避免误报） + +#### 密钥泄露响应 + +[key-rotation-runbook.md §紧急轮换](file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md) +「紧急轮换（泄露响应）」段落： + +1. **隔离**：撤销泄露密钥（provider 控制台 / HSM / CA） +2. **轮换**：按对应 slot 步骤紧急轮换（不遵循「双密钥并行」原则， + 直接切换到新密钥并废弃旧密钥） +3. **审计**：检索泄露密钥的使用日志，确认无异常调用 +4. **通报**：通知安全团队 + 受影响用户 +5. **复盘**：记录泄露原因，加固访问控制 + +### 3.3 变更管理（MGT-02 / MGT-05）— P0 补齐项 + +#### 已实现 + +- **Git 版本控制**：所有代码 / 配置 / Helm Chart 纳入 Git， + 变更可追溯。 +- **Helm Chart checksum**：ConfigMap 变更触发 Pod rollout + （[deployment.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml) + `checksum/config` annotation），配置变更自动生效。 +- **配置同步**：[config_sync.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py) + 支持配置版本管理 + 轮询同步。 +- **审计记录**：所有 RBAC 变更、deprovision 操作记录审计 + （[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py)）。 + +#### P0 补齐项：CI/CD 流水线规范 + +**当前状态：P0**。CI/CD 流水线（GitHub Actions / Gitea Actions）规范 +未文档化。 + +**P0 补齐计划**： + +1. 新增 `docs/CI-CD-STANDARDS.md`，包含： + - 分支策略（feature 分支 → PR → main） + - 必须的 CI 检查（ruff + pytest + 镜像漏洞扫描 P0） + - 部署审批流程（Staging → Production） + - 回滚流程（Helm rollback） +2. CI 集成镜像漏洞扫描（详见 `03-host.md` §3.1 P0 补齐项）。 +3. 部署变更审计：`kubectl rollout` 历史记录接入审计通道。 + +### 3.4 应急响应（MGT-03）— 待评估 + +**当前状态：待评估**。AgentKit 自身应急隔离能力已实现（销户 + session 撤销 ++ 密钥紧急轮换），但**完整应急响应预案**需与客户安全团队协同： + +- 事件分级标准（P0/P1/P2） +- 响应时限（P0 事件 15min 内响应） +- 通报链路（AgentKit admin → 客户安全团队 → 测评机构） +- 恢复流程（密钥轮换 → 服务恢复 → 数据完整性校验） + +**建议**：客户安全团队基于本套等保文档 + AgentKit 应急能力 +（销户/密钥轮换/审计日志）构建完整预案，并在认证前完成至少一次 +桌面演练。 + +### 3.5 安全审计与评估（MGT-04）— P1 参考 + +**当前状态：P1**。AgentKit 审计日志已记录，但定期评估流程未建立。 + +#### 已实现 + +- **审计日志覆盖**：RBAC 变更 / deprovision / 终端命令 / SCIM 操作 + 全部记录审计 + （[audit.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py) + + [terminal_security.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py)）。 +- **OTel 指标监控**：U1 OTel 暴露应用指标，可观测性平台监控异常 + （[metrics.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py)）。 + +#### P1 补齐计划 + +1. **定期漏洞扫描**：每月对依赖执行 `pip-audit` / `safety` 扫描。 +2. **代码安全评估**：每次重大版本发布前进行 SAST 扫描 + （如 Bandit / Semgrep）。 +3. **渗透测试**：认证后每年至少一次渗透测试（客户侧组织）。 +4. **审计日志定期审查**：admin 每季度审查审计日志异常模式 + （非自动告警，依赖人工 review — P1 可引入 SIEM 自动告警）。 + +### 3.6 配置管理（MGT-05）— 已实现 + +- **Helm values 标准化**：所有可配置项集中在 + [values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml)， + 含注释说明。 +- **配置优先级**：CLI 参数 > `agentkit.yaml` > 环境变量（`${VAR:-default}`） + > `.env` > 硬编码默认值（AGENTS.md）。 +- **配置版本管理**：[config_sync.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py) + 支持配置版本 + 轮询同步。 +- **密钥与配置分离**：含密钥的段（llm/auth/telemetry）由 env 注入， + 不进 ConfigMap；非密配置进 ConfigMap + （[values.yaml](file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml) + `config` 段注释）。 + +## 4. 测评建议 + +1. **MGT-01 / MGT-06 / MGT-07**：直接展示 key-rotation-runbook.md + + secret-inventory.md + audit.py break_glass 流程即可。 +2. **MGT-02 / MGT-05**：已实现部分展示 Helm Chart + config_sync， + P0 补齐 CI/CD 规范文档。 +3. **MGT-03**：待评估，由客户安全团队基于本套文档构建完整预案。 +4. **MGT-04**：P1 参考，写入整改计划（依赖扫描 + 季度审计 review）。 + +## 5. 密钥管理边界（重要说明） + +AgentKit 密钥管理的**边界**： + +1. **不持有 HSM/KMS 硬件**：HSM/KMS 由客户运维，AgentKit 通过 + PKCS#11 PIN 访问（slot 8）。 +2. **不签发证书**：TLS / mTLS 证书由客户 CA / cert-manager 签发， + AgentKit 仅消费。 +3. **不托管 LLM provider 密钥**：DashScope/DeepSeek/OpenAI/Anthropic + 的密钥在 provider 控制台生成，AgentKit 仅存储引用（slot 5）。 +4. **多实例密钥同步**：进程内 IDP 注册表不持久化，多实例部署时 + 各进程独立加载 `agentkit.yaml`；证书热轮换需配合配置同步 + （config_sync.py 轮询）或重启 + （[idp_registry.py](file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py) + ponytail 注释）。 diff --git a/docs/compliance/dengbao-level3/control-points.yaml b/docs/compliance/dengbao-level3/control-points.yaml new file mode 100644 index 0000000..02edd36 --- /dev/null +++ b/docs/compliance/dengbao-level3/control-points.yaml @@ -0,0 +1,491 @@ +# Fischer AgentKit — 等保三级控制点清单 +# +# 按 GB/T 22239-2019 第三级 6 维度组织。每个控制点含： +# id — 控制点唯一标识（DIM-NN） +# dimension — 维度（physical / network / host / application / data / management） +# title — 控制点名称 +# requirement — GB/T 22239-2019 三级要求摘要 +# implementation — AgentKit 实现映射（含状态说明） +# status — implemented / p0 / p1 / 待评估 +# implemented = 已实现，代码/配置已落地 +# p0 = 认证 readiness 必需，POC 采购关闭前补齐 +# p1 = 下一阶段补齐，写入整改计划 +# 待评估 = 依赖客户侧或需进一步评估 +# references — AgentKit 代码/配置文件绝对路径（file:/// 链接） +# +# 总计 44 个控制点：implemented=26, p0=6, p1=3, 待评估=9 + +# ============================================================================= +# 1. 物理安全（6 个，全部待评估 — 依赖客户机房） +# ============================================================================= + +- id: "PHY-01" + dimension: physical + title: "物理访问控制" + requirement: "机房出入口配置电子门禁，记录进入人员和时间" + implementation: "AgentKit 不运维物理机房。K8s 部署在客户内网，物理访问由客户机房运营方控制。AgentKit 仅提供部署形态证明（非 root 容器 + Helm Chart）。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + +- id: "PHY-02" + dimension: physical + title: "防盗窃和防破坏" + requirement: "主要设备设置防盗标识，机房设置监控系统" + implementation: "无物理设备责任。K8s worker node 由客户运维，机房监控由客户方提供。" + status: "待评估" + references: [] + +- id: "PHY-03" + dimension: physical + title: "防雷击" + requirement: "机房设置防雷保安器，接地电阻符合要求" + implementation: "无防雷责任。依赖客户机房防雷设施。" + status: "待评估" + references: [] + +- id: "PHY-04" + dimension: physical + title: "防火" + requirement: "机房设置火灾自动消防系统，检测报警器" + implementation: "无消防责任。依赖客户机房消防系统。" + status: "待评估" + references: [] + +- id: "PHY-05" + dimension: physical + title: "防水和防潮" + requirement: "机房安装水敏检测装置，防止水管安装" + implementation: "无防水责任。依赖客户机房防水设施。" + status: "待评估" + references: [] + +- id: "PHY-06" + dimension: physical + title: "防静电" + requirement: "机房采用防静电地板，设置静电消除器" + implementation: "无防静电责任。依赖客户机房防静电设施。" + status: "待评估" + references: [] + +# ============================================================================= +# 2. 网络安全（8 个） +# ============================================================================= + +- id: "NET-01" + dimension: network + title: "网络架构" + requirement: "划分网络区域，避免单点故障" + implementation: "单 Ingress 入口 + ClusterIP Service 架构。Ingress 是唯一对外入口，Redis/PG/OTel 全部 ClusterIP 内部访问。API 与 GUI 走分离路径（/api 与 /）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "NET-02" + dimension: network + title: "边界防护（Ingress TLS）" + requirement: "边界实施访问控制和安全过滤" + implementation: "Ingress 启用 TLS 终止，所有外网流量强制 HTTPS。TLS 证书通过 cert-manager 自动续期或手动签发（365 天轮换），私钥通过 Sealed Secret / External Secret 注入（KTD-4 / R2a）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "NET-03" + dimension: network + title: "访问控制（NetworkPolicy）" + requirement: "网络层访问控制策略，最小权限" + implementation: "P0 补齐项。Helm Chart 暂无 NetworkPolicy 模板，Pod 间东西向流量默认全通。计划新增 networkpolicy.yaml，默认 deny-all + 白名单放行（Ingress→Pod, Pod→Redis/PG/OTel）。当前缓解：Service 全部 ClusterIP，Redis/PG 由客户 K8s 网络策略保护。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + +- id: "NET-04" + dimension: network + title: "入侵防范" + requirement: "网络入侵检测，恶意流量检测" + implementation: "应用层已实现 RateLimiter 滑动窗口限流 + Webhook 独立限流 + JWT/SCIM token 恒定时间比较。网络层 IDS/IPS 依赖客户骨干网络。P1：接入客户 SIEM，将 OTel 日志投递到客户入侵检测平台。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "NET-05" + dimension: network + title: "安全审计（OTel）" + requirement: "网络审计日志，覆盖关键网络活动" + implementation: "OpenTelemetry（U1 强制依赖）实现网络活动审计：FastAPI Instrumentor 自动埋点每个 HTTP 请求 span；RBAC 变更/SCIM 失败/终端命令写入审计 + OTel span event；OTLP gRPC 导出到内部 collector。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "NET-06" + dimension: network + title: "恶意代码防范" + requirement: "网络边界恶意代码检测" + implementation: "AgentKit 不在网络边界位置，恶意代码检测由客户网络骨干（防病毒网关/沙箱）负责。AgentKit 侧缓解：容器镜像来自可信 CI 构建，不在运行时下载可执行文件；终端命令受 6 层白名单 + 危险检测约束。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +- id: "NET-07" + dimension: network + title: "安全审计日志保留（180 天）" + requirement: "日志保留期满足合规要求（≥180 天）" + implementation: "P0 补齐项。auth_audit_log 与 terminal_audit_logs 表无自动清理/归档策略。计划：Helm values 增加 audit.retentionDays=180（KTD-9），实现归档 CronJob 导出超过 180 天记录到对象存储并清理主表，OTel collector 配置 180 天保留策略。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +- id: "NET-08" + dimension: network + title: "资源控制（会话/带宽）" + requirement: "会话与带宽限制，防止资源耗尽" + implementation: "容器层 Pod resources requests/limits（CPU 250m~1000m / Mem 512Mi~2Gi）；应用层 RateLimiter IP 滑动窗口 + Webhook 独立限流；终端 session 1800s 闲置断开；JWT access 15min 强制重新认证。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + +# ============================================================================= +# 3. 主机安全（7 个，以容器/Pod 为单元） +# ============================================================================= + +- id: "HST-01" + dimension: host + title: "身份鉴别（ServiceAccount）" + requirement: "设备/系统身份标识与鉴别" + implementation: "AgentKit Pod 使用专用 ServiceAccount（不共享 default SA），Helm Chart 自动创建。可选 workload identity 注入 projected SA token 用于访问云 KMS/STS。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/serviceaccount.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "HST-02" + dimension: host + title: "访问控制（Pod 安全上下文）" + requirement: "操作系统/账户权限最小化，远程管理受控" + implementation: "podSecurityContext.runAsNonRoot=true（UID/GID 1000）；securityContext.allowPrivilegeEscalation=false + capabilities.drop=[ALL]；readOnlyRootFilesystem=false（需写临时文件，通过 emptyDir 挂载 /tmp 隔离）；未启用特权模式。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile" + +- id: "HST-03" + dimension: host + title: "安全审计（容器日志标准化采集）" + requirement: "设备审计日志，覆盖系统管理活动" + implementation: "已实现基础审计：stdout/stderr 由 K8s 采集 + OTel trace 导出 + auth_audit_log 表 + terminal_audit_logs 表 + OTel span event。P0 补齐：容器日志到客户日志平台的标准化采集链路 + 审计日志 JSON 化便于 SIEM 解析 + 180 天保留（与 NET-07 协同）。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/setup.py" + +- id: "HST-04" + dimension: host + title: "入侵防范（容器层）" + requirement: "主机入侵检测，系统加固" + implementation: "容器层已实现：非 root + drop ALL capabilities 限制系统调用；allowPrivilegeEscalation=false 阻止提权；资源 limits 防失控；Liveness/Readiness 探针故障自动重启。node 层 HIDS（Falco/OSSEC）依赖客户侧。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + +- id: "HST-05" + dimension: host + title: "恶意代码防范（主机防病毒）" + requirement: "主机防病毒，恶意代码检测" + implementation: "AgentKit 容器内不部署防病毒（无文件系统持久化，镜像来自可信 CI）。K8s node 层主机防病毒/HIDS 由客户运维。依赖客户机房 node 安全基线。" + status: "待评估" + references: [] + +- id: "HST-06" + dimension: host + title: "资源控制（CPU/内存/磁盘）" + requirement: "CPU/内存/磁盘限制，会话超时" + implementation: "Pod resources requests/limits（CPU 250m~1000m / Mem 512Mi~2Gi）；应用层 RateLimiter；终端 session 1800s 闲置断开；JWT access 15min / refresh 7d。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + +- id: "HST-07" + dimension: host + title: "镜像与补丁管理" + requirement: "系统补丁及时更新，镜像来源可信" + implementation: "P0 补齐项。已实现：多阶段构建 + 非 root appuser（UID 1001）+ python:3.11-slim 基础镜像 + HEALTHCHECK。P0 待补：CI 集成 Trivy 镜像漏洞扫描（HIGH/CRITICAL 阻断）+ image.digest 不可变引用。P1 增强：distroless 基础镜像 + cosign 签名。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/Dockerfile" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +# ============================================================================= +# 4. 应用安全（8 个，全部已实现 — AgentKit 核心责任） +# ============================================================================= + +- id: "APP-01" + dimension: application + title: "身份鉴别（SSO 多 IDP + JWT）" + requirement: "用户身份标识与鉴别，口令复杂度，登录失败处理" + implementation: "U4 SSO 多 IDP（OIDC authlib + SAML python3-saml，热证书轮换，R1a 按 sub 关联禁止邮箱合并）；本地密码 bcrypt rounds=12；JWT HS256 access 15min + refresh 7d（30d 记住我）含 sid+jti claims；refresh token 轮换 + reuse 检测撤销全 session；OAuth state CSRF 防护 TTL 5min 一次性消费。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/saml.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py" + +- id: "APP-02" + dimension: application + title: "访问控制（RBAC 3 级 + 终端 6 层）" + requirement: "基于角色的访问控制，最小权限，默认拒绝" + implementation: "3 级 RBAC（member/operator/admin）+ 9 权限位（CHAT/KB_QUERY/KB_WRITE/WORKFLOW_EXECUTE/TERMINAL_LOCAL_USE/TERMINAL_SERVER_USE/TERMINAL_WHITELIST_MANAGE/USER_MANAGE/SYSTEM_CONFIG）。R7a 终端安全 6 层白名单（blocklist→builtin→global→user→session→danger），shell 操作符永远绕过白名单强制走 danger detection。终端双层授权（RBAC 角色 + 个人授权标志）。SCIM token 恒定时间比较。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/permissions.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "APP-03" + dimension: application + title: "安全审计（OTel + audit.py）" + requirement: "应用审计日志，覆盖用户操作关键事件" + implementation: "U1 OTel 强制依赖。审计事件：role_change / deprovision（含 break_glass 标记），写入 auth_audit_log 表 + OTel span event（OTel 不可用降级 SQLite+日志）。SCIM 推送失败 span event scim.push.failure。终端命令审计 terminal_audit_logs（决策 + cwd + exit_code）。OTel 指标：agent.request.total / pii_filter.coverage 等。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "APP-04" + dimension: application + title: "通信完整性（TLS + JWT + Webhook 签名）" + requirement: "传输完整性保护，防篡改" + implementation: "传输层 Ingress TLS 终止（HTTPS）；JWT HS256 签名校验（篡改即失败）；Webhook 入站 fail-closed 签名校验失败返回 401；mTLS 客户端证书（Helm 预留 secrets.mtlsClientCert slot 用于下游 KMS/HSM）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/ingress.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + +- id: "APP-05" + dimension: application + title: "软件容错（fallback + fail-closed）" + requirement: "错误处理，故障隔离，降级机制" + implementation: "LLM 网关多 provider fallback；OTel 运行时错误降级 NoOpTracer 不崩溃；PII 脱敏 fail-closed（异常拒绝发送不降级原文，默认 fail_closed=True）；专家团队全失败回退单 agent；U3 DR-fixes（bitable 字段校验 DR-1 / LastViewDeletionError DR-4 / 工具调用容错 DR-5）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/gateway.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/tracer.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/repository.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/bitable/service.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/tools/bitable_tool.py" + +- id: "APP-06" + dimension: application + title: "资源控制（限流 + 会话超时）" + requirement: "会话并发控制，超时自动断开" + implementation: "RateLimiter IP 滑动窗口 + Webhook 独立限流 + nonce dedup；终端 session 1800s 闲置断开；JWT access 15min 过期；专家名称正则校验防注入；HandoffTransport 有界队列 maxsize=1024 + sentinel 关闭模式。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/middleware.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/terminal_server.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + +- id: "APP-07" + dimension: application + title: "应用账户生命周期（SCIM 2.0 + JIT + deprovision）" + requirement: "开户/变更/销户审计，SCIM 自动化" + implementation: "SCIM 2.0 端点（POST/PATCH/GET Users，bearer token 恒定时间比较）；OIDC/SAML JIT 创建本地用户 + user_idp_links（R1a 按 sub 关联禁止邮箱合并，随机密码不可本地登录）；deprovision 端点禁用账户 + 撤销全 session + 审计（含 break_glass 标记）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/providers/oidc.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "APP-08" + dimension: application + title: "代码安全（输入验证 + 依赖管理）" + requirement: "输入验证，输出编码，依赖漏洞管理" + implementation: "Pydantic v2 模型校验（extra=forbid 拒绝未知字段，SCIM/IDP 配置均启用）；禁止 any 类型（项目规则，用 Pydantic 模型或 Unknown）；专家名称正则 _EXPERT_NAME_RE 防 SQL/命令注入；API Key 恒定时间比较 hmac.compare_digest；Ruff lint+format（py311）；pyproject.toml 锁定精确版本；终端 6 层白名单防命令注入链。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/models.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/terminal_security.py" + +# ============================================================================= +# 5. 数据安全（8 个） +# ============================================================================= + +- id: "DAT-01" + dimension: data + title: "数据完整性（PG + pgvector）" + requirement: "数据传输/存储完整性，校验机制" + implementation: "PostgreSQL 事务 ACID 保证 + pgvector 扩展存储向量记忆；SQLAlchemy 2 async ORM 应用层事务；JWT HS256 签名防篡改；Webhook fail-closed 签名校验；Pydantic extra=forbid 防数据注入。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/channels.py" + +- id: "DAT-02" + dimension: data + title: "数据保密性（PII 脱敏 U2）" + requirement: "敏感数据加密存储/传输，密钥管理" + implementation: "U2 PII 脱敏 hook：正则版（手机/邮箱/身份证/银行卡）+ ner_hook（客户内网 NER 模型优先）+ fail-closed（异常拒绝发送不降级原文，默认 fail_closed=True）+ hash 审计（sha256 前 8 位，仅 hash 不含原文）。身份证正则优先于手机避免误匹配。OTel 指标 pii_filter.coverage / pii_filter.redacted。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "DAT-03" + dimension: data + title: "数据备份与恢复（PG backup runbook）" + requirement: "重要数据定期备份，恢复演练" + implementation: "P0 补齐项。已实现：PostgreSQL 持久化 + WAL 默认开启 + Redis AOF/RDB（客户运维）。P0 待补：新增 docs/deployment/pg-backup-runbook.md（每日 pg_dump 保留 7 天 + WAL 归档 PITR + 季度恢复演练 + 备份加密 SM4/AES-256）；Helm Chart 增加 backup 段（可选 pg-backup sidecar/CronJob）。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + +- id: "DAT-04" + dimension: data + title: "剩余信息保护（session revoke）" + requirement: "鉴别信息/文件/内存空间释放前清理" + implementation: "JWT V2 含 sid+jti claims，中间件校验 session 是否服务端撤销；销户撤销全 session；refresh token 轮换旧 token 进 denylist 30s 窗口；token reuse 检测撤销全 session；denylist 存 SHA-256 hash 不存明文；JIT/SCIM 用户随机密码不可本地登录。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/denylist.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/scim/router.py" + +- id: "DAT-05" + dimension: data + title: "个人信息保护（PII 最小化）" + requirement: "PII 采集/处理/传输最小化与脱敏" + implementation: "PII 脱敏后原文不落盘（仅 hash 用于审计），LLM prompt 不含原文 PII；PII 按类型分级（id_card/phone/email/bank_card），不同类型 hash 独立记录；记忆 4 层架构（SOUL/USER/MEMORY/DAILY）按敏感度存储；数据最小化原则。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +- id: "DAT-06" + dimension: data + title: "数据采集与分类分级" + requirement: "按敏感度分级保护" + implementation: "PII 按 4 类分级（id_card/phone/email/bank_card）独立 hash 审计；记忆 4 层架构（SOUL 最敏感→DAILY 最不敏感）；PII 脱敏覆盖发送至 LLM 的 prompt（边界：不覆盖终端命令/上传文档/bitable 结构化数据，由客户自行控制）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +- id: "DAT-07" + dimension: data + title: "数据销毁" + requirement: "数据销毁审计，不可恢复" + implementation: "账户销户 deprovision 禁用账户+撤销全 session；审计记录不可删除（审计表无 delete 接口，admin 不可删除审计）；PII 原文脱敏后不落盘无需专门销毁；PG 数据销毁由客户 DBA 发起 SQL 记录审计。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "DAT-08" + dimension: data + title: "国密算法合规（R11a）" + requirement: "敏感数据加密使用国密算法（政企客户要求）" + implementation: "P1 参考。当前使用国际算法：TLS（AES-GCM）/ bcrypt 密码哈希 / HS256 JWT / SHA-256 PII hash。未使用国密（SM2/SM3/SM4/SM9）。P1 升级路径：JWT 签名→SM2-HMAC、密码哈希→SM3（双算法并行兼容）、PII hash→SM3、TLS→国密套件、PG 字段级加密→SM4 TDE。注：国密不阻碍等保三级认证（三级要求加密而非国密），但政企客户常要求国密合规。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/jwt_utils.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/password.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/llm/pii_filter.py" + +# ============================================================================= +# 6. 管理安全（7 个） +# ============================================================================= + +- id: "MGT-01" + dimension: management + title: "密钥管理（10 secret slot + 轮换 runbook）" + requirement: "密钥生成/存储/分发/轮换/销毁" + implementation: "U5 10 个 secret slot（JWT/API Key/DB/IDP/第三方API/TLS/mTLS/KMS-HSM/OTel/Redis-PG）含轮换周期（90~365 天）。禁止明文 K8s Secret+Git 提交（KTD-4/R2a），后端二选一：Sealed Secrets 或 External Secrets Operator。SAML IDP 证书热轮换（不重启即时生效，idp_registry.update_certificate）。通用原则：零停机双密钥并行+审计先行+回滚预案+权限最小化。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/secret-inventory.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/idp_registry.py" + +- id: "MGT-02" + dimension: management + title: "变更管理（CI/CD 规范）" + requirement: "变更审批/测试/回滚/审计" + implementation: "P0 补齐项。已实现：Git 版本控制所有代码/配置/Helm Chart；Helm Chart checksum/config annotation ConfigMap 变更触发 rollout；config_sync.py 配置版本管理+轮询同步；审计记录 RBAC 变更。P0 待补：新增 docs/CI-CD-STANDARDS.md（分支策略+CI 检查 ruff/pytest/镜像扫描+部署审批+回滚流程）；CI 集成镜像漏洞扫描（与 HST-07 协同）；部署变更审计接入。" + status: "p0" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/templates/deployment.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py" + +- id: "MGT-03" + dimension: management + title: "应急响应" + requirement: "应急事件处置流程，隔离/恢复" + implementation: "待评估。AgentKit 自身应急隔离能力已实现（销户+session 撤销+密钥紧急轮换，见 MGT-06/MGT-07），但完整应急响应预案需与客户安全团队协同：事件分级标准+响应时限+通报链路+恢复流程。建议客户基于本套等保文档+AgentKit 应急能力构建完整预案，认证前完成至少一次桌面演练。" + status: "待评估" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + +- id: "MGT-04" + dimension: management + title: "安全审计与评估" + requirement: "定期安全评估，漏洞管理" + implementation: "P1 参考。已实现：审计日志覆盖 RBAC 变更/deprovision/终端命令/SCIM 操作；OTel 指标监控异常。P1 补齐：每月依赖漏洞扫描（pip-audit/safety）；每次重大版本发布前 SAST 扫描（Bandit/Semgrep）；认证后每年至少一次渗透测试（客户侧组织）；admin 每季度审查审计日志异常模式（P1 可引入 SIEM 自动告警）。" + status: "p1" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/telemetry/metrics.py" + +- id: "MGT-05" + dimension: management + title: "配置管理（Helm values + 配置同步）" + requirement: "基线配置，配置变更审计" + implementation: "Helm values 标准化所有可配置项含注释；配置优先级 CLI>agentkit.yaml>环境变量>.env>硬编码默认；config_sync.py 配置版本管理+轮询同步；密钥与配置分离（含密钥段由 env 注入不进 ConfigMap，非密配置进 ConfigMap）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/deploy/helm/agentkit/values.yaml" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/config_sync.py" + +- id: "MGT-06" + dimension: management + title: "应急通道（break-glass）" + requirement: "高权限应急通道，审计追溯" + implementation: "U4 R1b break-glass deprovisioning：/admin/users/{id}/deprovision 端点支持 break_glass=true 标记，审计记录 source=break_glass（SOURCE_BREAKGLASS）。流程：隔离（销户）→撤销全 session→审计（actor/target/reason/source/timestamp）→OTel span event audit.deprovision 接入审计通道→人工通报。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/routes/auth.py" + - "file:///Users/chiguyong/Code/Fischer-agentkit/src/agentkit/server/auth/audit.py" + +- id: "MGT-07" + dimension: management + title: "密钥泄露响应" + requirement: "泄露检测/隔离/轮换/通报" + implementation: "key-rotation-runbook.md §紧急轮换（泄露响应）段落：1.隔离（撤销泄露密钥 provider控制台/HSM/CA）→2.轮换（按对应 slot 步骤紧急轮换，不遵循双密钥并行直接切换废弃旧密钥）→3.审计（检索泄露密钥使用日志确认无异常）→4.通报（安全团队+受影响用户）→5.复盘（记录原因加固访问控制）。" + status: "implemented" + references: + - "file:///Users/chiguyong/Code/Fischer-agentkit/docs/deployment/key-rotation-runbook.md" diff --git a/docs/deployment/k8s-install.md b/docs/deployment/k8s-install.md new file mode 100644 index 0000000..8714888 --- /dev/null +++ b/docs/deployment/k8s-install.md @@ -0,0 +1,246 @@ +# Fischer AgentKit — K8s 部署指南 + +本文档指导在原生 Kubernetes 1.28+ 集群上通过 Helm Chart 部署 Fischer AgentKit。 +覆盖在线与离线两种安装场景，以及 Sealed Secrets / External Secrets 两种密钥后端。 + +## 1. 前置条件 + +| 组件 | 版本 | 说明 | +|------|------|------| +| Kubernetes | >= 1.28 | 原生 K8s（信创 K8s / OpenShift P1 支持） | +| Helm | 3.x | Chart 部署工具 | +| kubectl | 与集群对齐 | 集群操作 | +| docker / containerd | 任意 | 镜像构建与加载 | + +集群需已部署以下 Operator 之一（二选一，对应 `secrets.backend`）： + +- **Sealed Secrets**（`secrets.backend: sealed-secrets`） + - 安装：`helm install sealed-secrets sealed-secrets/sealed-secrets -n kube-system` + - 工具：`kubeseal`（用于生成 encryptedData） +- **External Secrets Operator**（`secrets.backend: external-secrets`） + - 安装：`helm install external-secrets external-secrets/external-secrets -n kube-system` + - 后端：Vault / AWS Secrets Manager / GCP Secret Manager / Azure Key Vault（运维预创建 SecretStore） + +依赖中间件（Redis + PostgreSQL）需在命名空间内可达： +- Redis：`redis://redis:6379/0`（可指向集群内 Redis 或外部托管） +- PostgreSQL（含 pgvector 扩展）：`postgresql+asyncpg://agentkit@postgres:5432/agentkit` + +> ponytail: Chart 默认不部署 Redis/PostgreSQL（避免与客户已有基础设施重复）。 +> 升级路径：subchart 方式集成 bitnami/redis + bitnami/postgresql（P1）。 + +## 2. 在线安装 + +### 2.1 准备密钥 + +按 `docs/deployment/secret-inventory.md` 准备 11 个 secret slot 的明文值， +然后根据所选 backend 生成密钥资源。 + +**Sealed Secrets 方式**（推荐单集群 / 离线场景）： + +```bash +# 1) 写明文到临时 Secret（注意：永远不要 git commit 明文 Secret） +kubectl create secret generic agentkit-secrets --dry-run=client -o yaml \ + --from-literal=jwt-signing-key='<32字节随机串>' \ + --from-literal=api-key='' \ + --from-literal=database-url='postgresql+asyncpg://agentkit:@postgres:5432/agentkit' \ + --from-literal=idp-signing-certs='{"feishu":"-----BEGIN CERT..."}' \ + --from-literal=third-party-api-keys='{"dashscope":"sk-...","deepseek":"sk-..."}' \ + --from-literal=tls-server-cert='' \ + --from-literal=mtls-client-cert='' \ + --from-literal=kms-hsm-pin='' \ + --from-literal=otel-exporter-token='Bearer ' \ + --from-literal=redis-password='' \ + --from-literal=postgres-password='' \ + > /tmp/agentkit-secret-plaintext.yaml + +# 2) 用 kubeseal 加密（需 controller 已运行） +kubeseal --controller-name=sealed-secrets --controller-namespace=kube-system \ + --format yaml < /tmp/agentkit-secret-plaintext.yaml > agentkit-sealedsecret.yaml + +# 3) 明文临时文件必须销毁 +shred -u /tmp/agentkit-secret-plaintext.yaml + +# 4) 应用 SealedSecret +kubectl apply -f agentkit-sealedsecret.yaml -n agentkit +``` + +**External Secrets 方式**（推荐多集群 / 云托管场景）： + +```bash +# 1) 在 Vault / AWS SM / GCP SM 中预置 10 个 key（路径见 templates/external-secret.yaml remoteRef.key） +# 2) 创建 SecretStore（指向外部密钥后端） +kubectl apply -f - <<'EOF' +apiVersion: external-secrets.io/v1beta1 +kind: SecretStore +metadata: + name: agentkit-vault-store + namespace: agentkit +spec: + provider: + vault: + server: https://vault.internal:8200 + path: secret + auth: + kubernetes: + mountPath: kubernetes + role: agentkit +EOF + +# 3) Chart 安装时 secrets.backend=external-secrets 自动生成 ExternalSecret +``` + +### 2.2 helm install + +```bash +# 创建 namespace +kubectl create namespace agentkit + +# 安装（默认 sealed-secrets backend） +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set ingress.host=agentkit.your-domain.com \ + --set image.repository=fischer-agentkit \ + --set image.tag=0.1.0 + +# 或使用 External Secrets backend +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set secrets.backend=external-secrets \ + --set secrets.externalSecret.secretStoreName=agentkit-vault-store \ + --set ingress.host=agentkit.your-domain.com + +# 或使用 override values 文件 +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + -f values-prod.yaml +``` + +### 2.3 启用 workload identity（KMS/HSM） + +```bash +helm install agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set workloadIdentity.enabled=true \ + --set workloadIdentity.audience=sts.your-cloud.com +``` + +## 3. 离线安装 + +离线场景（政企内网无公网）需先在联网环境打包，再拷贝到离线集群安装。 + +### 3.1 联网环境打包 + +```bash +cd deploy/offline +make all IMAGE_REPO=fischer-agentkit IMAGE_TAG=0.1.0 +# 产物：dist/agentkit-image-0.1.0.tar.gz + dist/agentkit-0.1.0.tgz +``` + +### 3.2 拷贝到离线集群 + +```bash +# 将整个 dist/ 目录拷贝到离线集群节点（scp / U 盘 / 内网传输） +scp -r dist/ user@offline-node:/opt/agentkit/dist +``` + +### 3.3 离线集群安装 + +```bash +ssh user@offline-node +cd /path/to/deploy/offline +make install DIST_DIR=dist NAMESPACE=agentkit RELEASE=agentkit +# install.sh 会：docker load 镜像 + helm install（image.pullPolicy=Never） +``` + +## 4. 安装后验证 + +```bash +# 1) Pod 状态 +kubectl get pods -n agentkit +# 期望：agentkit-xxx 1/1 Running + +# 2) Service / Ingress +kubectl get svc,ingress -n agentkit + +# 3) 健康检查 +# DEPLOY-1: 默认 release=agentkit 时 Deployment 名为 agentkit（fullname helper +# 在 release 名包含 chart 名时去重）。若改了 release 名，按 helm status 输出替换。 +kubectl exec -n agentkit deploy/agentkit -- \ + curl -s http://localhost:8001/api/v1/health +# 期望：{"status":"ok"} + +# 4) Secret 已注入 +kubectl exec -n agentkit deploy/agentkit -- env | grep -E 'JWT|API_KEY|DATABASE' | sed 's/=.*/=/' +# 期望：所有 env 都有值（不为空） + +# 5) helm 状态 +helm status agentkit -n agentkit +``` + +## 5. 故障排查 + +### 5.1 Pod CrashLoopBackOff + +```bash +# 查看日志 +kubectl logs -n agentkit deploy/agentkit --tail=50 + +# 常见原因： +# - Secret 未注入：检查 SealedSecret 是否已解封（kubectl get sealedsecret -n agentkit） +# - DB 连接失败：检查 postgres 服务可达性 + DATABASE_URL 正确性 +# - Redis 连接失败：检查 redis 服务 + REDIS_PASSWORD +``` + +### 5.2 SealedSecret 未解封 + +```bash +kubectl describe sealedsecret agentkit-secrets -n agentkit +# 若 status.conditions 显示 error，原因通常是： +# - sealed-secrets-controller 未运行（kubectl get pods -n kube-system | grep sealed-secrets） +# - 加密时用的 controller 与当前 controller 私钥不匹配（需重新 kubeseal） +``` + +### 5.3 ExternalSecret 同步失败 + +```bash +kubectl get externalsecret agentkit-secrets -n agentkit -o yaml +# 关注 status.conditions，常见： +# - SecretStore 不存在或不可达 +# - Vault / SM 权限不足 +# - remoteRef.key 在外部后端不存在 +``` + +### 5.4 Ingress 不通 + +```bash +# 检查 ingress controller +kubectl get pods -n ingress-nginx +# 检查 TLS secret +kubectl get secret agentkit-tls -n agentkit +# 检查 DNS 解析到 ingress controller LB +``` + +## 6. 卸载 + +```bash +helm uninstall agentkit -n agentkit +kubectl delete namespace agentkit +# 注意：SealedSecret 解封后的明文 Secret 需手动删除 +kubectl delete sealedsecret agentkit-secrets -n agentkit 2>/dev/null || true +``` + +## 7. 升级 + +```bash +# 镜像升级 +helm upgrade agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + --set image.tag=0.2.0 + +# 配置变更 +helm upgrade agentkit deploy/helm/agentkit/ \ + --namespace agentkit \ + -f values-prod.yaml +``` + +密钥轮换流程见 `docs/deployment/key-rotation-runbook.md`。 diff --git a/docs/deployment/key-rotation-runbook.md b/docs/deployment/key-rotation-runbook.md new file mode 100644 index 0000000..63df903 --- /dev/null +++ b/docs/deployment/key-rotation-runbook.md @@ -0,0 +1,300 @@ +# Fischer AgentKit — 密钥轮换 Runbook + +本文档给出全部 11 个 secret slot（KTD-4 / R2a）的轮换操作流程。 +所有轮换遵循「双密钥并行 → 切换 → 旧密钥退役」原则，避免服务中断。 + +## 通用轮换原则 + +1. **零停机**：新密钥生效后再淘汰旧密钥，避免单点切换失败。 +2. **审计先行**：轮换前记录当前密钥指纹（hash8），轮换后校验变更。 +3. **回滚预案**：保留旧密钥值至少一个轮换周期，便于快速回滚。 +4. **权限最小化**：轮换操作仅限归属角色，操作全程审计日志。 +5. **后端无关**：以下步骤适用于 Sealed Secrets 与 External Secrets 两种后端， + 差异处单独标注。 + +--- + +## Slot 1：JWT 签名密钥（90 天） + +**影响**：所有签发的 JWT access/refresh token 失效，用户需重新登录。 + +**步骤**： + +```bash +# 1) 生成新密钥（32 字节随机串，base64） +NEW_KEY=$(openssl rand -base64 32) + +# 2) 更新密钥后端 +# Sealed Secrets： +kubectl create secret generic agentkit-secrets-new --dry-run=client -o yaml \ + --from-literal=jwt-signing-key="${NEW_KEY}" | \ + kubeseal --controller-name=sealed-secrets --controller-namespace=kube-system \ + --format yaml --merge-into agentkit-sealedsecret.yaml +# External Secrets：在 Vault/SM 中更新 agentkit/jwt-signing-key + +# 3) 应用并等待同步 +kubectl apply -f agentkit-sealedsecret.yaml -n agentkit +# DEPLOY-1: 用 instance 标签定位 Deployment — 避免硬编码 fullname 拼错。 +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 4) 验证 +kubectl exec -n agentkit deploy/agentkit -- \ + python -c "import os; print('jwt key len:', len(os.environ['JWT_SIGNING_KEY']))" + +# 5) 用户重新登录（旧 token 失效） +``` + +**回滚**：保留旧密钥值，若新密钥异常，恢复 SealedSecret/ExternalSecret 并 restart。 + +--- + +## Slot 2：API Key（180 天） + +**影响**：使用旧 API Key 的客户端调用失败。 + +**步骤**： + +```bash +# 1) 生成新 API Key +NEW_KEY=$(openssl rand -hex 32) + +# 2) 更新密钥后端（同 Slot 1 步骤 2，key=api-key） + +# 3) 通知所有使用 API Key 的客户端更新（agentkit pair 重新生成） + +# 4) 应用并 restart +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证：用新 key 调用 /api/v1/auth/me 或 SCIM 端点 +``` + +**回滚**：恢复旧 key 值。 + +--- + +## Slot 3：DB 凭据 / DATABASE_URL（90 天） + +**影响**：数据库连接断开，需与 DBA 协同。 + +**步骤**： + +```bash +# 1) DBA 在 PostgreSQL 创建新密码（或新用户） +psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" + +# 2) 更新 DATABASE_URL（含新密码） +NEW_URL="postgresql+asyncpg://agentkit:new-pg-pwd@postgres:5432/agentkit" + +# 3) 更新密钥后端（key=database-url） + +# 4) restart pod（连接池重建） +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证：触发一次 DB 操作（如 GET /api/v1/memory） + +# 6) DBA 回收旧密码（确认新连接正常后） +``` + +**注意**：PostgreSQL `ALTER USER` 立即生效，旧连接保持直到连接池重建。 +建议在低峰期操作。 + +**回滚**：DBA 恢复旧密码 + 恢复 DATABASE_URL。 + +--- + +## Slot 4：IDP 签名证书（365 天） + +**影响**：OIDC/SAML 登录校验失败。多 IDP 时仅影响轮换的 IdP。 + +**步骤**： + +```bash +# 1) 从 IdP 管理后台导出新签名证书 PEM + +# 2) 更新 JSON map（保留其他 IdP 证书不变） +# 例如轮换 feishu 证书： +NEW_CERTS='{"feishu":"-----BEGIN CERTIFICATE-----\n新证书\n-----END CERTIFICATE-----","azure_ad":"<原证书>"}' + +# 3) 更新密钥后端（key=idp-signing-certs） + +# 4) restart pod（证书热轮换不重启，但 Helm 部署需 restart 加载新 Secret） +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证：通过该 IdP 发起一次 SSO 登录 +``` + +**回滚**：恢复旧证书 JSON map。 + +--- + +## Slot 5：第三方 API Key / LLM providers（90 天） + +**影响**：对应 provider 的 LLM 调用失败。JSON map 结构，单 provider 轮换不影响其他。 + +**步骤**： + +```bash +# 1) 在 provider 控制台生成新 key（DashScope / DeepSeek / OpenAI / Anthropic） + +# 2) 更新 JSON map（保留其他 provider 不变） +NEW_KEYS='{"dashscope":"sk-new1","deepseek":"sk-new2","openai":"sk-old"}' + +# 3) 更新密钥后端（key=third-party-api-keys） + +# 4) restart pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证：调用对应 provider 的模型（如 agentkit chat 发一条测试消息） + +# 6) 在 provider 控制台撤销旧 key +``` + +**回滚**：恢复旧 key（需 provider 控制台未撤销旧 key，建议保留旧 key 一个周期）。 + +--- + +## Slot 6：TLS 服务器证书（365 天） + +**影响**：Ingress TLS 终止证书过期，浏览器告警 / mTLS 客户端握手失败。 + +**步骤**： + +```bash +# 1) 签发新证书（cert-manager 自动续期，或手动从 CA 申请） +# cert-manager 模式：证书到期前自动续期，无需手动操作。 + +# 2) 若手动：更新 kubernetes.io/tls 类型 Secret +kubectl create secret tls agentkit-tls \ + --cert=new.crt --key=new.key --dry-run=client -o yaml | \ + kubectl apply -f - -n agentkit + +# 3) Ingress controller 自动感知新证书（无需 restart） + +# 4) 验证 +echo | openssl s_client -connect agentkit.your-domain.com:443 -servername agentkit.your-domain.com 2>/dev/null | openssl x509 -noout -dates +``` + +**注意**：本 slot 不在 SealedSecret/ExternalSecret 的 encryptedData 内统一管理 +（TLS Secret 类型为 `kubernetes.io/tls`，与普通 Secret 不同）。 +SealedSecret 模板中登记仅用于清单追踪，实际由 cert-manager 或运维单独管理。 + +**回滚**：恢复旧证书 Secret。 + +--- + +## Slot 7：mTLS 客户端证书（180 天） + +**影响**：与下游服务（KMS/HSM、内部 API）mTLS 握手失败。 + +**步骤**： + +```bash +# 1) 从内部 PKI 签发新客户端证书 + 私钥 + CA 证书 + +# 2) 更新密钥后端（key=mtls-client-cert，PEM 文本） +NEW_PEM="$(cat client.crt client.key ca.crt)" + +# 3) 更新密钥后端 + +# 4) restart pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 5) 验证：触发一次 mTLS 调用（如 KMS 签名操作） +``` + +**回滚**：恢复旧 PEM。 + +--- + +## Slot 8：KMS/HSM PKCS#11 PIN（365 天） + +**影响**：HSM 访问失败，密钥托管 / 签名 / 解密不可用。 + +**步骤**： + +```bash +# 1) HSM 管理员通过 HSM 管理工具轮换 PIN（具体命令依 HSM 型号） + +# 2) 更新密钥后端（key=kms-hsm-pin） +NEW_PIN='' + +# 3) restart pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 4) 验证：触发一次 HSM 操作（如 PKCS#11 签名） +``` + +**注意**：HSM PIN 轮换需物理 / 管理员协同，建议提前预约维护窗口。 + +**回滚**：恢复旧 PIN（HSM 管理工具支持回滚 PIN）。 + +--- + +## Slot 9：OTel exporter token（90 天） + +**影响**：遥测数据上报失败（trace/metrics 丢失）。服务本身不受影响。 + +**步骤**： + +```bash +# 1) 在 OTel collector / 可观测性平台生成新 token + +# 2) 更新密钥后端（key=otel-exporter-token） +# DEPLOY-5: OTEL_EXPORTER_OTLP_HEADERS 期望 'key=value' 格式（OTel SDK 规范）， +# 不是裸 'Bearer '。secret 值必须形如 'Authorization=Bearer '。 +NEW_TOKEN='Authorization=Bearer new-otel-token' + +# 3) restart pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 4) 验证：在 collector 端确认收到新 trace/metrics +``` + +**回滚**：恢复旧 token。 + +--- + +## Slot 10：Redis-PG 密码（90 天） + +**影响**：Redis / PostgreSQL 连接断开。需与基础设施层协同。 + +**步骤**： + +```bash +# 1) Redis 轮换密码 +redis-cli -a '' CONFIG SET requirepass '' + +# 2) PostgreSQL 轮换密码（DBA） +psql -U postgres -c "ALTER USER agentkit WITH PASSWORD 'new-pg-pwd';" + +# 3) 更新密钥后端（两个 key：redis-password + postgres-password） + +# 4) 更新 Redis 服务配置（若 Redis 由 docker-compose / StatefulSet 管理， +# 需同步更新 requirepass 并 restart Redis） + +# 5) restart agentkit pod +kubectl rollout restart deployment -l app.kubernetes.io/instance=agentkit -n agentkit + +# 6) 验证 +kubectl exec -n agentkit deploy/agentkit -- \ + python -c "import redis,os; redis.from_url(os.environ['REDIS_URL'],password=os.environ['REDIS_PASSWORD']).ping()" +``` + +**注意**：Redis `CONFIG SET requirepass` 立即生效，旧连接保持到断开。 +建议低峰期操作，且 Redis 持久化配置（redis.conf）需同步更新，避免重启后失效。 + +**回滚**：恢复 Redis / PG 旧密码 + 恢复 Secret。 + +--- + +## 紧急轮换（泄露响应） + +发生密钥泄露时，立即按以下流程处置： + +1. **隔离**：撤销泄露密钥（provider 控制台 / HSM / CA） +2. **轮换**：按上述对应 slot 步骤紧急轮换 +3. **审计**：检索泄露密钥的使用日志，确认无异常调用 +4. **通报**：通知安全团队 + 受影响用户 +5. **复盘**：记录泄露原因，加固访问控制 + +紧急轮换不遵循「双密钥并行」原则，直接切换到新密钥并废弃旧密钥。 diff --git a/docs/deployment/secret-inventory.md b/docs/deployment/secret-inventory.md new file mode 100644 index 0000000..5a41688 --- /dev/null +++ b/docs/deployment/secret-inventory.md @@ -0,0 +1,67 @@ +# Fischer AgentKit — 密钥清单（Secret Inventory） + +本文档列出 Helm Chart 涉及的全部 11 个 secret slot（KTD-4 / R2a）， +每个 slot 含：名称、用途、来源、轮换周期、归属角色、对应环境变量。 + +> 密钥后端二选一：Sealed Secrets（`secrets.backend: sealed-secrets`）或 +> External Secrets Operator（`secrets.backend: external-secrets`）。 +> 禁止明文 Kubernetes Secret + Git 提交。 + +## 密钥清单总表 + +| # | Slot 名称 | Secret Key | 环境变量 | 用途 | 来源 | 轮换周期 | 归属 | +|---|-----------|-----------|----------|------|------|----------|------| +| 1 | JWT 签名密钥 | `jwt-signing-key` | `JWT_SIGNING_KEY` | JWT access(15m)/refresh(7d) token HS256 签名 | 运维生成（32+ 字节随机串） | 90 天 | 安全团队 | +| 2 | API Key | `api-key` | `API_KEY` | 服务间调用 / SCIM bearer / `agentkit pair` 鉴权 | 运维生成 | 180 天 | 平台运维 | +| 3 | DB 凭据 | `database-url` | `DATABASE_URL` | 应用层 PG 连接 DSN（含用户名+密码） | DBA 提供 | 90 天 | DBA | +| 4 | IDP 签名证书 | `idp-signing-certs` | `IDP_SIGNING_CERTS` | OIDC/SAML IdP 签名证书 PEM（JSON map，多 IDP 热轮换） | 各 IDP 管理员提供 | 365 天 | 身份团队 | +| 5 | 第三方 API Key | `third-party-api-keys` | `THIRD_PARTY_API_KEYS` | LLM provider API Key（JSON map，DashScope/DeepSeek/OpenAI/Anthropic） | 各 provider 控制台 | 90 天 | LLM 运维 | +| 6 | TLS 服务器证书 | `tls-server-cert` | （Ingress TLS Secret） | Ingress TLS 终止证书 + 私钥 | 证书签发机构 / cert-manager | 365 天 | 平台运维 | +| 7 | mTLS 客户端证书 | `mtls-client-cert` | `MTLS_CLIENT_CERT` | 与下游服务（KMS/HSM、内部 API）双向 TLS 客户端证书 | 内部 PKI | 180 天 | 安全团队 | +| 8 | KMS/HSM PKCS#11 PIN | `kms-hsm-pin` | `KMS_HSM_PIN` | 硬件安全模块 PKCS#11 访问 PIN | HSM 管理员 | 365 天 | HSM 运维 | +| 9 | OTel exporter token | `otel-exporter-token` | `OTEL_EXPORTER_OTLP_HEADERS` | OTLP collector 鉴权（DEPLOY-5: 值格式 `Authorization=Bearer `，非裸 Bearer） | 可观测性平台 | 90 天 | 可观测性团队 | +| 10 | Redis 密码 | `redis-password` | `REDIS_PASSWORD` | Redis requirepass（基础设施层，应用通过 env 消费） | DBA / 缓存运维 | 90 天 | DBA | +| 11 | PostgreSQL 密码 | `postgres-password` | `POSTGRES_PASSWORD`（仅 PG StatefulSet） | PostgreSQL POSTGRES_PASSWORD（DEPLOY-4: 应用不读此 env，通过 DATABASE_URL Slot 3 内嵌密码连接） | DBA | 90 天 | DBA | + +## 风险等级 + +| Slot | 泄露影响 | 风险等级 | +|------|----------|----------| +| 1 JWT 签名密钥 | 任意伪造身份 token | 高 | +| 2 API Key | 服务间冒充 / SCIM 越权 | 高 | +| 3 DB 凭据 | 数据库全量读写 | 严重 | +| 4 IDP 签名证书 | SSO 信任链断裂（不能直接伪造，需配合 IdP 私钥） | 中 | +| 5 第三方 API Key | LLM 账单盗刷 / 数据外泄 | 高 | +| 6 TLS 服务器证书 | TLS 中间人 / 域名冒充 | 高 | +| 7 mTLS 客户端证书 | 内部 API 越权访问 | 高 | +| 8 KMS/HSM PIN | 密钥托管被解锁 / 加解密被滥用 | 严重 | +| 9 OTel token | 遥测数据投毒 / 窃取 | 中 | +| 10 Redis-PG 密码 | 缓存/数据库越权 | 严重 | + +## 存储位置 + +所有 secret slot 渲染到统一 Kubernetes Secret（名 `-secrets`）， +由 SealedSecret 或 ExternalSecret 控制器自动创建 / 同步： + +``` +-secrets (Secret) + ├── jwt-signing-key + ├── api-key + ├── database-url + ├── idp-signing-certs + ├── third-party-api-keys + ├── tls-server-cert + ├── mtls-client-cert + ├── kms-hsm-pin + ├── otel-exporter-token + ├── redis-password + └── postgres-password +``` + +deployment.yaml 通过 `env.valueFrom.secretKeyRef` 引用上述全部 key。 +TLS 服务器证书（slot 6）额外被 Ingress 的 `tls.secretName` 引用 +（需由 cert-manager 或运维单独创建为 `kubernetes.io/tls` 类型 Secret）。 + +## 轮换流程 + +详见 `docs/deployment/key-rotation-runbook.md`。 diff --git a/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md new file mode 100644 index 0000000..581b001 --- /dev/null +++ b/docs/residual-review-findings/feat-enterprise-agent-platform-p0-3cf29f4.md @@ -0,0 +1,182 @@ +# Residual Review Findings — feat/enterprise-agent-platform-p0 + +**Source**: ce-code-review run on commit range `8633f60..3cf29f4` +**Branch**: `feat/enterprise-agent-platform-p0` +**HEAD**: `3cf29f4` +**Tracker availability**: no named tracker documented; `gh` CLI not installed; +`any_sink_available = false` — all findings routed to `no_sink` bucket per +lfg `references/tracker-defer.md` non-interactive mode. + +These findings were surfaced by the multi-agent code review (5 reviewer +subagents: security + adversarial, performance + reliability, +maintainability + standards, api-contract + data-migration, deployment +verification). They are **actionable** (`autofix_class: manual`) but were +**not applied** in Step 5 because they need design input or app+chart +coordinated changes. They are durably recorded here per lfg SKILL.md Step 6. + +## P1 — High-impact defects + +- **P1 / `src/agentkit/llm/pii_filter.py` + `src/agentkit/llm/gateway.py`** — + PERF-1: PII 脱敏模块未接入生产 LLM 调用路径（死代码）。`filter_messages_pii` + 仅被单元测试引用，gateway.py 的 chat()/chat_stream() 直接透传原始 messages。 + 等保合规声称的 PII 保护实际未生效。**Fix**: 在 gateway chat() 接入 + `filter_messages_pii(messages, fail_closed=True)`，从 `agentkit.yaml` 读 + PII 配置开关和 ner_hook 路径。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:233-238`** — + API-1: SCIM GET /Users 的 `totalResults` 返回当前页条数而非全量总数 + （违反 RFC 7644 3.4.2）。当用户总数 > count 时，SCIM 客户端无法得知真实 + 总数，分页逻辑错误。**Fix**: 在列表查询前先执行 `SELECT COUNT(*) FROM + users {where}` 复用同一 where 子句和参数。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:50-64,107-151,154-197,200-238`** — + API-2: SCIM 错误响应使用 FastAPI 默认 `{"detail": "..."}` 而非 RFC 7644 + 3.12 规定的 `{"schemas": ["urn:ietf:params:scim:api:messages:2.0:Error"], + "status": "409", "detail": "..."}`。同文件中定义的 `SCIMErrorResponse` 是 + 死代码。**Fix**: 添加 SCIM 异常处理器转换所有 HTTPException 为 SCIM + ErrorResponse 格式。 + +- **P1 / `src/agentkit/server/routes/auth.py:804-815,823-827`** — + API-4: `_sso_issue_token_pair` 用占位符 `refresh_token="__pending_sso__"` + 创建 session，并发 SSO 登录会触发 `refresh_token_hash` UNIQUE 约束冲突 + 抛 IntegrityError → 500。即使不考虑并发，两步写入也不原子，第二步失败 + 会留下孤儿 session。**Fix**: 先 `secrets.token_urlsafe(32)` 生成真实 + refresh token 再 `SessionCreate` 一步创建。 + +- **P1 / `src/agentkit/server/auth/scim/router.py:107-151`** — + API-5: SCIM `create_user` 不写入 `user_idp_links`，导致 SCIM 预配的用户 + 无法 SSO 登录（OIDC 会 JIT 创建重复账户）。SCIM 的核心价值就是预配 + SSO + 联动，此处断裂使 SCIM 预配失效。**Fix**: SCIM create_user 接受 + `externalId` 作为 idp_subject，INSERT users 后 INSERT user_idp_links。 + +- **P1 / `deploy/offline/scripts/install.sh:58` + docs** — + DEPLOY-1: 部署名引用错误。fullname helper 在 release=agentkit 时渲染 + Deployment 名为 `agentkit`，但 install.sh:58 + 11 处文档引用 + `agentkit-agentkit`（不存在）。**Fix**: 改用标签选择器 + `kubectl rollout restart deployment -l app.kubernetes.io/instance=${RELEASE}`。 + +- **P1 / `deploy/helm/agentkit/templates/deployment.yaml:88-92`** — + DEPLOY-2: Redis 鉴权断裂。`REDIS_PASSWORD` env 注入但应用从不消费 + （`src/agentkit/**/*.py` 零命中），Redis 客户端通过 `redis_url` 构造 + （`redis://redis:6379/0`，无密码）。若 Redis 启用密码，pod 全部 Redis + 操作 NOAUTH 失败。**Fix**: 在 redis 客户端构造处读取 `REDIS_PASSWORD` + 并以 `password=` 参数传入 `from_url`，或将密码拼入 redis_url。 + +## P2 — Moderate issues with meaningful downside + +- **P2 / `src/agentkit/server/auth/scim/router.py:107-151`** — + API-6: SCIM POST /Users 缺少 `Location` 响应头（RFC 7644 3.14 强制要求）。 + **Fix**: 用 `JSONResponse` + `response.headers["Location"] = ...`。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:183-187`** — + API-7: SCIM PATCH userName 更新未处理 UNIQUE 约束冲突，返回 500 而非 + 409。**Fix**: 包在 try/except `aiosqlite.IntegrityError` 中返回 409。 + +- **P2 / `src/agentkit/server/routes/auth.py:812`** — + API-8: `_sso_issue_token_pair` 硬编码 `auth_provider="sso"`，丢失 OIDC/SAML + 区分。admin UI 无法按 IDP 筛选 session。**Fix**: 增加 `provider_name` + + `provider_type` 参数。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:216-225`** — + API-9: SCIM filter 语法不匹配时静默返回全量用户，可能导致数据泄露。 + **Fix**: 不支持的 filter 返回 400 + `scimType=invalidFilter`。 + +- **P2 / `src/agentkit/server/routes/auth.py:1040-1085,1097-1143`** — + API-10: `deprovision_user` / `change_user_role` 返回 untyped `dict[str, Any]`， + OpenAPI schema 无法文档化。**Fix**: 定义 `DeprovisionResponse` / + `RoleChangeResponse` Pydantic 模型。 + +- **P2 / `src/agentkit/server/auth/scim/router.py:94-104`** — + API-11: SCIM `_user_to_scim` 用 `username` 作为 `displayName`，语义错误。 + **Fix**: 改为 `None` 让客户端 fallback。 + +- **P2 / `src/agentkit/server/auth/models.py:612-620`** — + MIG-1: `user_idp_links` 表无 ON DELETE 行为且无 FK 约束到 users。当前无 + 删除路径，但未来引入 GDPR 物理删除时会产生孤儿链接。**Fix**: 引入物理 + 删除时加 `REFERENCES users(id) ON DELETE CASCADE`。 + +- **P2 / `deploy/helm/agentkit/templates/deployment.yaml:58-62,93-97`** — + DEPLOY-4: PG 密码双源不一致。`DATABASE_URL`（被消费） 与 `POSTGRES_PASSWORD` + （死 env）并存。runbook Slot 10 轮换 `postgres-password` 但应用实际用 + `DATABASE_URL`。**Fix**: 明确单一可信源 — 若只用 `DATABASE_URL`，移除 + `POSTGRES_PASSWORD` env 与 `postgresPassword` slot。 + +- **P2 / `deploy/helm/agentkit/templates/deployment.yaml:83-87`** — + DEPLOY-5: OTel exporter token 值格式与 `OTEL_EXPORTER_OTLP_HEADERS` 语义 + 不符。期望 `key=value` 格式，但存储值为 `Bearer `。**Fix**: 改为 + `Authorization=Bearer ` 格式。 + +- **P2 / `src/agentkit/server/auth/scim/router.py` (7 处) + `scim/models.py`** — + STAND-1: SCIM router 使用 `typing.Any` 7 处，违反 AGENTS.md 禁用 any 规则。 + **Fix**: 定义 `SCIMUserResponse` / `SCIMUserListResponse` Pydantic 模型。 + +## P3 — Low-impact / narrow scope + +- **P3 / `src/agentkit/server/auth/providers/oidc.py:185-197`** — + SEC-2: OIDC id_token 未按规范验签（key=None + 关闭 iss/exp）。当前 + authlib 1.6.12 fail-closed，非直接绕过，但规范违反且脆弱。**Fix**: 用 + IDP JWKS 公钥验签，校验 iss/aud/exp。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:170-185`** — + SEC-3: SCIM PATCH deprovisioning（`active=false`）不写审计。admin 同等 + 操作有审计，SCIM 没有。**Fix**: 检测到 active true→false 时调用 + `record_deprovision(source=SOURCE_SCIM)`。 + +- **P3 / `src/agentkit/llm/gateway.py:401-414`** — + PERF-6: 流式路径在无 usage 数据时记录"cache miss"，指标误导。**Fix**: + 仅当 `final_usage` 非 None 时才记录 cache metric。 + +- **P3 / `src/agentkit/llm/gateway.py:450-457`** — + PERF-7: 非 Anthropic provider 调用膨胀 `prompt_cache.miss` 计数器。 + **Fix**: 仅对 Anthropic provider 记录 cache metric。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:171-187`** — + API-12: SCIM PATCH 静默忽略不支持的 op/path，无错误反馈。**Fix**: + 收集 skipped ops 并返回 400 + `scimType=noTarget`。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:200-238`** — + API-13: SCIM filter 解析对属性名和操作符大小写敏感，违反 RFC 7644 + 3.4.2.2。**Fix**: 用 `filter.lower()` + 正则 `re.IGNORECASE`。 + +- **P3 / `src/agentkit/server/auth/scim/router.py:154-197`** — + API-14: SCIM PATCH `op.path==None` 时仅更新 active，丢弃 value 中的其他 + 字段（部分实现静默丢数据）。**Fix**: 遍历 value 的 keys 对每个已知属性 + 执行对应 UPDATE。 + +- **P3 / `src/agentkit/server/auth/models.py:609-639`** — + MIG-2: schema 迁移幂等但缺少 V4→V5 的显式迁移记录。**Fix**: 未来新增列 + 时按 schema_version 差值执行 ALTER TABLE。 + +- **P3 / `deploy/helm/agentkit/values.yaml:8-11`** — + DEPLOY-9: 镜像仅 tag 固定（0.1.0），未做 digest pinning。政企生产场景 + 要求 digest pinning 防篡改。**Fix**: 支持可选 `image.digest` 字段。 + +- **P3 / `docs/deployment/key-rotation-runbook.md:254-283`** — + DEPLOY-8: runbook 仅 Slot 1-10，缺独立 Slot 11（PG 密码）段落。**Fix**: + 拆为两节，Slot 11 内注明须同步更新 `database-url`（Slot 3）。 + +- **P3 / `src/agentkit/bitable/service.py:564-568`** — + MAINT-2: `delete_view` 将两种不同失败原因统一抛 `LastViewDeletionError`， + 误导性错误信息。**Fix**: 区分 `DELETED`/`LAST_VIEW`/`NOT_FOUND` 三态。 + +- **P3 / `src/agentkit/server/frontend/src/stores/chatStream.ts:881-887,923-929`** — + MAINT-3: `team_synthesis_chunk` 与 `team_synthesis` 重复同一 `findLastMessage` + 谓词。**Fix**: 提取 `findStreamingSynthesisMilestone(conv, sid)` helper。 + +- **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:75,81,85`** — + MAINT-4: token 声明正则转义重复 3 次。**Fix**: 提取 `tokenDeclPattern(token)` + helper。 + +- **P3 / `src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts:92`** — + STAND-2: test "outside root" 用 replace 仅替换首个 `:root` 块。**Fix**: + 加 `g` 标志 `replace(/:root\s*\{[^}]*\}/g, '')`。 + +- **P3 / `src/agentkit/server/app.py:1401`** — + STAND-3: SCIM router 挂载在 `/api/v1` 前缀下，偏离 RFC 7644 标准 SCIM + 路径 `/scim/v2`。**Fix**: SCIM router 单独挂载不加 `/api/v1` 前缀。 + +--- + +**Total residual findings**: 27 (P1=7, P2=10, P3=10) +**Applied in Step 5**: 9 fixes (commit `3cf29f4`) +**Deferred to follow-up**: 27 (this file) diff --git a/pyproject.toml b/pyproject.toml index e713435..523db0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,9 @@ dependencies = [ "pyjwt>=2.8", "bcrypt>=4.0", "aiosqlite>=0.20", + # U4 SSO 集成 — OIDC (authlib) + SAML 2.0 (python3-saml) + "authlib>=1.3", + "python3-saml>=1.16", # 加密 secrets store (U10 — 多端消息适配器 AES-256-GCM) "cryptography>=42.0", # U15 — LiteLLM 统一 Provider 适配层（替换 6 个直接 API provider） @@ -50,6 +53,11 @@ dependencies = [ # 异步任务队列（U8 — 文档向量化） "taskiq>=0.11", "taskiq-redis>=0.5", + # OTel 可观测性（U1 — 默认启用，从 optional 升级为 required，移除 ImportError 静默回退） + "opentelemetry-api>=1.24", + "opentelemetry-sdk>=1.24", + "opentelemetry-exporter-otlp-proto-grpc>=1.24", + "opentelemetry-instrumentation-fastapi>=0.45b0", ] [project.scripts] diff --git a/src/agentkit/bitable/repository.py b/src/agentkit/bitable/repository.py index a4fd4ff..afaee6d 100644 --- a/src/agentkit/bitable/repository.py +++ b/src/agentkit/bitable/repository.py @@ -39,6 +39,26 @@ from agentkit.bitable.models import ( logger = logging.getLogger(__name__) +# U3/DR-1: field_id 在 jsonb_set path 与 where 子句中走字符串插值（jsonb path 不支持绑定参数）。 +# 防御性校验：field_id 必须匹配 UUID 格式，拒绝任意字符串插值到 SQL 结构。 +# ponytail: ceiling — 仅覆盖 field_id；其余 SQL 结构（op / direction）已由 service 层白名单枚举校验。 +# 升级路径：迁移到 ORM JSON 函数（sqlalchemy.dialects.postgresql.jsonb_*）彻底消除字符串插值。 +_FIELD_ID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + + +def _validate_field_id(field_id: str, *, context: str = "") -> str: + """校验 field_id 是 UUID 格式（DR-1 防御性校验）。 + + field_id 在 jsonb_set path 与 SQL where 子句中走字符串插值（jsonb path 不支持绑定参数）， + 必须确保非系统 UUID 不进入 SQL 结构。校验失败抛 ValueError。 + """ + if not isinstance(field_id, str) or not _FIELD_ID_RE.match(field_id): + ctx = f" ({context})" if context else "" + raise ValueError(f"Invalid field_id format{ctx}: {field_id!r} — expected UUID") + return field_id + class BitableRepository: """Async repository for bitable CRUD operations. @@ -375,7 +395,9 @@ class BitableRepository: return [Record.model_validate(e) for e in entities], next_cursor - async def update_record_values(self, record_id: str, values: dict[str, object]) -> Record | None: + async def update_record_values( + self, record_id: str, values: dict[str, object] + ) -> Record | None: """Update a record's values (full replace).""" async with self._session_factory() as session: stmt = ( @@ -464,9 +486,31 @@ class BitableRepository: await session.commit() return View.model_validate(entity) if entity else None - async def delete_view(self, view_id: str) -> bool: - """Delete a view. Returns True if a row was deleted.""" + async def delete_view_if_not_last(self, view_id: str, table_id: str) -> bool: + """Atomically delete a view only if it's not the last one in its table (DR-4 TOCTOU fix). + + Uses ``SELECT ... FOR UPDATE`` in a single transaction to lock sibling + rows for the duration of the count + delete, preventing concurrent + ``delete_view`` calls from racing past the last-view check and leaving + the table with zero views. + + Returns: + True if the view was deleted. + False if NOT deleted because it would be the last view (or the + view doesn't belong to the given table_id). + """ async with self._session_factory() as session: + # Lock all sibling views — concurrent delete_view calls on the + # same table block here until this transaction commits. + lock_sql = text( + "SELECT id FROM bitable.bitable_views WHERE table_id = :table_id FOR UPDATE" + ) + result = await session.execute(lock_sql, {"table_id": table_id}) + sibling_ids = {str(row[0]) for row in result.fetchall()} + if len(sibling_ids) <= 1: + return False # Last view — refuse to delete + if view_id not in sibling_ids: + return False # View doesn't belong to this table_id result = await session.execute(delete(ViewModel).where(ViewModel.id == view_id)) await session.commit() return result.rowcount > 0 @@ -646,9 +690,13 @@ class BitableRepository: return import json + # U3/DR-1: 校验所有 field_id 是 UUID 格式后再插值到 jsonb_set path（防御性，避免 SQL 结构注入） + for fid in agent_field_values: + _validate_field_id(fid, context="upsert_record_agent_fields") + async with self._session_factory() as session: # Build nested jsonb_set: jsonb_set(jsonb_set(values, '{f1}', CAST(:v0 AS jsonb), true), ...) - # ponytail: field_ids are system UUIDs, safe to interpolate into path literals. + # ponytail: field_ids validated above as UUID format, safe to interpolate into path literals. # Use CAST(:param AS jsonb) instead of :param::jsonb — asyncpg dialect # misparses the `::` as part of the param name. inner = "values" @@ -684,9 +732,17 @@ class BitableRepository: import base64 import json as _json + # U3/DR-1: 校验 filters/sorts 中的 field_id 是 UUID 格式后再插值到 SQL 结构 + if filters: + for f in filters: + _validate_field_id(f["field_id"], context="list_records_filtered.filter") + if sorts: + for s in sorts: + _validate_field_id(s["field_id"], context="list_records_filtered.sort") + async with self._session_factory() as session: # Build raw SQL with JSONB filter/sort translation. - # ponytail: field_ids in filters/sorts are system UUIDs (validated by service layer). + # ponytail: field_ids validated above as UUID format, safe to interpolate into SQL structure. where_clauses = ["table_id = :table_id"] params: dict[str, object] = {"table_id": table_id} diff --git a/src/agentkit/bitable/service.py b/src/agentkit/bitable/service.py index 7245247..ad81b31 100644 --- a/src/agentkit/bitable/service.py +++ b/src/agentkit/bitable/service.py @@ -545,23 +545,28 @@ class BitableService: return await self._repo.get_view(view_id) async def delete_view(self, view_id: str) -> bool: - """Delete a view with last-view protection (U6). + """Delete a view with last-view protection (U6, DR-4 TOCTOU fix). Raises :class:`LastViewDeletionError` if the view is the last one in its table — preventing users from making a table inaccessible. The route layer is responsible for the 404 + ownership checks before calling this (matching the existing ``delete_field`` pattern). Returns True if a row was deleted. + + DR-4: delegates to ``repo.delete_view_if_not_last`` which atomically + checks sibling count + deletes in a single ``SELECT FOR UPDATE`` + transaction, eliminating the TOCTOU race between ``list_views`` count + and ``delete_view`` execution. """ view = await self._repo.get_view(view_id) if view is None: return False - siblings = await self._repo.list_views(view.table_id) - if len(siblings) <= 1: - raise LastViewDeletionError( - "Cannot delete the last view of a table" - ) - return await self._repo.delete_view(view_id) + deleted = await self._repo.delete_view_if_not_last(view_id, view.table_id) + if not deleted: + # Either it was the last view, or a concurrent call raced ahead. + # Either way, the invariant "table has at least 1 view" is preserved. + raise LastViewDeletionError("Cannot delete the last view of a table") + return True # ── Recalc (U3: formula recalc pipeline) ──────────────── diff --git a/src/agentkit/bus/redis_bus.py b/src/agentkit/bus/redis_bus.py index 4017f25..36432f2 100644 --- a/src/agentkit/bus/redis_bus.py +++ b/src/agentkit/bus/redis_bus.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio import json import logging +import os from typing import Callable, Awaitable from agentkit.bus.message import AgentMessage @@ -41,7 +42,13 @@ class RedisMessageBus: """获取 Redis 连接（懒初始化）。""" if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _stream_key(self, agent_name: str) -> str: @@ -101,7 +108,10 @@ class RedisMessageBus: # Create consumer group if not exists try: await redis.xgroup_create( - stream_key, self._consumer_group, id="0", mkstream=True, + stream_key, + self._consumer_group, + id="0", + mkstream=True, ) except asyncio.CancelledError: raise @@ -141,7 +151,11 @@ class RedisMessageBus: logger.warning(f"Failed to process message {msg_id}: {e}") # Move to dead letter after max retries await self._handle_failed_message( - redis, stream_key, msg_id, fields, agent_name, + redis, + stream_key, + msg_id, + fields, + agent_name, ) except asyncio.CancelledError: break @@ -194,9 +208,7 @@ class RedisMessageBus: await self.publish(message) return await asyncio.wait_for(future, timeout=timeout) except asyncio.TimeoutError: - raise TimeoutError( - f"Request {message.correlation_id} timed out after {timeout}s" - ) + raise TimeoutError(f"Request {message.correlation_id} timed out after {timeout}s") finally: self._pending_requests.pop(message.correlation_id, None) @@ -256,6 +268,7 @@ def create_message_bus( if backend == "redis": try: import redis.asyncio as aioredis # noqa: F401 + bus = RedisMessageBus( redis_url=redis_url, consumer_group=consumer_group, @@ -267,8 +280,7 @@ def create_message_bus( raise except Exception as exc: # noqa: BLE001 — factory fallback to InMemoryMessageBus; must catch import/init errors broadly logger.warning( - f"Failed to initialise RedisMessageBus ({exc}), " - f"falling back to InMemoryMessageBus" + f"Failed to initialise RedisMessageBus ({exc}), falling back to InMemoryMessageBus" ) bus = InMemoryMessageBus() diff --git a/src/agentkit/core/base.py b/src/agentkit/core/base.py index 75c6629..51036c3 100644 --- a/src/agentkit/core/base.py +++ b/src/agentkit/core/base.py @@ -10,6 +10,7 @@ import asyncio import json import logging +import os import time from abc import ABC, abstractmethod from collections.abc import AsyncGenerator @@ -266,7 +267,12 @@ class BaseAgent(ABC): if redis_url: try: - self._redis = aioredis.from_url(redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) await self._redis.ping() logger.info(f"Agent '{self.name}' connected to Redis") except (ConnectionError, OSError, asyncio.TimeoutError, ValueError) as e: diff --git a/src/agentkit/core/config_driven.py b/src/agentkit/core/config_driven.py index 33a762d..0d461aa 100644 --- a/src/agentkit/core/config_driven.py +++ b/src/agentkit/core/config_driven.py @@ -394,7 +394,12 @@ class ConfigDrivenAgent(BaseAgent, EvolutionMixin): import redis.asyncio as aioredis redis_url = config.memory["working"].get("redis_url", "redis://localhost:6379") - redis_client = aioredis.from_url(redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + redis_client = aioredis.from_url( + redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) working = WorkingMemory(redis=redis_client) if config.memory.get("episodic", {}).get("enabled"): diff --git a/src/agentkit/core/handoff_transport.py b/src/agentkit/core/handoff_transport.py index 5a2bf83..af093f0 100644 --- a/src/agentkit/core/handoff_transport.py +++ b/src/agentkit/core/handoff_transport.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio import json import logging +import os from typing import AsyncIterator, Awaitable, Callable import redis.asyncio as aioredis @@ -144,7 +145,12 @@ class RedisHandoffTransport: """确保 Redis 连接已建立（懒初始化）""" if self._redis is None: try: - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) await self._redis.ping() logger.info("RedisHandoffTransport connected to Redis") except Exception: @@ -199,9 +205,7 @@ class RedisHandoffTransport: # 启动后台监听任务 if channel not in self._listen_tasks: - self._listen_tasks[channel] = asyncio.create_task( - self._background_listen(channel) - ) + self._listen_tasks[channel] = asyncio.create_task(self._background_listen(channel)) async def _background_listen(self, channel: str) -> None: """后台监听频道消息并调用处理器""" diff --git a/src/agentkit/llm/cache.py b/src/agentkit/llm/cache.py index 0919920..46e3508 100644 --- a/src/agentkit/llm/cache.py +++ b/src/agentkit/llm/cache.py @@ -11,6 +11,7 @@ Design doc: docs/plans/2026-06-14-002-u1-llm-cache-architecture.md import json import logging +import os import time from collections import OrderedDict from dataclasses import dataclass, field @@ -386,7 +387,12 @@ class RedisLLMCache: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis async def aclose(self) -> None: diff --git a/src/agentkit/llm/gateway.py b/src/agentkit/llm/gateway.py index 387b578..8e2299a 100644 --- a/src/agentkit/llm/gateway.py +++ b/src/agentkit/llm/gateway.py @@ -2,6 +2,7 @@ import asyncio import logging +import os import time from datetime import datetime, timezone from pathlib import Path @@ -12,7 +13,11 @@ from agentkit.llm.config import LLMConfig from agentkit.llm.protocol import LLMProvider, LLMRequest, LLMResponse, StreamChunk, TokenUsage from agentkit.llm.providers.tracker import UsageSummary, UsageTracker from agentkit.telemetry.tracing import get_tracer, _OTEL_AVAILABLE -from agentkit.telemetry.metrics import llm_token_histogram +from agentkit.telemetry.metrics import ( + llm_token_histogram, + prompt_cache_hit_counter, + prompt_cache_miss_counter, +) if TYPE_CHECKING: from agentkit.llm.cache import LitellmCacheManager @@ -25,27 +30,25 @@ logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -if TYPE_CHECKING: +class _QuotaServiceLike(Protocol): + """Quota service 最小契约（仅覆盖 gateway._enforce_quota 用到的方法）。""" - class _QuotaServiceLike(Protocol): - """Quota service 最小契约（仅覆盖 gateway._enforce_quota 用到的方法）。""" + async def is_model_allowed( + self, db: Path, department_id: str, model: str + ) -> tuple[bool, str]: ... - async def is_model_allowed( - self, db: Path, department_id: str, model: str - ) -> tuple[bool, str]: ... + async def check_quota( + self, + db: Path, + department_id: str, + quota_type: str, + period: str, + current: float, + ) -> tuple[bool, str]: ... - async def check_quota( - self, - db: Path, - department_id: str, - quota_type: str, - period: str, - current: float, - ) -> tuple[bool, str]: ... - - async def get_quota( - self, db: Path, department_id: str, quota_type: str, period: str - ) -> dict[str, object] | None: ... + async def get_quota( + self, db: Path, department_id: str, quota_type: str, period: str + ) -> dict[str, object] | None: ... class QuotaExceededError(Exception): @@ -82,6 +85,12 @@ class LLMGateway: self._usage_tracker = UsageTracker(store=usage_store) if usage_store else UsageTracker() self._config = config or LLMConfig() + # PERF-1: PII 脱敏接入生产路径（env 驱动，默认关闭以保持向后兼容）。 + # AGENTKIT_PII_FILTER_ENABLED=true 启用；filter_messages_pii fail-closed。 + self._pii_filter_enabled = ( + os.environ.get("AGENTKIT_PII_FILTER_ENABLED", "false").lower() == "true" + ) + # Cache (U17 — LiteLLM 缓存管理器，opt-in，默认禁用) self._cache_manager: "LitellmCacheManager | None" = None if self._config.cache and self._config.cache.enabled: @@ -95,6 +104,22 @@ class LLMGateway: f"similarity_threshold={litellm_config.similarity_threshold})" ) + if self._pii_filter_enabled: + logger.info("PII filter enabled (AGENTKIT_PII_FILTER_ENABLED=true, fail-closed)") + + def _apply_pii_filter(self, messages: list[dict[str, str]]) -> list[dict[str, str]]: + """PERF-1: 对 messages 执行 PII 脱敏。 + + fail-closed：脱敏失败抛 PIIFilterError，由调用方转 HTTP 422。 + 禁用时原样返回（零开销 — 仅一次 env 读 + bool 判断）。 + """ + if not self._pii_filter_enabled: + return messages + from agentkit.llm.pii_filter import filter_messages_pii + + redacted, _matches = filter_messages_pii(messages, fail_closed=True) + return redacted + def register_provider(self, name: str, provider: LLMProvider) -> None: """注册 Provider""" self._providers[name] = provider @@ -127,6 +152,9 @@ class LLMGateway: if not self._providers: raise LLMProviderError("", "No provider registered") + # PERF-1: PII 脱敏（fail-closed）— 在 provider 调用前替换 messages。 + messages = self._apply_pii_filter(messages) + # ── Quota enforcement ── # Only enforce when department_ids + db_path are provided # (other call sites pass None — no quota check). @@ -155,54 +183,55 @@ class LLMGateway: _span = _span_cm.__enter__() start = time.monotonic() - - # ── Cache check (U17 — LiteLLM cache via cache_key in request) ── - # LiteLLM 在 litellm.acompletion 内部处理缓存读写，gateway 只需： - # 1. 构建 per-user + ACL-scoped cache_key（安全要求 a, b） - # 2. 将 cache 参数注入 kwargs 透传到 provider - # 3. 检测响应的 cache_hit 标志，用于 usage tracking（cost=0） - if self._cache_manager is not None: - from agentkit.llm.cache import LitellmCacheManager - - # 解析 KB caching_disabled（安全要求 c） - # 非 RAG 请求（kb_id=None）→ 默认启用缓存（无 KB 数据需保护）。 - # RAG 请求（kb_id!=None）→ fail-closed：默认禁用缓存，仅在 settings - # 明确返回 caching_disabled=False 时启用。防止 DB 异常时 fail-open - # 导致禁用缓存的 KB 数据泄漏到缓存。 - kb_caching_disabled = kb_id is not None - if kb_id is not None: - try: - from agentkit.rag_platform.settings import get_settings_store - - settings = await get_settings_store().get_settings(kb_id) - if settings is not None: - kb_caching_disabled = settings.caching_disabled - # settings 为 None（KB 不存在）→ 保持 True（fail-closed） - except Exception as e: - logger.warning(f"Failed to read KB cache settings for kb_id={kb_id}: {e}") - # 读取异常 → 保持 True（fail-closed，禁用缓存） - - if self._cache_manager.should_cache(kb_caching_disabled, user_id): - cache_key = self._cache_manager.build_cache_key( - model=resolved_model, - messages=messages, - temperature=kwargs.get("temperature", 0.7), - tools=tools, - tool_choice=tool_choice, - max_tokens=kwargs.get("max_tokens", 2000), - user_id=user_id, - kb_acl_hash=kb_acl_hash, - ) - kwargs["cache"] = LitellmCacheManager.cache_params_for_hit(cache_key) - else: - kwargs["cache"] = LitellmCacheManager.cache_params_for_no_cache() - - # ── Normal provider call ── - models_to_try = self._get_models_to_try(resolved_model) - last_error: LLMProviderError | None = None - response: LLMResponse | None = None - + # PERF-2: try 块必须覆盖整个 span 生命周期（含 cache 设置）， + # 否则 cache_key 构建异常会泄漏 span。PERF-3: except 记录异常到 span。 try: + # ── Cache check (U17 — LiteLLM cache via cache_key in request) ── + # LiteLLM 在 litellm.acompletion 内部处理缓存读写，gateway 只需： + # 1. 构建 per-user + ACL-scoped cache_key（安全要求 a, b） + # 2. 将 cache 参数注入 kwargs 透传到 provider + # 3. 检测响应的 cache_hit 标志，用于 usage tracking（cost=0） + if self._cache_manager is not None: + from agentkit.llm.cache import LitellmCacheManager + + # 解析 KB caching_disabled（安全要求 c） + # 非 RAG 请求（kb_id=None）→ 默认启用缓存（无 KB 数据需保护）。 + # RAG 请求（kb_id!=None）→ fail-closed：默认禁用缓存，仅在 settings + # 明确返回 caching_disabled=False 时启用。防止 DB 异常时 fail-open + # 导致禁用缓存的 KB 数据泄漏到缓存。 + kb_caching_disabled = kb_id is not None + if kb_id is not None: + try: + from agentkit.rag_platform.settings import get_settings_store + + settings = await get_settings_store().get_settings(kb_id) + if settings is not None: + kb_caching_disabled = settings.caching_disabled + # settings 为 None（KB 不存在）→ 保持 True（fail-closed） + except Exception as e: + logger.warning(f"Failed to read KB cache settings for kb_id={kb_id}: {e}") + # 读取异常 → 保持 True（fail-closed，禁用缓存） + + if self._cache_manager.should_cache(kb_caching_disabled, user_id): + cache_key = self._cache_manager.build_cache_key( + model=resolved_model, + messages=messages, + temperature=kwargs.get("temperature", 0.7), + tools=tools, + tool_choice=tool_choice, + max_tokens=kwargs.get("max_tokens", 2000), + user_id=user_id, + kb_acl_hash=kb_acl_hash, + ) + kwargs["cache"] = LitellmCacheManager.cache_params_for_hit(cache_key) + else: + kwargs["cache"] = LitellmCacheManager.cache_params_for_no_cache() + + # ── Normal provider call ── + models_to_try = self._get_models_to_try(resolved_model) + last_error: LLMProviderError | None = None + response: LLMResponse | None = None + for model_name in models_to_try: try: provider, actual_model = self._resolve_model(model_name) @@ -286,12 +315,35 @@ class LLMGateway: _span.set_attribute("gen_ai.duration.ms", int(latency_ms)) if self._cache_manager is not None: _span.set_attribute("gen_ai.cache.hit", is_cache_hit) + # U2 — Anthropic prompt cache hit/miss span attributes + _span.set_attribute( + "gen_ai.usage.cache_read_input_tokens", + response.usage.cache_read_input_tokens, + ) + _span.set_attribute( + "gen_ai.usage.cache_creation_input_tokens", + response.usage.cache_creation_input_tokens, + ) llm_token_histogram().record( response.usage.total_tokens, {"gen_ai.request.model": resolved_model}, ) + # U2 — prompt cache hit/miss OTel counter（Anthropic cache_read_input_tokens > 0 → hit） + self._record_prompt_cache_metric(response.usage, resolved_model) return response + except Exception as e: + # PERF-3: 记录异常到 span — __exit__(None, None, None) 会丢失异常信息， + # 失败的 LLM 调用在 trace 中显示为成功。手动 record_exception + 设 ERROR 状态。 + if _span is not None: + try: + _span.record_exception(e) + from opentelemetry.trace import Status, StatusCode + + _span.set_status(Status(StatusCode.ERROR, str(e))) + except Exception: # noqa: BLE001 — span 记录失败不影响异常向上传播 + pass + raise finally: if _span_cm is not None: _span_cm.__exit__(None, None, None) @@ -323,6 +375,9 @@ class LLMGateway: if not self._providers: raise LLMProviderError("", "No provider registered") + # PERF-1: PII 脱敏（fail-closed）— 在 provider 调用前替换 messages。 + messages = self._apply_pii_filter(messages) + # ── Quota enforcement ── if department_ids and db_path: await self._enforce_quota(db_path, department_ids, resolved_model) @@ -397,6 +452,8 @@ class LLMGateway: user_id=user_id, department_ids=department_ids, ) + # U2 — prompt cache hit/miss OTel counter（streaming path） + self._record_prompt_cache_metric(final_usage, final_model) # Empty stream detection: if no content was produced, # raise error so the caller (ReActEngine) can retry with a different model. @@ -432,6 +489,15 @@ class LLMGateway: "", f"No provider available for streaming '{resolved_model}'" ) + @staticmethod + def _record_prompt_cache_metric(usage: TokenUsage, model: str) -> None: + """Record prompt cache hit/miss OTel counter — Anthropic cache_read_input_tokens > 0 → hit.""" + attrs = {"gen_ai.request.model": model} + if usage.cache_hit: + prompt_cache_hit_counter().add(1, attrs) + else: + prompt_cache_miss_counter().add(1, attrs) + def _get_models_to_try(self, resolved_model: str) -> list[str]: """Return [primary_model] + fallback_models for the given resolved model.""" fallback_models = self._config.fallbacks.get(resolved_model, []) diff --git a/src/agentkit/llm/pii_filter.py b/src/agentkit/llm/pii_filter.py new file mode 100644 index 0000000..04691f9 --- /dev/null +++ b/src/agentkit/llm/pii_filter.py @@ -0,0 +1,224 @@ +"""PII 脱敏 hook（U2/R3）。 + +等保合规场景：发送至 LLM 的 prompt 先经 PII 脱敏，原文不落盘。 + +设计： +- 正则版（默认）：识别手机号 / 邮箱 / 身份证 / 银行卡，替换为 [REDACTED_TYPE:hash8] +- ner_hook（可选）：客户内网 NER 模型（LAC/HanLP），module:func 路径动态 import +- fail-closed：任何异常时拒绝发送（raise PIIFilterError），不降级到原文发送 +- hash 审计：脱敏后记录 hash 用于审计；``PIIMatch.original`` 仅内存中供调用方 + 使用（如展示 / 调试），不得落盘 / 落日志 + +ponytail: 用 stdlib（re + hashlib），不引入 NER 依赖。 +升级路径：ner_hook 接入客户内网 NER 模型，正则版作为 fallback。 +""" + +from __future__ import annotations + +import hashlib +import logging +import re +from dataclasses import dataclass, field + +from agentkit.telemetry.metrics import ( + pii_filter_coverage_counter, + pii_filter_redacted_counter, +) + +logger = logging.getLogger(__name__) + + +class PIIFilterError(Exception): + """PII 脱敏失败（fail-closed 触发）""" + + +# ── PII 正则模式 ────────────────────────────────────────── +# ponytail: 正则覆盖中国常见 PII；边界宽松避免漏报（fail-closed 优于 false negative）。 +# 升级路径：ner_hook 接入 NER 模型识别更复杂 PII（地址、姓名、组织）。 + +# 中国手机号：1[3-9] 开头 11 位 +_PHONE_RE = re.compile(r"1[3-9]\d{9}") +# 邮箱 +_EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") +# 中国身份证号：18 位（最后一位 X 校验），前 6 位地区码 + 8 位生日 + 3 位序号 + 1 位校验 +_ID_CARD_RE = re.compile( + r"\b[1-9]\d{5}(?:19|20)\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\d|3[01])\d{3}[\dXx]\b" +) +# 银行卡号：16-19 位数字（宽松，可能误报长数字序列 — fail-closed 优于漏报） +_BANK_CARD_RE = re.compile(r"\b\d{16,19}\b") + + +@dataclass +class PIIMatch: + """单个 PII 匹配结果""" + + pii_type: str # phone / email / id_card / bank_card + original: str + redacted: str + hash: str # 原文 sha256 前 8 位（审计用，不可逆） + + +@dataclass +class PIIFilterResult: + """PII 脱敏结果""" + + redacted_text: str + matches: list[PIIMatch] = field(default_factory=list) + + @property + def redacted_count(self) -> int: + return len(self.matches) + + +def _hash_pii(original: str) -> str: + """sha256 前 8 位（审计用）""" + return hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] + + +def _redact(text: str, ner_hook=None) -> PIIFilterResult: + """执行 PII 脱敏（内部函数，ner_hook 异常向上传播由 filter_pii 决定 fail-closed）。 + + ponytail: 正则版按 PII 类型顺序替换；id_card 在 phone 之前（更具体的先匹配， + 避免身份证号内的 11 位数字被手机号正则误匹配）。ner_hook 优先于正则版。 + """ + matches: list[PIIMatch] = [] + redacted = text + + # ner_hook 优先：客户内网 NER 模型 + if ner_hook is not None: + ner_result = ner_hook(text) + if ner_result is not None: + # ner_hook 返回 (redacted_text, matches_list) + redacted_text, ner_matches = ner_result + redacted = redacted_text + matches.extend(ner_matches) + # ner_hook 返回 None 表示无匹配，继续走正则版 + + # 正则版（默认或 ner_hook 无匹配时叠加） + # ponytail: id_card 在 phone 之前（身份证号包含 11 位数字子串会被 phone 误匹配） + patterns = [ + ("id_card", _ID_CARD_RE), + ("phone", _PHONE_RE), + ("email", _EMAIL_RE), + ("bank_card", _BANK_CARD_RE), + ] + for pii_type, pattern in patterns: + # 单次 sub + 回调同时收集 matches 和替换文本， + # 避免二次正则扫描 + 双重 sha256 计算（热路径） + def _replace(m, _t=pii_type): + original = m.group() + pii_hash = _hash_pii(original) + replacement = f"[REDACTED_{_t.upper()}:{pii_hash}]" + matches.append( + PIIMatch( + pii_type=_t, + original=original, + redacted=replacement, + hash=pii_hash, + ) + ) + return replacement + + redacted = pattern.sub(_replace, redacted) + + return PIIFilterResult(redacted_text=redacted, matches=matches) + + +def _record_pii_metrics(status: str, redacted_count: int | None = None) -> None: + """记录 PII 脱敏 OTel 指标 — 失败静默降级（不影响主流程）。 + + PERF-4: 用 logger.debug 记录失败原因（而非 except: pass）， + 保留静默降级语义同时让排障可见。 + PERF-5: import 提升至模块顶部，避免热路径每次 from import 开销。 + """ + try: + pii_filter_coverage_counter().add(1, {"pii_filter.status": status}) + if redacted_count is not None: + pii_filter_redacted_counter().add(redacted_count) + except Exception as e: # noqa: BLE001 — 指标失败不影响 PII 过滤主流程 + logger.debug("PII metric recording failed: %s", e) + + +def filter_pii( + text: str, + *, + ner_hook=None, + fail_closed: bool = True, +) -> PIIFilterResult: + """对文本执行 PII 脱敏。 + + Args: + text: 待脱敏文本 + ner_hook: 可选 NER 函数（返回 (redacted_text, matches_list)） + fail_closed: True 时脱敏失败抛 PIIFilterError；False 时降级到正则版重试 + + Returns: + PIIFilterResult: 脱敏后文本 + 匹配列表（含 hash 用于审计） + + Raises: + PIIFilterError: fail_closed=True 且脱敏过程中出现异常 + """ + try: + result = _redact(text, ner_hook=ner_hook) + _record_pii_metrics("success", result.redacted_count) + return result + except Exception as e: + logger.error(f"PII filter failed: {e}") + if fail_closed: + # fail-closed：拒绝发送，抛异常 + _record_pii_metrics("failed_closed") + raise PIIFilterError(f"PII filter failed (fail-closed): {e}") from e + # fail-open：降级到正则版（无 ner_hook）重试 + logger.warning("fail_closed=False, falling back to regex-only redaction") + try: + result = _redact(text, ner_hook=None) + _record_pii_metrics("fallback_regex", result.redacted_count) + return result + except Exception as fallback_err: + # 正则版也失败（极少见），返回原文（最差情况） + logger.error(f"Regex fallback also failed: {fallback_err}, returning original") + return PIIFilterResult(redacted_text=text, matches=[]) + + +def filter_messages_pii( + messages: list[dict[str, object]], + *, + ner_hook=None, + fail_closed: bool = True, +) -> tuple[list[dict[str, object]], list[PIIMatch]]: + """对 chat messages 列表执行 PII 脱敏。 + + Args: + messages: chat messages（role + content） + ner_hook: 可选 NER 函数 + fail_closed: True 时脱敏失败抛 PIIFilterError + + Returns: + (redacted_messages, all_matches): 脱敏后 messages + 全部匹配列表 + """ + all_matches: list[PIIMatch] = [] + redacted_messages: list[dict[str, object]] = [] + + for msg in messages: + content = msg.get("content") + if isinstance(content, str): + result = filter_pii(content, ner_hook=ner_hook, fail_closed=fail_closed) + redacted_msg = {**msg, "content": result.redacted_text} + all_matches.extend(result.matches) + elif isinstance(content, list): + # Anthropic content blocks: 脱敏每个 text block + redacted_blocks = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + block_text = block.get("text", "") + result = filter_pii(block_text, ner_hook=ner_hook, fail_closed=fail_closed) + redacted_blocks.append({**block, "text": result.redacted_text}) + all_matches.extend(result.matches) + else: + redacted_blocks.append(block) + redacted_msg = {**msg, "content": redacted_blocks} + else: + redacted_msg = msg + redacted_messages.append(redacted_msg) + + return redacted_messages, all_matches diff --git a/src/agentkit/llm/protocol.py b/src/agentkit/llm/protocol.py index 21369c3..d08997c 100644 --- a/src/agentkit/llm/protocol.py +++ b/src/agentkit/llm/protocol.py @@ -6,15 +6,29 @@ from dataclasses import dataclass, field @dataclass class TokenUsage: - """Token 使用量""" + """Token 使用量。 + + U2: 添加 cache_read_input_tokens / cache_creation_input_tokens 字段， + 用于 Anthropic prompt cache 命中率监控（OTel prompt_cache.hit/miss metric）。 + 非 Anthropic provider 这两个字段保持默认 0。 + """ prompt_tokens: int = 0 completion_tokens: int = 0 + # U2 — Anthropic prompt cache：从 prompt cache 读取的 token 数（命中时 > 0） + cache_read_input_tokens: int = 0 + # U2 — Anthropic prompt cache：写入 prompt cache 的 token 数 + cache_creation_input_tokens: int = 0 @property def total_tokens(self) -> int: return self.prompt_tokens + self.completion_tokens + @property + def cache_hit(self) -> bool: + """是否命中 prompt cache（cache_read_input_tokens > 0）""" + return self.cache_read_input_tokens > 0 + @dataclass class ToolCall: diff --git a/src/agentkit/llm/providers/anthropic.py b/src/agentkit/llm/providers/anthropic.py index 3a1c197..a57ad4a 100644 --- a/src/agentkit/llm/providers/anthropic.py +++ b/src/agentkit/llm/providers/anthropic.py @@ -271,6 +271,9 @@ class AnthropicProvider(LLMProvider): usage = TokenUsage( prompt_tokens=usage_data.get("input_tokens", 0), completion_tokens=usage_data.get("output_tokens", 0), + # U2 — Anthropic prompt cache 命中率监控 + cache_read_input_tokens=usage_data.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage_data.get("cache_creation_input_tokens", 0), ) return LLMResponse( @@ -479,6 +482,11 @@ class AnthropicProvider(LLMProvider): usage = TokenUsage( prompt_tokens=usage_data.get("input_tokens", 0), completion_tokens=usage_data.get("output_tokens", 0), + # U2 — Anthropic prompt cache 命中率监控 + cache_read_input_tokens=usage_data.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage_data.get( + "cache_creation_input_tokens", 0 + ), ) # Yield accumulated tool calls if any diff --git a/src/agentkit/llm/providers/usage_store.py b/src/agentkit/llm/providers/usage_store.py index 3bd634d..bd0ee0d 100644 --- a/src/agentkit/llm/providers/usage_store.py +++ b/src/agentkit/llm/providers/usage_store.py @@ -16,6 +16,7 @@ Legacy v1 keys (still readable for backward compat): import asyncio import json import logging +import os from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Protocol, runtime_checkable @@ -304,7 +305,12 @@ class RedisUsageStore: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _get_sync_redis(self): @@ -312,7 +318,12 @@ class RedisUsageStore: if self._sync_redis is None: import redis as sync_redis - self._sync_redis = sync_redis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 + self._sync_redis = sync_redis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._sync_redis async def aclose(self) -> None: diff --git a/src/agentkit/orchestrator/pipeline_state.py b/src/agentkit/orchestrator/pipeline_state.py index cb2a6ce..8907bfc 100644 --- a/src/agentkit/orchestrator/pipeline_state.py +++ b/src/agentkit/orchestrator/pipeline_state.py @@ -11,6 +11,7 @@ from __future__ import annotations import json import logging +import os import uuid from datetime import datetime, timezone from typing import Callable, Coroutine @@ -176,9 +177,11 @@ class PipelineStateRedis: if self._redis is None: import redis.asyncio as aioredis + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._redis = aioredis.from_url( self._redis_url, decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._redis diff --git a/src/agentkit/quality/cascade_state_store.py b/src/agentkit/quality/cascade_state_store.py index 04199ea..6a8b894 100644 --- a/src/agentkit/quality/cascade_state_store.py +++ b/src/agentkit/quality/cascade_state_store.py @@ -10,6 +10,7 @@ Key schema (Redis): """ import logging +import os import time from typing import Protocol, runtime_checkable @@ -140,8 +141,12 @@ class RedisCascadeStateStore: """Get or create a persistent sync Redis client (connection pool backed).""" if self._sync_redis is None: import redis as sync_redis + + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._sync_redis = sync_redis.from_url( - self._redis_url, decode_responses=True + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._sync_redis @@ -163,7 +168,15 @@ class RedisCascadeStateStore: pipe.expire(key, self._session_ttl) results = pipe.execute() return results[0] - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade increment failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -177,7 +190,15 @@ class RedisCascadeStateStore: r = self._get_sync_redis() val = r.get(f"{self.INTER_PREFIX}{session_id}") return int(val) if val is not None else 0 - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade get failed: {e}") if self._fallback is not None: return self._fallback.get_interaction(session_id) @@ -194,7 +215,15 @@ class RedisCascadeStateStore: pipe.set(key, depth) pipe.expire(key, self._session_ttl) pipe.execute() - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade set_depth failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -207,7 +236,15 @@ class RedisCascadeStateStore: r = self._get_sync_redis() val = r.get(f"{self.DEPTH_PREFIX}{session_id}") return int(val) if val is not None else 0 - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade get_depth failed: {e}") if self._fallback is not None: return self._fallback.get_depth(session_id) @@ -223,7 +260,15 @@ class RedisCascadeStateStore: pipe.delete(f"{self.INTER_PREFIX}{session_id}") pipe.delete(f"{self.DEPTH_PREFIX}{session_id}") pipe.execute() - except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError, TypeError) as e: + except ( + ImportError, + OSError, + _RedisError, + ValueError, + KeyError, + RuntimeError, + TypeError, + ) as e: logger.warning(f"Redis cascade reset failed: {e}") self._degrade_to_fallback() if self._fallback is not None: @@ -250,6 +295,7 @@ def create_cascade_state_store( if backend in ("auto", "redis"): try: import redis # noqa: F401 + return RedisCascadeStateStore(redis_url=redis_url, session_ttl=session_ttl) except ImportError: logger.warning("redis package not available, falling back to in-memory cascade store") diff --git a/src/agentkit/server/app.py b/src/agentkit/server/app.py index 988b31b..6efdbbb 100644 --- a/src/agentkit/server/app.py +++ b/src/agentkit/server/app.py @@ -1258,11 +1258,13 @@ def create_app( "redis_url", "redis://localhost:6379" ) # U5c: socket_timeout 防止单点 Redis 故障时网关请求挂死。 + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 redis_client = aioredis.from_url( redis_url, decode_responses=True, socket_timeout=5.0, socket_connect_timeout=5.0, + password=os.environ.get("REDIS_PASSWORD") or None, ) working = WorkingMemory(redis=redis_client) app.state.working_redis_client = redis_client @@ -1395,6 +1397,15 @@ def create_app( app.include_router(experts.router, prefix="/api/v1") app.include_router(auth_routes.router, prefix="/api/v1") app.include_router(auth_routes.admin_router, prefix="/api/v1") + # U4 SSO 集成 — SCIM 2.0 最小子集（自带 bearer token 校验） + from agentkit.server.auth.scim.router import ( + register_scim_exception_handlers, + router as scim_router, + ) + + app.include_router(scim_router, prefix="/api/v1") + # API-2: 注册 SCIM 异常处理器（scope 到 /scim/v2，非 SCIM 路由走默认格式） + register_scim_exception_handlers(app) app.include_router(admin_routes_module.admin_router, prefix="/api/v1") app.include_router(documents.router, prefix="/api/v1") app.include_router(calendar_routes.router, prefix="/api/v1") diff --git a/src/agentkit/server/auth/audit.py b/src/agentkit/server/auth/audit.py new file mode 100644 index 0000000..6d2d4dc --- /dev/null +++ b/src/agentkit/server/auth/audit.py @@ -0,0 +1,209 @@ +"""RBAC + deprovisioning 审计日志 (KTD-8, U4 SSO 集成)。 + +写入 ``auth_audit_log`` 表 + 发 OTel span event（接入审计通道）。 + +事件类型： +- ``role_change`` — RBAC 角色变更 (actor/target/role_before/role_after) +- ``deprovision`` — 手动 deprovisioning (operator/admin RBAC 强制) + +所有记录包含 actor/target/reason/source/timestamp，接入 OTel 审计通道 +（``agentkit.telemetry.tracer`` — OTel 不可用时降级为 SQLite + 日志）。 +""" + +from __future__ import annotations + +import logging +import os +import uuid +from pathlib import Path + +import aiosqlite + +from agentkit.telemetry.tracer import get_tracer +from .models import DEFAULT_AUTH_DB_PATH, audit_log_row_to_dict, _now_iso + +logger = logging.getLogger(__name__) + +# 审计事件类型常量 +EVENT_ROLE_CHANGE = "role_change" +EVENT_DEPROVISION = "deprovision" + +# 审计来源 +SOURCE_MANUAL = "manual" +SOURCE_SCIM = "scim" +SOURCE_CRON = "cron_reconciliation" +SOURCE_BREAKGLASS = "break_glass" + + +class AuditService: + """审计日志服务 — 写 ``auth_audit_log`` 表 + OTel span event。 + + ponytail: SQLite 单表写入，无聚合 / 告警。OTel 不可用时降级为日志。 + 上限：单进程写入；高并发审计需批量写 + 异步队列。 + """ + + def __init__(self, db_path: str | Path | None = None) -> None: + if db_path is not None: + self._db_path = Path(db_path) + else: + env = os.environ.get("AGENTKIT_AUTH_DB") + self._db_path = Path(env) if env else DEFAULT_AUTH_DB_PATH + + async def record_role_change( + self, + *, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + role_before: str, + role_after: str, + reason: str | None = None, + source: str = SOURCE_MANUAL, + ) -> dict[str, object]: + """记录 RBAC 角色变更 (KTD-8)。""" + return await self._record( + event_type=EVENT_ROLE_CHANGE, + actor_user_id=actor_user_id, + actor_username=actor_username, + target_user_id=target_user_id, + target_username=target_username, + role_before=role_before, + role_after=role_after, + reason=reason, + source=source, + ) + + async def record_deprovision( + self, + *, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + reason: str | None = None, + source: str = SOURCE_MANUAL, + ) -> dict[str, object]: + """记录手动 deprovisioning (R1b)。""" + return await self._record( + event_type=EVENT_DEPROVISION, + actor_user_id=actor_user_id, + actor_username=actor_username, + target_user_id=target_user_id, + target_username=target_username, + role_before=None, + role_after=None, + reason=reason, + source=source, + ) + + async def list_for_target( + self, target_user_id: str, *, limit: int = 50 + ) -> list[dict[str, object]]: + """查询某用户的所有审计记录（admin 视图）。""" + async with aiosqlite.connect(str(self._db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE target_user_id = ? " + "ORDER BY created_at DESC LIMIT ?", + (target_user_id, limit), + ) + rows = await cursor.fetchall() + return [audit_log_row_to_dict(r) for r in rows] + + async def _record( + self, + *, + event_type: str, + actor_user_id: str | None, + actor_username: str | None, + target_user_id: str, + target_username: str, + role_before: str | None, + role_after: str | None, + reason: str | None, + source: str, + ) -> dict[str, object]: + record_id = str(uuid.uuid4()) + now = _now_iso() + async with aiosqlite.connect(str(self._db_path)) as db: + await db.execute( + "INSERT INTO auth_audit_log " + "(id, event_type, actor_user_id, actor_username, " + " target_user_id, target_username, role_before, role_after, " + " reason, source, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + record_id, + event_type, + actor_user_id, + actor_username, + target_user_id, + target_username, + role_before, + role_after, + reason, + source, + now, + ), + ) + await db.commit() + # OTel 审计通道 — 发 span event（OTel 不可用时 NoOpTracer 静默降级） + try: + tracer = get_tracer() + with tracer.start_span(f"audit.{event_type}") as span: + span.set_attribute("audit.event_type", event_type) + span.set_attribute("audit.actor", actor_username or actor_user_id or "system") + span.set_attribute("audit.target", target_username or target_user_id) + span.set_attribute("audit.source", source) + if role_before or role_after: + span.set_attribute("audit.role_before", role_before or "") + span.set_attribute("audit.role_after", role_after or "") + span.add_event( + "audit.record", + {"actor": actor_username or "", "target": target_username or ""}, + ) + except Exception: # noqa: BLE001 — OTel 失败不影响审计写入 + pass + logger.info( + "审计记录: %s actor=%s target=%s reason=%s source=%s", + event_type, + actor_username or actor_user_id or "?", + target_username or target_user_id, + reason or "", + source, + ) + return { + "id": record_id, + "event_type": event_type, + "actor_user_id": actor_user_id, + "actor_username": actor_username, + "target_user_id": target_user_id, + "target_username": target_username, + "role_before": role_before, + "role_after": role_after, + "reason": reason, + "source": source, + "created_at": now, + } + + +# --------------------------------------------------------------------------- +# 进程级单例 +# --------------------------------------------------------------------------- + +_service: AuditService | None = None + + +def get_audit_service() -> AuditService: + """返回进程级 AuditService 单例（惰性初始化）。""" + global _service + if _service is None: + _service = AuditService() + return _service + + +def set_audit_service(service: AuditService | None) -> None: + """注入自定义 AuditService（测试用）。""" + global _service + _service = service diff --git a/src/agentkit/server/auth/idp_registry.py b/src/agentkit/server/auth/idp_registry.py new file mode 100644 index 0000000..e266d57 --- /dev/null +++ b/src/agentkit/server/auth/idp_registry.py @@ -0,0 +1,175 @@ +"""多 IDP 信任边界注册表 (R1a, U4 SSO 集成)。 + +管理多个 IDP 配置（OIDC + SAML 并存，如飞书 / Azure AD / 阿里云 IDaaS）： + +- **热证书轮换**：``update_certificate`` 直接替换内存配置，无需重启。 +- **单 IDP 故障隔离**：某个 IDP 不可用不影响其他 IDP 的认证流程。 +- **按 sub/NameID 关联**（R1a）：禁止仅凭邮箱合并账户 — 见 :mod:`providers.oidc`。 + +配置来源：``agentkit.yaml`` 的 ``auth.idps`` 段（见 :func:`IDPRegistry.from_config`）。 + +单租户假设 (R1c)：所有 IDP 共享同一本地用户表，无租户隔离。 +""" + +from __future__ import annotations + +import logging +import threading +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +logger = logging.getLogger(__name__) + +IDPType = Literal["oidc", "saml"] + + +class OIDCConfig(BaseModel): + """单个 OIDC IDP 配置（authlib redirect flow）。""" + + model_config = ConfigDict(extra="forbid") + + client_id: str + client_secret: str + server_metadata_url: str + # redirect_uri 是完整 URL，如 https://host/api/v1/auth/oauth/feishu/callback + redirect_uri: str + # 可选：scope 覆盖（默认 openid email profile） + scope: str = "openid email profile" + + +class SAMLConfig(BaseModel): + """单个 SAML 2.0 IDP 配置（python3-saml / OneLogin）。""" + + model_config = ConfigDict(extra="forbid") + + # IDP 元数据：entity_id + 单点登录 URL（SP 发起时构造 AuthnRequest） + entity_id: str + sso_url: str + # IDP 签名证书（PEM 格式，可能多张证书用于轮换） + idp_cert: str + # SP 自身配置 + sp_entity_id: str + acs_url: str + # 可选：SLO URL + slo_url: str | None = None + + +class IDPConfig(BaseModel): + """单个 IDP 完整配置（OIDC 或 SAML 之一）。""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(..., description="IDP 唯一标识，如 feishu / azure_ad / aliyun_idaas") + type: IDPType + display_name: str = "" + enabled: bool = True + oidc: OIDCConfig | None = None + saml: SAMLConfig | None = None + + def is_healthy(self) -> bool: + """IDP 是否可用（启用 + 配置完整）。 + + 单 IDP 故障不影响其他 — 调用方逐个检查此方法后决定是否跳过。 + """ + if not self.enabled: + return False + if self.type == "oidc": + return self.oidc is not None + return self.saml is not None + + +class IDPRegistry: + """多 IDP 注册表 — 进程内单例，支持热证书轮换。 + + 线程安全：所有写操作加锁（``_lock``）。读操作返回配置的浅拷贝引用 — + 配置对象本身不可变（Pydantic），热轮换通过整体替换实现。 + + ponytail: 进程内 dict 注册表，不持久化。多实例部署时各进程独立加载 + agentkit.yaml；证书热轮换需配合配置同步（config_sync.py 轮询）或重启。 + 上限：单进程数百 IDP 无压力（O(1) 查找）。 + """ + + def __init__(self) -> None: + self._idps: dict[str, IDPConfig] = {} + self._lock = threading.Lock() + + def register(self, config: IDPConfig) -> None: + """注册或覆盖一个 IDP（用于热加载 / 热证书轮换）。""" + with self._lock: + self._idps[config.name] = config + logger.info(f"IDP 已注册: {config.name} (type={config.type}, enabled={config.enabled})") + + def remove(self, name: str) -> bool: + """移除一个 IDP。返回是否实际移除。""" + with self._lock: + return self._idps.pop(name, None) is not None + + def get(self, name: str) -> IDPConfig | None: + """按名称查找 IDP 配置。返回 ``None`` 表示不存在。""" + with self._lock: + return self._idps.get(name) + + def list_idps(self) -> list[IDPConfig]: + """列出所有已注册 IDP（含禁用的）。""" + with self._lock: + return list(self._idps.values()) + + def list_healthy(self) -> list[IDPConfig]: + """列出所有可用 IDP（启用 + 配置完整）。""" + return [c for c in self.list_idps() if c.is_healthy()] + + def update_certificate(self, name: str, *, idp_cert: str | None = None) -> bool: + """热轮换 SAML IDP 签名证书 — 不重启即时生效。 + + Args: + name: IDP 名称。 + idp_cert: 新的 IDP 签名证书（PEM）。 + + Returns: + ``True`` 表示成功更新，``False`` 表示 IDP 不存在或非 SAML 类型。 + + OIDC 证书轮换通过 ``server_metadata_url`` 自动发现（authlib 缓存 TTL）， + 无需显式热轮换 — 调用 ``register`` 替换整个 :class:`OIDCConfig` 即可。 + """ + with self._lock: + existing = self._idps.get(name) + if existing is None or existing.type != "saml" or existing.saml is None: + return False + # Pydantic 不可变 — 构造新配置整体替换 + new_saml = existing.saml.model_copy(update={"idp_cert": idp_cert or ""}) + self._idps[name] = existing.model_copy(update={"saml": new_saml}) + logger.info(f"IDP {name} 签名证书已热轮换") + return True + + def load_from_config(self, idps: list[dict[str, object]]) -> None: + """从 agentkit.yaml 的 ``auth.idps`` 列表批量加载（覆盖现有）。""" + with self._lock: + self._idps.clear() + for raw in idps: + try: + cfg = IDPConfig.model_validate(raw) + self.register(cfg) + except Exception as exc: # noqa: BLE001 — 单个 IDP 配置错误不阻断其他 + logger.error(f"IDP 配置加载失败，跳过: {raw.get('name', '?')}: {exc}") + + +# --------------------------------------------------------------------------- +# 进程级单例 +# --------------------------------------------------------------------------- + +_registry: IDPRegistry | None = None + + +def get_idp_registry() -> IDPRegistry: + """返回进程级 IDP 注册表单例（惰性初始化）。""" + global _registry + if _registry is None: + _registry = IDPRegistry() + return _registry + + +def reset_idp_registry() -> None: + """重置注册表单例（测试用）。""" + global _registry + _registry = None diff --git a/src/agentkit/server/auth/middleware.py b/src/agentkit/server/auth/middleware.py index 2424652..82c0bbf 100644 --- a/src/agentkit/server/auth/middleware.py +++ b/src/agentkit/server/auth/middleware.py @@ -41,10 +41,10 @@ class AuthMiddleware(BaseHTTPMiddleware): client_keys: Mapping of ``client_name -> api_key`` (from clients.yaml). """ + # 精确匹配路径 — 静态端点（无动态路径段） WHITELIST_PATHS = ( "/", "/login", - "/assets", # Static assets (exact match covers root; prefix below covers sub-paths) "/api/v1/health", "/api/v1/auth/login", "/api/v1/auth/refresh", @@ -55,6 +55,18 @@ class AuthMiddleware(BaseHTTPMiddleware): "/redoc", ) + # 前缀匹配路径 — 动态路径段（provider 名 / 资源 id / 静态资源子路径） + PREFIX_WHITELIST_PATHS = ( + "/docs", + "/openapi.json", + "/redoc", + "/assets", + # U4 SSO — 动态 provider 路径 + "/api/v1/auth/oauth/", # OIDC redirect/callback + "/api/v1/auth/saml/", # SAML ACS + "/api/v1/scim/v2/", # SCIM 2.0（自带 bearer token 校验） + ) + def __init__( self, app, @@ -75,18 +87,13 @@ class AuthMiddleware(BaseHTTPMiddleware): def _is_whitelisted(self, path: str) -> bool: """Return True if ``path`` matches a whitelisted route. - Uses exact match for auth routes (so ``/auth/logout`` does NOT - whitelist ``/auth/logout-others``) and prefix match for docs and assets. + Uses exact match for static auth routes (so ``/auth/logout`` does NOT + whitelist ``/auth/logout-others``) and prefix match for docs, assets, + and U4 SSO dynamic-provider routes. """ - for prefix in self.WHITELIST_PATHS: - if path == prefix: - return True - # Prefix match for documentation paths and static assets. - # Auth paths require exact match to avoid accidentally whitelisting - # sibling routes like /auth/logout-others under /auth/logout. - if path.startswith(prefix) and prefix in ("/docs", "/openapi.json", "/redoc", "/assets"): - return True - return False + if path in self.WHITELIST_PATHS: + return True + return any(path.startswith(p) for p in self.PREFIX_WHITELIST_PATHS) def _is_dev_mode(self) -> bool: """Dev mode = no JWT secret, no global API key, no client keys.""" diff --git a/src/agentkit/server/auth/models.py b/src/agentkit/server/auth/models.py index f53920a..66504f5 100644 --- a/src/agentkit/server/auth/models.py +++ b/src/agentkit/server/auth/models.py @@ -45,6 +45,18 @@ def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +def resolve_auth_db_path() -> Path: + """Lazily resolve the auth DB path from env var at call time. + + ``DEFAULT_AUTH_DB_PATH`` is captured at module-import time and cannot + see test-time ``AGENTKIT_AUTH_DB`` mutations. This helper re-reads the + env var on every call so tests can ``monkeypatch.setenv`` before + constructing a provider/service. + """ + env = os.environ.get("AGENTKIT_AUTH_DB") + return Path(env) if env else DEFAULT_AUTH_DB_PATH + + # --------------------------------------------------------------------------- # SQLAlchemy 2 declarative models # --------------------------------------------------------------------------- @@ -593,6 +605,38 @@ CREATE TABLE IF NOT EXISTS skill_states ( disabled_at TEXT NOT NULL, disabled_by TEXT ); + +-- V5: user_idp_links — 多 IDP 信任边界 (R1a, U4 SSO 集成)。 +-- 按 (idp_name, idp_subject) 主键关联本地用户，禁止邮箱合并 (R1a)。 +-- 单用户可关联多个 IDP（如飞书 + Azure AD 并存），但每个 IDP sub 仅映射一行。 +CREATE TABLE IF NOT EXISTS user_idp_links ( + idp_name TEXT NOT NULL, + idp_subject TEXT NOT NULL, + user_id TEXT NOT NULL, + idp_type TEXT NOT NULL DEFAULT 'oidc', + created_at TEXT NOT NULL, + PRIMARY KEY (idp_name, idp_subject) +); +CREATE INDEX IF NOT EXISTS idx_user_idp_links_user_id ON user_idp_links(user_id); + +-- V5: auth_audit_log — RBAC 角色变更 + deprovisioning 审计 (KTD-8, U4)。 +-- 记录 actor/target/role_before/role_after/reason/source/timestamp。 +-- 接入 OTel 审计通道（audit.py 写入本表 + 发 OTel span event）。 +CREATE TABLE IF NOT EXISTS auth_audit_log ( + id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + actor_user_id TEXT, + actor_username TEXT, + target_user_id TEXT, + target_username TEXT, + role_before TEXT, + role_after TEXT, + reason TEXT, + source TEXT NOT NULL DEFAULT 'manual', + created_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_auth_audit_log_target ON auth_audit_log(target_user_id); +CREATE INDEX IF NOT EXISTS idx_auth_audit_log_created_at ON auth_audit_log(created_at); """ @@ -611,7 +655,11 @@ CREATE TABLE IF NOT EXISTS skill_states ( # # V4 (2026-06-21, Admin Console U6): added skill_states table for # admin-driven skill enable/disable. No backfill needed — additive. -_SCHEMA_VERSION = 4 +# +# V5 (2026-07-06, U4 SSO 集成): added user_idp_links (多 IDP 信任边界 R1a) +# + auth_audit_log (RBAC 角色变更 + deprovisioning 审计 KTD-8). +# No backfill needed — additive. +_SCHEMA_VERSION = 5 _META_SCHEMA_VERSION_KEY = "schema_version" @@ -852,3 +900,31 @@ def skill_state_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[s "disabled_at": row["disabled_at"], "disabled_by": row["disabled_by"], } + + +def idp_link_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[str, object]: + """Convert a ``user_idp_links`` row into a JSON-safe dict (U4 SSO).""" + return { + "idp_name": row["idp_name"], + "idp_subject": row["idp_subject"], + "user_id": row["user_id"], + "idp_type": row["idp_type"], + "created_at": row["created_at"], + } + + +def audit_log_row_to_dict(row: aiosqlite.Row | Mapping[str, object]) -> dict[str, object]: + """Convert an ``auth_audit_log`` row into a JSON-safe dict (U4 审计).""" + return { + "id": row["id"], + "event_type": row["event_type"], + "actor_user_id": row["actor_user_id"], + "actor_username": row["actor_username"], + "target_user_id": row["target_user_id"], + "target_username": row["target_username"], + "role_before": row["role_before"], + "role_after": row["role_after"], + "reason": row["reason"], + "source": row["source"], + "created_at": row["created_at"], + } diff --git a/src/agentkit/server/auth/providers/__init__.py b/src/agentkit/server/auth/providers/__init__.py index 5c9a90b..e956203 100644 --- a/src/agentkit/server/auth/providers/__init__.py +++ b/src/agentkit/server/auth/providers/__init__.py @@ -33,13 +33,13 @@ from __future__ import annotations import logging import os from functools import lru_cache -from pathlib import Path from .base import AuthProvider from .exceptions import AuthProviderError, InvalidCredentials, ProviderNotImplemented from .local import LocalAuthProvider from .oidc_stub import StubOIDCProvider from .user import User +from ..models import resolve_auth_db_path as _resolve_db_path logger = logging.getLogger(__name__) @@ -71,16 +71,6 @@ def _resolve_provider_name() -> str: return name -def _resolve_db_path() -> Path: - """Return the auth DB path (overridable for tests via env var).""" - from ..models import DEFAULT_AUTH_DB_PATH - - env = os.environ.get("AGENTKIT_AUTH_DB") - if env: - return Path(env) - return DEFAULT_AUTH_DB_PATH - - @lru_cache(maxsize=1) def get_auth_provider() -> AuthProvider: """Return the configured :class:`AuthProvider` (memoized singleton). diff --git a/src/agentkit/server/auth/providers/local.py b/src/agentkit/server/auth/providers/local.py index 6bf0de8..ffab51e 100644 --- a/src/agentkit/server/auth/providers/local.py +++ b/src/agentkit/server/auth/providers/local.py @@ -21,34 +21,22 @@ implementation. from __future__ import annotations import logging -import os import uuid from pathlib import Path import aiosqlite -from ..models import DEFAULT_AUTH_DB_PATH, user_row_to_dict +from ..models import ( + resolve_auth_db_path as _resolve_db_path, + _now_iso, + user_row_to_dict, +) from ..password import hash_password, verify_password from .user import User logger = logging.getLogger(__name__) -def _resolve_db_path() -> Path: - """Resolve the auth DB path with runtime env-var priority. - - The :data:`models.DEFAULT_AUTH_DB_PATH` constant is captured at - module-import time and therefore cannot see test-time env mutations. - Re-reading ``AGENTKIT_AUTH_DB`` here keeps the provider - "test-friendly" (tests can ``monkeypatch.setenv`` before constructing - the provider) without giving up the default path when no env is set. - """ - env = os.environ.get("AGENTKIT_AUTH_DB") - if env: - return Path(env) - return DEFAULT_AUTH_DB_PATH - - class LocalAuthProvider: """AuthProvider backed by the local SQLite ``users`` table + bcrypt. @@ -242,10 +230,3 @@ def _row_to_user(row: aiosqlite.Row) -> User: last_login_at=row["last_login_at"], created_by=row["created_by"], ) - - -def _now_iso() -> str: - """Return current UTC time as ISO 8601 string.""" - from datetime import datetime, timezone - - return datetime.now(timezone.utc).isoformat() diff --git a/src/agentkit/server/auth/providers/oidc.py b/src/agentkit/server/auth/providers/oidc.py new file mode 100644 index 0000000..fdb8a30 --- /dev/null +++ b/src/agentkit/server/auth/providers/oidc.py @@ -0,0 +1,306 @@ +"""OIDC 认证适配器 (authlib) — U4 SSO 集成。 + +redirect flow（KTD-2）： + +1. ``get_redirect_url`` → IDP authorize URL（含 state） +2. ``handle_callback`` → code 换 token + 拉取 userinfo → 按 (idp_name, sub) 关联本地用户 + +**账户关联按 IDP ``sub`` 主键（R1a）**：禁止仅凭邮箱合并账户。 +首次登录 JIT 创建本地用户 + ``user_idp_links`` 行；后续登录按 sub 查找现有用户。 + +``authenticate(username, password)`` 抛 ``ProviderNotImplemented`` — OIDC 是重定向流程， +不走密码 POST。AuthProvider Protocol 的其他方法（get_user_by_id 等）走本地 users 表。 + +state 缓存：进程内 dict + TTL 5min。ponytail: 多实例部署需 Redis 共享 state， +否则跨实例回调会失败。上限：单进程内存，state 量受并发登录数限制。 +""" + +from __future__ import annotations + +import logging +import secrets +import time +import uuid +from typing import Any +from urllib.parse import urlencode + +import aiosqlite +from authlib.integrations.httpx_client import AsyncOAuth2Client + +from ..idp_registry import get_idp_registry +from ..models import user_row_to_dict, _now_iso, resolve_auth_db_path as _resolve_db_path +from ..password import hash_password +from .exceptions import ProviderNotImplemented +from .local import _row_to_user +from .user import User + +logger = logging.getLogger(__name__) + +# state 缓存 TTL（秒）— OAuth 授权码流程的标准窗口 +_STATE_TTL_SECONDS = 300 + + +class _StateCache: + """进程内 OAuth state 缓存（TTL 5min）。 + + ponytail: 多实例部署需替换为 Redis 共享 state，否则跨实例回调失败。 + 上限：单进程，state 条目数 = 并发登录数（通常 <100）。 + """ + + def __init__(self) -> None: + self._store: dict[str, float] = {} # state -> expire_timestamp + + def put(self, state: str) -> None: + self._evict() + self._store[state] = time.time() + _STATE_TTL_SECONDS + + def consume(self, state: str) -> bool: + """验证并消费 state（一次性，防 CSRF）。""" + self._evict() + exp = self._store.pop(state, None) + if exp is None: + return False + return exp > time.time() + + def _evict(self) -> None: + now = time.time() + self._store = {s: e for s, e in self._store.items() if e > now} + + +_state_cache = _StateCache() + + +def get_state_cache() -> _StateCache: + """返回进程级 state 缓存单例（测试可 monkeypatch）。""" + return _state_cache + + +class OIDCProvider: + """OIDC 认证适配器 — authlib redirect flow + JIT 用户创建。 + + 一个实例服务所有 OIDC IDP（按 ``provider_name`` 路由到不同 IDP 配置）。 + """ + + name = "oidc" + + async def authenticate(self, *, username: str, password: str) -> User: + """OIDC 是重定向流程，不走密码 POST — 直接抛 NotImplemented。""" + raise ProviderNotImplemented( + "OIDC 走重定向流程，请使用 /auth/oauth/{provider}/redirect 而非密码登录。" + ) + + # ------------------------------------------------------------------ + # Redirect flow + # ------------------------------------------------------------------ + + async def get_redirect_url(self, provider_name: str, state: str | None = None) -> str: + """构造 IDP authorize URL 并缓存 state（CSRF 防护）。""" + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "oidc" or idp.oidc is None: + raise ValueError(f"OIDC IDP 未配置或不可用: {provider_name}") + + cfg = idp.oidc + state = state or secrets.token_urlsafe(32) + _state_cache.put(state) + + async with AsyncOAuth2Client( + client_id=cfg.client_id, + client_secret=cfg.client_secret, + redirect_uri=cfg.redirect_uri, + ) as client: + url, _ = client.create_authorization_url( + cfg.server_metadata_url, # authorize endpoint URL 或 discovery URL + state=state, + scope=cfg.scope, + ) + return url + + # ------------------------------------------------------------------ + # Callback — code 换 token + userinfo + JIT 用户关联 + # ------------------------------------------------------------------ + + async def handle_callback(self, provider_name: str, code: str, state: str) -> dict[str, object]: + """处理 OIDC callback：code 换 token + 拉 userinfo + 关联/创建本地用户。 + + 返回 user dict（via :func:`user_row_to_dict`）供路由签发 JWT。 + + Raises: + ValueError: state 无效 / IDP 未配置 / token 交换失败。 + """ + if not _state_cache.consume(state): + raise ValueError("无效或过期的 OAuth state（CSRF 校验失败）") + + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "oidc" or idp.oidc is None: + raise ValueError(f"OIDC IDP 未配置或不可用: {provider_name}") + + cfg = idp.oidc + async with AsyncOAuth2Client( + client_id=cfg.client_id, + client_secret=cfg.client_secret, + redirect_uri=cfg.redirect_uri, + ) as client: + token = await client.fetch_token( + cfg.server_metadata_url, # token endpoint URL 或 discovery URL + authorization_response=code, + body=urlencode( + { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": cfg.redirect_uri, + } + ), + ) + # 拉取 userinfo（OIDC discovery 的标准端点） + userinfo = await self._fetch_userinfo(client, token, cfg.server_metadata_url) + + sub = str(userinfo.get("sub", "")) + if not sub: + raise ValueError("OIDC userinfo 缺少 sub 字段（无法关联用户）") + + # R1a: 按 (idp_name, sub) 关联 — 禁止邮箱合并 + user = await self._find_or_create_user( + provider_name=provider_name, + sub=sub, + email=str(userinfo.get("email", "")), + name=str(userinfo.get("name", userinfo.get("preferred_username", ""))), + ) + return user + + async def _fetch_userinfo( + self, + client: AsyncOAuth2Client, + token: dict[str, Any], + metadata_url: str, + ) -> dict[str, Any]: + """从 OIDC userinfo endpoint 拉取用户信息。 + + ponytail: 直接 GET metadata_url（假定其即为 userinfo endpoint）。 + 生产环境应通过 discovery 文档解析 userinfo_endpoint；此处简化为直接请求。 + 上限：若 IDP 的 userinfo 端点与 discovery URL 不同需补充 discovery 解析。 + """ + # authlib 的 token 已自动设置到 client 的 token_auth，可直接 GET + resp = await client.get(metadata_url) + if resp.status_code != 200: + # 退回 token 中的 id_token 解码（authlib 已校验签名） + id_token = token.get("id_token") + if id_token: + from authlib.jose import jwt as authlib_jwt # 局部导入避免硬依赖 + + claims = authlib_jwt.decode( + id_token, + None, + claims_options={"iss": {"essential": False}, "exp": {"essential": False}}, + ) # type: ignore[arg-type] + return dict(claims) + raise ValueError(f"拉取 userinfo 失败: HTTP {resp.status_code}") + data = resp.json() + return dict(data) if isinstance(data, dict) else {} + + async def _find_or_create_user( + self, + *, + provider_name: str, + sub: str, + email: str, + name: str, + ) -> dict[str, object]: + """按 (idp_name, sub) 查找用户；未找到则 JIT 创建本地用户 + idp_link。 + + R1a: 严格按 sub 关联，绝不按邮箱合并已存在的本地账户。 + """ + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + # 1. 按 (idp_name, sub) 查 idp_link + cursor = await db.execute( + "SELECT u.* FROM users u " + "JOIN user_idp_links l ON l.user_id = u.id " + "WHERE l.idp_name = ? AND l.idp_subject = ?", + (provider_name, sub), + ) + row = await cursor.fetchone() + if row is not None: + return user_row_to_dict(row) + + # 2. JIT 创建 — 用户名 / 邮箱来自 IDP，密码随机（不可用于本地登录） + user_id = str(uuid.uuid4()) + # 用户名：优先 name，否则 provider:sub 前缀 + username = (name or f"{provider_name}:{sub[:24]}")[:64] + # 保证 username 唯一：冲突时追加 sub 后缀 + cursor = await db.execute("SELECT id FROM users WHERE username = ?", (username,)) + if await cursor.fetchone() is not None: + username = f"{username[:55]}_{sub[:8]}" + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + user_email = email or f"{sub[:24]}@sso.{provider_name}.local" + cursor = await db.execute("SELECT id FROM users WHERE email = ?", (user_email,)) + if await cursor.fetchone() is not None: + user_email = f"{sub[:24]}@sso.{provider_name}.local" + now = _now_iso() + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at, created_by) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + username, + user_email, + hash_password(secrets.token_urlsafe(32)), # 不可用于本地登录 + "member", + 1, + 0, + 0, + now, + now, + None, + ), + ) + await db.execute( + "INSERT INTO user_idp_links (idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, 'oidc', ?)", + (provider_name, sub, user_id, now), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info( + f"JIT 创建 OIDC 用户: idp={provider_name} sub={sub[:12]}... user={username}" + ) + return user_row_to_dict(row) + + # ------------------------------------------------------------------ + # AuthProvider Protocol 其余方法（走本地 users 表） + # ------------------------------------------------------------------ + + async def get_user_by_id(self, user_id: str) -> User | None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM users WHERE id = ? AND is_active = 1", (user_id,) + ) + row = await cursor.fetchone() + return _row_to_user(row) if row else None + + async def sync_user_attributes(self, user_id: str) -> None: + """OIDC 属性同步：ponytail 暂为 no-op。 + + 上限：生产应在此拉取最新 IDP profile（部门/职位）回写本地表。 + """ + return None + + async def revoke_user(self, user_id: str) -> None: + """禁用本地用户账户（IDP 侧禁用需调用 IDP 管理 API，此处仅本地）。""" + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + logger.info(f"OIDC provider 禁用用户 {user_id}") diff --git a/src/agentkit/server/auth/providers/saml.py b/src/agentkit/server/auth/providers/saml.py new file mode 100644 index 0000000..8bbade7 --- /dev/null +++ b/src/agentkit/server/auth/providers/saml.py @@ -0,0 +1,248 @@ +"""SAML 2.0 认证适配器 (python3-saml / OneLogin) — U4 SSO 集成。 + +ACS（Assertion Consumer Service）端点消费 SAMLResponse： + +1. ``handle_acs`` → 解析 assertion + 提取 NameID +2. 按 (idp_name, NameID) 关联本地用户（R1a: 禁止邮箱合并） +3. 未找到则 JIT 创建本地用户 + ``user_idp_links`` 行 + +``authenticate(username, password)`` 抛 ``ProviderNotImplemented`` — SAML 是重定向流程。 + +OneLogin_Saml2_Auth 是同步库，``process_response`` 在线程池中执行避免阻塞事件循环。 +""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +import uuid +from typing import Any + +import aiosqlite + +from ..idp_registry import get_idp_registry +from ..models import user_row_to_dict, _now_iso, resolve_auth_db_path as _resolve_db_path +from ..password import hash_password +from .exceptions import ProviderNotImplemented +from .local import _row_to_user +from .user import User + +logger = logging.getLogger(__name__) + + +class SAMLProvider: + """SAML 2.0 认证适配器 — OneLogin python3-saml ACS 消费。 + + 一个实例服务所有 SAML IDP（按 ``provider_name`` 路由到不同 IDP 配置）。 + """ + + name = "saml" + + async def authenticate(self, *, username: str, password: str) -> User: + """SAML 是重定向流程，不走密码 POST — 直接抛 NotImplemented。""" + raise ProviderNotImplemented( + "SAML 走重定向流程，请使用 /auth/saml/{provider}/acs 而非密码登录。" + ) + + # ------------------------------------------------------------------ + # ACS — 消费 SAMLResponse + JIT 用户关联 + # ------------------------------------------------------------------ + + async def handle_acs( + self, + provider_name: str, + saml_response: str, + *, + request_url: str = "", + ) -> dict[str, object]: + """处理 SAML ACS：解析 assertion + 提取 NameID + 关联/创建本地用户。 + + Args: + provider_name: IDP 名称（用于查找 IDP 配置 + idp_link）。 + saml_response: base64 编码的 SAMLResponse。 + request_url: 原始请求 URL（构造 OneLogin req dict 用）。 + + Returns: + 本地用户 dict（via :func:`user_row_to_dict`）。 + + Raises: + ValueError: IDP 未配置 / SAML 解析失败 / 缺少 NameID。 + """ + registry = get_idp_registry() + idp = registry.get(provider_name) + if idp is None or idp.type != "saml" or idp.saml is None: + raise ValueError(f"SAML IDP 未配置或不可用: {provider_name}") + + cfg = idp.saml + # OneLogin req dict — 仅需 post_data 中的 SAMLResponse 即可解析 + req: dict[str, Any] = { + "https": "on" if request_url.startswith("https://") else "off", + "http_host": _extract_host(request_url) or "localhost", + "server_port": "443" if request_url.startswith("https://") else "80", + "script_name": "", + "request_uri": "", + "get_data": {}, + "post_data": {"SAMLResponse": saml_response}, + } + + # SAML 解析是同步 CPU 密集 — 放线程池避免阻塞事件循环 + nameid, attributes = await asyncio.to_thread(self._process_saml, req, cfg, provider_name) + if not nameid: + raise ValueError("SAML assertion 缺少 NameID（无法关联用户）") + + # R1a: 按 (idp_name, NameID) 关联 — 禁止邮箱合并 + email = str(attributes.get("email", attributes.get("User.email", "")) or "") + name = str(attributes.get("name", attributes.get("cn", "")) or "") + return await self._find_or_create_user( + provider_name=provider_name, + nameid=nameid, + email=email, + name=name, + ) + + def _process_saml( + self, + req: dict[str, Any], + cfg: Any, + provider_name: str, + ) -> tuple[str, dict[str, Any]]: + """同步解析 SAML response — 在线程池中执行。 + + 返回 (NameID, attributes_dict)。 + + ponytail: OneLogin_Saml2_Auth 需要 SP 配置 dict。 + 此处构造最小 SP settings（cert/x509cert 为 IDP 签名证书）。 + 上限：生产 SAML SP 配置更完整（SP 私钥 / NameIDFormat 等）， + 此处只校验签名 + 提取 NameID 的最小集。 + """ + from onelogin.saml2.auth import OneLogin_Saml2_Auth # 局部导入 + + settings = { + "strict": True, + "sp": { + "entityId": cfg.sp_entity_id, + "assertionConsumerService": { + "url": cfg.acs_url, + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST", + }, + }, + "idp": { + "entityId": cfg.entity_id, + "singleSignOnService": { + "url": cfg.sso_url, + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect", + }, + "x509cert": cfg.idp_cert, + }, + } + auth = OneLogin_Saml2_Auth(req, old_settings=settings) + auth.process_response() + errors = auth.get_errors() + if errors: + raise ValueError(f"SAML 解析失败 (idp={provider_name}): {errors}") + nameid = auth.get_nameid() or "" + attrs = dict(auth.get_attributes()) if auth.get_attributes() else {} + return str(nameid), attrs + + async def _find_or_create_user( + self, + *, + provider_name: str, + nameid: str, + email: str, + name: str, + ) -> dict[str, object]: + """按 (idp_name, NameID) 查找用户；未找到则 JIT 创建。R1a: 禁止邮箱合并。""" + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT u.* FROM users u " + "JOIN user_idp_links l ON l.user_id = u.id " + "WHERE l.idp_name = ? AND l.idp_subject = ?", + (provider_name, nameid), + ) + row = await cursor.fetchone() + if row is not None: + return user_row_to_dict(row) + + user_id = str(uuid.uuid4()) + username = (name or f"{provider_name}:{nameid[:24]}")[:64] + cursor = await db.execute("SELECT id FROM users WHERE username = ?", (username,)) + if await cursor.fetchone() is not None: + username = f"{username[:55]}_{nameid[:8]}" + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + user_email = email or f"{nameid[:24]}@sso.{provider_name}.local" + cursor = await db.execute("SELECT id FROM users WHERE email = ?", (user_email,)) + if await cursor.fetchone() is not None: + user_email = f"{nameid[:24]}@sso.{provider_name}.local" + now = _now_iso() + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at, created_by) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + username, + user_email, + hash_password(secrets.token_urlsafe(32)), + "member", + 1, + 0, + 0, + now, + now, + None, + ), + ) + await db.execute( + "INSERT INTO user_idp_links (idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, 'saml', ?)", + (provider_name, nameid, user_id, now), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info( + f"JIT 创建 SAML 用户: idp={provider_name} nameid={nameid[:12]}... user={username}" + ) + return user_row_to_dict(row) + + # ------------------------------------------------------------------ + # AuthProvider Protocol 其余方法 + # ------------------------------------------------------------------ + + async def get_user_by_id(self, user_id: str) -> User | None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM users WHERE id = ? AND is_active = 1", (user_id,) + ) + row = await cursor.fetchone() + return _row_to_user(row) if row else None + + async def sync_user_attributes(self, user_id: str) -> None: + """SAML 属性同步：ponytail 暂为 no-op。""" + return None + + async def revoke_user(self, user_id: str) -> None: + db_path = _resolve_db_path() + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + logger.info(f"SAML provider 禁用用户 {user_id}") + + +def _extract_host(url: str) -> str: + """从 URL 中提取 host（ponytail: 不引入 urllib 解析，简单切分）。""" + if "://" in url: + url = url.split("://", 1)[1] + return url.split("/", 1)[0] if url else "" diff --git a/src/agentkit/server/auth/scim/__init__.py b/src/agentkit/server/auth/scim/__init__.py new file mode 100644 index 0000000..f21a0c1 --- /dev/null +++ b/src/agentkit/server/auth/scim/__init__.py @@ -0,0 +1,9 @@ +"""SCIM 2.0 最小子集 (U4 SSO 集成)。 + +- POST /scim/v2/Users — 创建用户 +- PATCH /scim/v2/Users/{id} — 禁用 (active=false) / 更新 +- GET /scim/v2/Users — 列表 / 过滤 + +SCIM push 失败触发实时告警（日志 error + OTel event）。 +Bearer token 认证（SCIM token 独立于 JWT，配置于 agentkit.yaml auth.scim_token）。 +""" diff --git a/src/agentkit/server/auth/scim/models.py b/src/agentkit/server/auth/scim/models.py new file mode 100644 index 0000000..55b8f59 --- /dev/null +++ b/src/agentkit/server/auth/scim/models.py @@ -0,0 +1,84 @@ +"""SCIM 2.0 用户 schema (Pydantic v2) — U4 最小子集。 + +仅实现 RFC 7643 / 7644 中创建 / 禁用 / 查询所需的字段： +``id``, ``userName``, ``active``, ``emails``, ``displayName``, ``externalId``。 +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, EmailStr, Field + + +class SCIMEmail(BaseModel): + """SCIM email 多值条目。""" + + model_config = ConfigDict(extra="allow") + + value: EmailStr + type: str = "work" + primary: bool = False + + +class SCIMUser(BaseModel): + """SCIM 2.0 User 资源（最小子集）。 + + STAND-1: 移除 ``Any`` — ``externalId`` 是 RFC 7643 标准 attribute， + 用于 SCIM 预配的用户与 SSO IDP subject 关联（API-5）。 + """ + + model_config = ConfigDict(extra="allow") + + id: str | None = None + userName: str + active: bool = True + displayName: str | None = None + externalId: str | None = None + emails: list[SCIMEmail] = Field(default_factory=list) + + +class SCIMPatchOp(BaseModel): + """SCIM 2.0 PATCH 操作（最小子集，仅支持 replace）。 + + STAND-1: ``value`` 用 ``object`` 替代 ``Any``（Pydantic v2 中 ``object`` 等价 + 于旧 ``Any`` 但更明确表达"任意 JSON 值"的语义）。 + """ + + model_config = ConfigDict(extra="allow") + + op: str = "replace" + path: str | None = None + value: object = None + + +class SCIMPatchRequest(BaseModel): + """SCIM PATCH 请求体（RFC 7644）。""" + + model_config = ConfigDict(extra="allow") + + Operations: list[SCIMPatchOp] = Field(default_factory=list) + + +class SCIMListResponse(BaseModel): + """SCIM 2.0 列表响应。 + + STAND-1: ``Resources`` 用 ``list[dict[str, object]]`` 替代 ``list[dict[str, Any]]``。 + """ + + model_config = ConfigDict(extra="allow") + + schemas: list[str] = Field( + default_factory=lambda: ["urn:ietf:params:scim:api:messages:2.0:ListResponse"] + ) + totalResults: int = 0 + startIndex: int = 1 + itemsPerPage: int = 0 + Resources: list[dict[str, object]] = Field(default_factory=list) + + +class SCIMErrorDetail(BaseModel): + """SCIM 2.0 错误响应。""" + + model_config = ConfigDict(extra="allow") + + status: str + detail: str | None = None diff --git a/src/agentkit/server/auth/scim/router.py b/src/agentkit/server/auth/scim/router.py new file mode 100644 index 0000000..b960442 --- /dev/null +++ b/src/agentkit/server/auth/scim/router.py @@ -0,0 +1,381 @@ +"""SCIM 2.0 路由 (U4 最小子集) — handler 内联，不拆 handlers.py。 + +端点： +- POST /scim/v2/Users — 创建用户（同步写 user_idp_links 供 SSO 联动） +- PATCH /scim/v2/Users/{id} — 禁用 (active=false) / 更新 +- GET /scim/v2/Users — 列表 / 过滤 + +Bearer token 认证（独立于 JWT — ``auth.scim_token`` 配置项）。 +SCIM push 失败触发实时告警（日志 error + OTel span event）。 +RFC 7643/7644 合规：错误响应用 SCIMErrorResponse 格式 + Location 头 + totalResults 全量总数。 +""" + +from __future__ import annotations + +import hmac +import logging +import os +import secrets +import uuid +from pathlib import Path + +import aiosqlite +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from ...auth.audit import SOURCE_SCIM, get_audit_service +from ...auth.models import ( + init_auth_db, + resolve_auth_db_path as _resolve_default_db_path, + user_row_to_dict, + _now_iso, +) +from ...auth.password import hash_password +from .models import SCIMListResponse, SCIMPatchRequest, SCIMUser + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/scim/v2", tags=["scim"]) + +_SCIM_USER_SCHEMA = "urn:ietf:params:scim:schemas:core:2.0:User" +_SCIM_ERROR_SCHEMA = "urn:ietf:params:scim:api:messages:2.0:Error" + + +def _resolve_db_path(request: Request) -> Path: + path = getattr(request.app.state, "auth_db_path", None) + # API-3: 用 resolve_auth_db_path() 在调用时读取 env var， + # 让测试 monkeypatch.setenv("AGENTKIT_AUTH_DB", tmp) 可见。 + return Path(path) if path else _resolve_default_db_path() + + +def _resolve_scim_token(request: Request) -> str | None: + """从 app.state 或环境变量解析 SCIM bearer token。""" + token = getattr(request.app.state, "scim_token", None) + if token: + return token + return os.environ.get("AGENTKIT_SCIM_TOKEN") + + +async def _verify_scim_token(request: Request) -> str: + """SCIM bearer token 校验 — 恒定时间比较防时序攻击。""" + expected = _resolve_scim_token(request) + if not expected: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="SCIM 未配置 bearer token（auth.scim_token）", + ) + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + raise HTTPException(status_code=401, detail="缺少 SCIM Bearer token") + provided = auth_header[7:] + if not hmac.compare_digest(provided, expected): + raise HTTPException(status_code=401, detail="SCIM token 无效") + return provided + + +def _alert_scim_failure(operation: str, detail: str, user_id: str | None = None) -> None: + """SCIM 推送失败实时告警 — 日志 error + OTel span event。 + + ponytail: 仅日志 + OTel event。生产可扩展为 webhook / 告警通道。 + 上限：告警去重 / 限流需在告警通道侧实现。 + """ + logger.error("SCIM 推送失败 [op=%s user=%s]: %s", operation, user_id or "?", detail) + try: + from agentkit.telemetry.tracer import get_tracer + + tracer = get_tracer() + with tracer.start_span("scim.push.failure") as span: + span.set_attribute("scim.operation", operation) + span.set_attribute("scim.error", detail) + if user_id: + span.set_attribute("scim.user_id", user_id) + span.add_event("scim.alert", {"operation": operation, "detail": detail}) + except Exception: # noqa: BLE001 — OTel 不可用不阻断主流程 + pass + + +async def _ensure_db(request: Request) -> Path: + db_path = _resolve_db_path(request) + await init_auth_db(db_path) + return db_path + + +# --------------------------------------------------------------------------- +# RFC 7644 3.12 — SCIM 错误响应格式 +# --------------------------------------------------------------------------- + + +def _scim_error_response(status_code: int, detail: str) -> JSONResponse: + """API-2: 构造 RFC 7644 3.12 规定的 SCIM 错误响应格式。""" + return JSONResponse( + status_code=status_code, + content={ + "schemas": [_SCIM_ERROR_SCHEMA], + "status": str(status_code), + "detail": detail, + }, + ) + + +# API-2: APIRouter 在旧版 FastAPI 无 exception_handler 方法，改为导出注册函数 +# 由 app.py 在挂载 SCIM router 后调用，scope 到 /scim/v2 路径。 + + +def register_scim_exception_handlers(app) -> None: + """注册 SCIM 异常处理器 — 仅对 /scim/v2 路径生效，不影响其他路由。 + + API-2: APIRouter 在旧版 FastAPI 无 exception_handler 方法，改为在 app 上注册 + 并通过 request.url.path scope 到 SCIM 路径。非 SCIM 路由走 FastAPI 默认格式。 + """ + + @app.exception_handler(HTTPException) + async def _scim_http_handler(request: Request, exc: HTTPException): + path = request.url.path + is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") + if not is_scim: + # 非 SCIM 路由 — 复制 FastAPI 默认 HTTPException handler 行为（含 headers） + headers = getattr(exc, "headers", None) + return JSONResponse( + status_code=exc.status_code, + content={"detail": exc.detail}, + headers=headers, + ) + return _scim_error_response(exc.status_code, str(exc.detail)) + + @app.exception_handler(RequestValidationError) + async def _scim_validation_handler(request: Request, exc: RequestValidationError): + path = request.url.path + is_scim = path.startswith("/scim/v2") or path.startswith("/api/v1/scim/v2") + if not is_scim: + # 非 SCIM 路由 — 复制 FastAPI 默认 RequestValidationError handler 行为 + from fastapi.encoders import jsonable_encoder + + return JSONResponse( + status_code=422, + content={"detail": jsonable_encoder(exc.errors())}, + ) + return _scim_error_response(400, f"请求体校验失败: {exc}") + + +# --------------------------------------------------------------------------- +# SCIM User 资源转换 — STAND-1: 用 dict[str, object] 替代 Any +# --------------------------------------------------------------------------- + + +def _user_to_scim(row: dict[str, object]) -> dict[str, object]: + """本地 user dict → SCIM 2.0 User JSON。 + + API-11: displayName 改为 None（让客户端用 userName fallback）， + 因为 users 表无 name/full_name 列，username 不是 displayName 的合理来源。 + """ + email = str(row.get("email", "") or "") + return { + "schemas": [_SCIM_USER_SCHEMA], + "id": str(row["id"]), + "userName": str(row["username"]), + "active": bool(row.get("is_active", True)), + "displayName": None, + "emails": [{"value": email, "type": "work", "primary": True}] if email else [], + } + + +# --------------------------------------------------------------------------- +# SCIM 端点 +# --------------------------------------------------------------------------- + + +@router.post("/Users", status_code=status.HTTP_201_CREATED) +async def create_user( + payload: SCIMUser, + request: Request, + _token: str = Depends(_verify_scim_token), +) -> JSONResponse: + """SCIM POST /Users — 创建本地用户（随机密码，不可本地登录）。 + + API-5: 同时写 user_idp_links 行，让 SCIM 预配的用户能通过 SSO 登录， + 避免 OIDC JIT 创建重复账户。externalId 作为 idp_subject， + 缺省时用 userName。 + """ + db_path = await _ensure_db(request) + user_id = str(uuid.uuid4()) + now = _now_iso() + email = payload.emails[0].value if payload.emails else f"{user_id}@scim.local" + # externalId 是 RFC 7643 标准 attribute，作为 IDP subject 关联本地用户 + external_id = getattr(payload, "externalId", None) or payload.userName + try: + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + await db.execute( + "INSERT INTO users " + "(id, username, email, password_hash, role, is_active, " + " is_terminal_authorized, is_server_terminal_authorized, " + " created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + user_id, + payload.userName, + str(email), + hash_password(secrets.token_urlsafe(32)), + "member", + 1 if payload.active else 0, + 0, + 0, + now, + now, + ), + ) + # API-5: 写 user_idp_links 行，idp_name='scim'，idp_subject=externalId + # 表无 id 列（PK = idp_name + idp_subject），省去 uuid 生成。 + await db.execute( + "INSERT INTO user_idp_links " + "(idp_name, idp_subject, user_id, idp_type, created_at) " + "VALUES (?, ?, ?, ?, ?)", + ("scim", external_id, user_id, "scim", now), + ) + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + logger.info( + "SCIM 创建用户 userName=%s id=%s externalId=%s", payload.userName, user_id, external_id + ) + # API-6: RFC 7644 3.14 强制要求 Location 头 + body = _user_to_scim(user_row_to_dict(row)) + response = JSONResponse(status_code=status.HTTP_201_CREATED, content=body) + response.headers["Location"] = f"/scim/v2/Users/{user_id}" + return response + except aiosqlite.IntegrityError as exc: + _alert_scim_failure("create", f"用户名或邮箱冲突: {exc}", user_id) + return _scim_error_response(409, "userName 或 email 已存在") + except Exception as exc: # noqa: BLE001 + _alert_scim_failure("create", str(exc), user_id) + return _scim_error_response(500, "SCIM 创建失败") + + +@router.patch("/Users/{user_id}") +async def patch_user( + user_id: str, + payload: SCIMPatchRequest, + request: Request, + _token: str = Depends(_verify_scim_token), +) -> dict[str, object]: + """SCIM PATCH /Users/{id} — 禁用 (active=false) 或更新字段。 + + API-7: userName UNIQUE 冲突时返回 409（而非 500）。 + SEC-3: active true→false 触发审计记录（与 admin deprovision 对齐）。 + """ + db_path = await _ensure_db(request) + now = _now_iso() + # SEC-3: 收集 deprovision 审计信息，在事务提交后调用（避免 SQLite 写锁竞争） + _pending_audit: dict[str, object] | None = None + try: + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + return _scim_error_response(404, "用户不存在") + prev_active = bool(row["is_active"]) + for op in payload.Operations: + if op.op.lower() == "replace": + if op.path in (None, "active"): + active = ( + bool(op.value) + if not isinstance(op.value, dict) + else bool(op.value.get("active", True)) + ) + await db.execute( + "UPDATE users SET is_active = ?, updated_at = ? WHERE id = ?", + (1 if active else 0, now, user_id), + ) + # SEC-3: active true→false 写审计（延后到事务提交后） + if prev_active and not active: + _pending_audit = { + "target_user_id": user_id, + "target_username": str(row["username"]), + } + elif op.path == "userName": + try: + await db.execute( + "UPDATE users SET username = ?, updated_at = ? WHERE id = ?", + (str(op.value)[:64], now, user_id), + ) + except aiosqlite.IntegrityError: + # API-7: UNIQUE 冲突返回 409 + return _scim_error_response(409, "userName 已存在") + await db.commit() + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + assert row is not None + # SEC-3: 事务提交后写审计（避免 SQLite "database is locked"） + if _pending_audit is not None: + await get_audit_service().record_deprovision( + actor_user_id=None, + actor_username="scim", + target_user_id=str(_pending_audit["target_user_id"]), + target_username=str(_pending_audit["target_username"]), + reason="scim_patch_active_false", + source=SOURCE_SCIM, + ) + return _user_to_scim(user_row_to_dict(row)) + except HTTPException: + raise + except Exception as exc: # noqa: BLE001 + _alert_scim_failure("patch", str(exc), user_id) + return _scim_error_response(500, "SCIM 更新失败") + + +@router.get("/Users") +async def list_users( + request: Request, + filter: str | None = Query(default=None, alias="filter"), + startIndex: int = Query(default=1, ge=1), + count: int = Query(default=100, ge=1, le=200), + _token: str = Depends(_verify_scim_token), +) -> dict[str, object]: + """SCIM GET /Users — 列表 / 过滤。 + + API-1: totalResults 返回全量总数（COUNT 查询），而非当前页条数。 + API-9: 不支持的 filter 语法返回 400 + invalidFilter，不静默全量返回。 + API-13: filter 解析大小写不敏感（RFC 7644 3.4.2.2）。 + """ + db_path = await _ensure_db(request) + where = "" + params: list[object] = [] + if filter: + filter_lower = filter.lower() + # API-13: 大小写不敏感匹配 + if "username eq" in filter_lower: + # 从原 filter 中提取引号内的值（保留原大小写） + val = filter.split(" eq", 1)[1].strip().strip('"').strip("'") + where = "WHERE username = ?" + params.append(val) + elif "active eq" in filter_lower: + val = filter.split(" eq", 1)[1].strip().strip('"').strip("'").lower() + where = "WHERE is_active = ?" + params.append(1 if val in ("true", "1") else 0) + else: + # API-9: 不支持的 filter 返回 400 + return _scim_error_response( + 400, f"Unsupported filter syntax (scimType=invalidFilter): {filter}" + ) + # API-1: 先 COUNT 全量总数 + count_sql = f"SELECT COUNT(*) FROM users {where}" + list_sql = f"SELECT * FROM users {where} ORDER BY created_at ASC LIMIT ? OFFSET ?" + list_params = [*params, count, startIndex - 1] + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + # totalResults + cursor = await db.execute(count_sql, tuple(params)) + total_results = (await cursor.fetchone())[0] + # 当前页 + cursor = await db.execute(list_sql, tuple(list_params)) + rows = await cursor.fetchall() + resources = [_user_to_scim(user_row_to_dict(r)) for r in rows] + return SCIMListResponse( + totalResults=total_results, + startIndex=startIndex, + itemsPerPage=len(resources), + Resources=resources, + ).model_dump() diff --git a/src/agentkit/server/frontend/src/stores/chatStream.ts b/src/agentkit/server/frontend/src/stores/chatStream.ts index 39f35c9..020d95f 100644 --- a/src/agentkit/server/frontend/src/stores/chatStream.ts +++ b/src/agentkit/server/frontend/src/stores/chatStream.ts @@ -869,7 +869,8 @@ export function dispatchWsEvent( case "team_synthesis_chunk": { // U4: 团队综合流式 chunk — 首次创建 streaming 占位，后续累加。 - // P2 fix: 用 synthesis_id 去重，避免 chunk 附身到上一次孤儿 streaming milestone。 + // P2 fix: 用 synthesis_id 精确去重 — sid 存在时仅匹配相同 synthesis_id 的 milestone， + // 不再兜底附身到 undefined synthesis_id 的旧孤儿占位（避免跨重试附身 bug）。 const conversationId = state.resolveIncomingConvId(); if (!conversationId) break; const conv = state.conversations.value.find( @@ -882,13 +883,11 @@ export function dispatchWsEvent( (m) => m.message_type === "milestone" && m.status === "streaming" && - (sid ? m.synthesis_id === sid || m.synthesis_id === undefined : true), + (sid ? m.synthesis_id === sid : true), ); if (existing) { state.updateMessage(conversationId, existing.id, { content: (existing.content || "") + (event.data.chunk || ""), - // 首次附身到无 synthesis_id 的 milestone 时，标记上 synthesis_id - ...(sid && !existing.synthesis_id ? { synthesis_id: sid } : {}), }); } else { const synthMsg: IChatMessage = { @@ -926,7 +925,7 @@ export function dispatchWsEvent( (m) => m.message_type === "milestone" && m.status === "streaming" && - (sid ? m.synthesis_id === sid || m.synthesis_id === undefined : true), + (sid ? m.synthesis_id === sid : true), ); if (existing) { state.updateMessage(conversationId, existing.id, { diff --git a/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts new file mode 100644 index 0000000..e028777 --- /dev/null +++ b/src/agentkit/server/frontend/tests/unit/bitable-tokens.test.ts @@ -0,0 +1,97 @@ +/** + * DR-3: Bitable design tokens — static structural assertion. + * + * Parses `bitable-tokens.css` and asserts all required `--bitable-*` tokens + * are defined inside `:root`. Prevents accidental token removal/renaming + * that would silently break var() references in bitable components. + * + * ponytail: pure regex parse — no CSS parser dependency. Ceiling: doesn't + * validate token values (only presence). Upgrade path: postcss + stylelint + * custom rules for value-level validation. + */ +import { describe, expect, it } from 'vitest' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +const CSS_PATH = resolve( + __dirname, + '../../src/styles/bitable-tokens.css', +) + +const cssContent: string = readFileSync(CSS_PATH, 'utf-8') + +/**Extract the first `:root { ... }` block content (without the selector).*/ +function extractRootBlock(css: string): string { + const match = /:root\s*\{([^}]*)\}/.exec(css) + if (!match || match[1] === undefined) { + throw new Error('No :root block found in bitable-tokens.css') + } + return match[1] +} + +const rootBlock = extractRootBlock(cssContent) + +/**Required token names grouped by category (mirrors the CSS file structure).*/ +const REQUIRED_TOKENS = { + color: [ + '--bitable-color-primary', + '--bitable-color-bg', + '--bitable-color-bg-secondary', + '--bitable-color-bg-tertiary', + '--bitable-color-text', + '--bitable-color-text-secondary', + '--bitable-color-text-tertiary', + '--bitable-color-text-placeholder', + '--bitable-color-border', + '--bitable-color-border-hover', + '--bitable-color-border-split', + ], + cf: [ + '--bitable-cf-red-bg', + '--bitable-cf-red-fg', + '--bitable-cf-orange-bg', + '--bitable-cf-orange-fg', + '--bitable-cf-yellow-bg', + '--bitable-cf-yellow-fg', + '--bitable-cf-green-bg', + '--bitable-cf-green-fg', + '--bitable-cf-blue-bg', + '--bitable-cf-blue-fg', + '--bitable-cf-purple-bg', + '--bitable-cf-purple-fg', + '--bitable-cf-gray-bg', + '--bitable-cf-gray-fg', + '--bitable-cf-neutral-bg', + '--bitable-cf-neutral-fg', + ], + drawer: ['--bitable-drawer-width', '--bitable-drawer-width-wide'], +} as const + +describe('bitable-tokens.css — DR-3 design token contract', () => { + it('defines a :root block', () => { + expect(rootBlock.length).toBeGreaterThan(0) + }) + + it.each(REQUIRED_TOKENS.color)('defines color token %s in :root', (token) => { + // Match `--token-name:` (declaration), not just substring presence. + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it.each(REQUIRED_TOKENS.cf)('defines conditional-format token %s in :root', (token) => { + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it.each(REQUIRED_TOKENS.drawer)('defines drawer token %s in :root', (token) => { + const declPattern = new RegExp(`${token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*:`) + expect(declPattern.test(rootBlock)).toBe(true) + }) + + it('does not hardcode tokens outside :root (no component-level overrides)', () => { + // ponytail: bitable tokens are exclusively :root-scoped; component-level + // overrides would break the single-source-of-truth invariant. + const outsideRoot = cssContent.replace(/:root\s*\{[^}]*\}/, '') + expect(outsideRoot).not.toMatch(/--bitable-[a-z-]+\s*:/) + }) +}) diff --git a/src/agentkit/server/routes/auth.py b/src/agentkit/server/routes/auth.py index 4fff714..ac77089 100644 --- a/src/agentkit/server/routes/auth.py +++ b/src/agentkit/server/routes/auth.py @@ -23,9 +23,10 @@ from __future__ import annotations import hmac import logging +import secrets from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, Literal import aiosqlite import jwt @@ -780,6 +781,162 @@ async def change_password( return {"changed": True, "revoked_other_sessions": revoked} +# --------------------------------------------------------------------------- +# SSO routes — OIDC redirect/callback + SAML ACS (U4) +# --------------------------------------------------------------------------- + + +class SSORedirectResponse(BaseModel): + """SSO 重定向响应 — 返回 IDP authorize URL 供前端跳转。""" + + model_config = ConfigDict(extra="forbid") + + redirect_url: str + state: str + + +async def _sso_issue_token_pair( + request: Request, + user_dict: dict[str, object], + *, + provider_type: str = "sso", + provider_name: str | None = None, +) -> TokenResponse: + """SSO 登录成功后签发 JWT pair + 创建 session（复用 login 流程）。 + + API-4: 一次性生成真实 refresh_token 再创建 session（避免占位符并发冲突）。 + API-8: auth_provider 携带 provider_type + provider_name（如 'oidc-keycloak'）， + 供 admin session 列表按 IDP 筛选 + 审计追溯。 + SEC-1: 与本地 login 一致地校验 is_active。 + """ + # SEC-1: SSO 流程必须与本地 login（auth.py:377）一致地校验 is_active， + # 否则已 deprovision 的用户可通过 IDP 回调重新拿到 JWT + session。 + if not bool(user_dict.get("is_active", True)): + raise HTTPException(status_code=403, detail="Account is disabled") + db_path = await _ensure_db(request) + secret = _resolve_jwt_secret(request) + svc: SessionService = get_session_service() + info = _client_info(request) + + # API-4: 先生成真实 refresh_token，避免占位符 "__pending_sso__" 在并发 SSO + # 登录时触发 refresh_token_hash UNIQUE 约束冲突。 + refresh_token = secrets.token_urlsafe(32) + # API-8: auth_provider 形如 "oidc-keycloak" / "saml-azuread" / "sso" + auth_provider = f"{provider_type}-{provider_name}" if provider_name else provider_type + + session = await svc.create( + SessionCreate( + user_id=str(user_dict["id"]), + refresh_token=refresh_token, + device_fingerprint=info["fingerprint"], + device_label=info["label"], + ip=info["ip"], + user_agent=info["user_agent"], + auth_provider=auth_provider, + ttl_seconds=_refresh_ttl_seconds(False), + ) + ) + token_pair = create_token_pair( + user_id=str(user_dict["id"]), + username=str(user_dict["username"]), + role=str(user_dict["role"]), + secret=secret, + session_id=session.id, + ) + async with aiosqlite.connect(str(db_path)) as db: + await db.execute( + "UPDATE users SET last_login_at = ?, updated_at = ? WHERE id = ?", + (_now_iso(), _now_iso(), str(user_dict["id"])), + ) + await db.commit() + return TokenResponse( + access_token=token_pair.access_token, + refresh_token=token_pair.refresh_token, + expires_in=int(ACCESS_TOKEN_TTL.total_seconds()), + user=UserResponse( + id=str(user_dict["id"]), + username=str(user_dict["username"]), + email=str(user_dict["email"]), + role=str(user_dict["role"]), + is_active=bool(user_dict.get("is_active", True)), + is_terminal_authorized=bool(user_dict.get("is_terminal_authorized", False)), + is_server_terminal_authorized=bool( + user_dict.get("is_server_terminal_authorized", False) + ), + ), + session_id=session.id, + ) + + +@router.get("/oauth/{provider}/redirect", response_model=SSORedirectResponse) +async def oidc_redirect(provider: str, request: Request) -> SSORedirectResponse: + """OIDC 重定向 — 返回 IDP authorize URL（前端跳转）。""" + from agentkit.server.auth.providers.oidc import OIDCProvider + + try: + url = await OIDCProvider().get_redirect_url(provider) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + # state 已由 provider 写入 _state_cache，从 URL 中提取回传前端 + from urllib.parse import parse_qs, urlparse + + parsed = urlparse(url) + state_values = parse_qs(parsed.query).get("state", []) + state = state_values[0] if state_values else "" + return SSORedirectResponse(redirect_url=url, state=state) + + +@router.get("/oauth/{provider}/callback", response_model=TokenResponse) +async def oidc_callback( + provider: str, + code: str, + state: str, + request: Request, +) -> TokenResponse: + """OIDC callback — code 换 token + userinfo + JIT 用户关联 → JWT pair。""" + await _ensure_db(request) + from agentkit.server.auth.providers.oidc import OIDCProvider + + try: + user_dict = await OIDCProvider().handle_callback(provider, code, state) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return await _sso_issue_token_pair( + request, user_dict, provider_type="oidc", provider_name=provider + ) + + +class SAMLACSRequest(BaseModel): + """SAML ACS 请求体。""" + + model_config = ConfigDict(extra="forbid") + + SAMLResponse: str + RelayState: str | None = None + + +@router.post("/saml/{provider}/acs", response_model=TokenResponse) +async def saml_acs( + provider: str, + payload: SAMLACSRequest, + request: Request, +) -> TokenResponse: + """SAML ACS — 消费 SAMLResponse + NameID 提取 + JIT 用户关联 → JWT pair。""" + await _ensure_db(request) + from agentkit.server.auth.providers.saml import SAMLProvider + + request_url = str(request.url) + try: + user_dict = await SAMLProvider().handle_acs( + provider, payload.SAMLResponse, request_url=request_url + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return await _sso_issue_token_pair( + request, user_dict, provider_type="saml", provider_name=provider + ) + + # --------------------------------------------------------------------------- # Admin routes # --------------------------------------------------------------------------- @@ -889,6 +1046,146 @@ async def admin_revoke_user_session( return {"revoked": ok} +# --------------------------------------------------------------------------- +# Admin deprovisioning + RBAC role change (U4 — R1b / KTD-8) +# --------------------------------------------------------------------------- + + +class DeprovisionRequest(BaseModel): + """手动 deprovisioning 请求体。""" + + model_config = ConfigDict(extra="forbid") + + reason: str | None = None + break_glass: bool = False # 高权限账户 break-glass 标记 + + +class DeprovisionResponse(BaseModel): + """API-10: deprovision 响应体（让 OpenAPI schema 可文档化）。""" + + model_config = ConfigDict(extra="forbid") + + deprovisioned: bool + revoked_sessions: int + + +class RoleChangeResponse(BaseModel): + """API-10: 角色变更响应体。""" + + model_config = ConfigDict(extra="forbid") + + role_changed: bool + role_before: str + role_after: str + revoked_sessions: int + + +@admin_router.post("/users/{user_id}/deprovision", response_model=DeprovisionResponse) +async def deprovision_user( + user_id: str, + payload: DeprovisionRequest, + request: Request, + admin: dict[str, Any] = Depends(_require_admin), +) -> DeprovisionResponse: + """手动 deprovisioning (R1b) — operator/admin RBAC 强制。 + + 1. 禁用本地用户 (is_active=0) + 2. 撤销该用户所有活跃 session + 3. 记录审计 (actor/target/reason/source/timestamp)，接入 OTel + 4. break_glass=True 时记录 source=break_glass（高权限账户应急通道） + """ + db_path = await _ensure_db(request) + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + raise HTTPException(status_code=404, detail="User not found") + await db.execute( + "UPDATE users SET is_active = 0, updated_at = ? WHERE id = ?", + (_now_iso(), user_id), + ) + await db.commit() + # 撤销所有 session + svc: SessionService = get_session_service() + revoked = await svc.revoke_all_for_user(user_id, reason="admin_revoked") + # 审计记录 + from agentkit.server.auth.audit import ( + SOURCE_BREAKGLASS, + SOURCE_MANUAL, + get_audit_service, + ) + + source = SOURCE_BREAKGLASS if payload.break_glass else SOURCE_MANUAL + await get_audit_service().record_deprovision( + actor_user_id=str(admin.get("user_id") or ""), + actor_username=str(admin.get("username") or ""), + target_user_id=user_id, + target_username=str(row["username"]), + reason=payload.reason, + source=source, + ) + return DeprovisionResponse(deprovisioned=True, revoked_sessions=revoked) + + +class RoleChangeRequest(BaseModel): + """RBAC 角色变更请求体。""" + + model_config = ConfigDict(extra="forbid") + + role: Literal["member", "operator", "admin"] + reason: str | None = None + + +@admin_router.post("/users/{user_id}/role", response_model=RoleChangeResponse) +async def change_user_role( + user_id: str, + payload: RoleChangeRequest, + request: Request, + admin: dict[str, Any] = Depends(_require_admin), +) -> RoleChangeResponse: + """RBAC 角色变更 (KTD-8) — admin only。 + + 记录 actor/target/role_before/role_after/reason/source/timestamp，接入 OTel 审计。 + 变更后撤销该用户所有现有 session（强制重新登录以刷新 JWT role claim）。 + """ + db_path = await _ensure_db(request) + from agentkit.server.auth.audit import SOURCE_MANUAL, get_audit_service + + async with aiosqlite.connect(str(db_path)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT * FROM users WHERE id = ?", (user_id,)) + row = await cursor.fetchone() + if row is None: + raise HTTPException(status_code=404, detail="User not found") + role_before = str(row["role"]) + await db.execute( + "UPDATE users SET role = ?, updated_at = ? WHERE id = ?", + (payload.role, _now_iso(), user_id), + ) + await db.commit() + # 审计记录 + await get_audit_service().record_role_change( + actor_user_id=str(admin.get("user_id") or ""), + actor_username=str(admin.get("username") or ""), + target_user_id=user_id, + target_username=str(row["username"]), + role_before=role_before, + role_after=payload.role, + reason=payload.reason, + source=SOURCE_MANUAL, + ) + # 撤销所有 session（强制重新登录刷新 role claim） + svc: SessionService = get_session_service() + revoked = await svc.revoke_all_for_user(user_id, reason="role_changed") + return RoleChangeResponse( + role_changed=True, + role_before=role_before, + role_after=payload.role, + revoked_sessions=revoked, + ) + + # Backwards-compat /me endpoint (kept for the existing frontend) @router.get("/me", response_model=UserResponse) async def me( diff --git a/src/agentkit/server/task_store.py b/src/agentkit/server/task_store.py index d1c9d42..69f69c1 100644 --- a/src/agentkit/server/task_store.py +++ b/src/agentkit/server/task_store.py @@ -3,6 +3,7 @@ import asyncio import json import logging +import os from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any @@ -15,6 +16,7 @@ logger = logging.getLogger(__name__) @dataclass class TaskRecord: """Stored task record with full lifecycle data""" + task_id: str agent_name: str skill_name: str | None @@ -57,9 +59,15 @@ class TaskRecord: status=TaskStatus(data.get("status", "pending")), output_data=data.get("output_data"), error_message=data.get("error_message"), - created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.now(timezone.utc), - started_at=datetime.fromisoformat(data["started_at"]) if data.get("started_at") else None, - completed_at=datetime.fromisoformat(data["completed_at"]) if data.get("completed_at") else None, + created_at=datetime.fromisoformat(data["created_at"]) + if data.get("created_at") + else datetime.now(timezone.utc), + started_at=datetime.fromisoformat(data["started_at"]) + if data.get("started_at") + else None, + completed_at=datetime.fromisoformat(data["completed_at"]) + if data.get("completed_at") + else None, progress=data.get("progress", 0.0), progress_message=data.get("progress_message", ""), metadata=data.get("metadata", {}), @@ -124,14 +132,19 @@ class InMemoryTaskStore: if expired: logger.info(f"TaskStore cleaned up {len(expired)} expired records") - def create(self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None) -> TaskRecord: + def create( + self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None + ) -> TaskRecord: """Create a new task record""" if len(self._tasks) >= self._max_records: # Remove oldest completed task oldest = None for rec in self._tasks.values(): if rec.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): - if oldest is None or (rec.completed_at and (oldest.completed_at is None or rec.completed_at < oldest.completed_at)): + if oldest is None or ( + rec.completed_at + and (oldest.completed_at is None or rec.completed_at < oldest.completed_at) + ): oldest = rec if oldest: del self._tasks[oldest.task_id] @@ -223,9 +236,11 @@ class RedisTaskStore: if self._redis is None: import redis.asyncio as aioredis + # DEPLOY-2: 显式传入 REDIS_PASSWORD env var（Helm 注入但 redis-py 不自动读取）。 self._redis = aioredis.from_url( self._redis_url, decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, ) return self._redis @@ -245,7 +260,9 @@ class RedisTaskStore: # ── CRUD ─────────────────────────────────────────────────── - async def create(self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None) -> TaskRecord: + async def create( + self, task_id: str, agent_name: str, input_data: dict, skill_name: str | None = None + ) -> TaskRecord: """Create a new task record in Redis.""" redis = await self._get_redis() @@ -305,7 +322,9 @@ end return encoded """ - async def update_status(self, task_id: str, status: TaskStatus, reset_ttl: bool = False, **kwargs) -> TaskRecord: + async def update_status( + self, task_id: str, status: TaskStatus, reset_ttl: bool = False, **kwargs + ) -> TaskRecord: """Update task status and optional fields atomically via Lua script. Args: @@ -322,7 +341,15 @@ return encoded # Build flat list of key-value pairs for the merge fields merge_fields = {"status": status.value} for k, value in kwargs.items(): - if k in ("started_at", "completed_at", "output_data", "error_message", "progress", "progress_message", "metadata"): + if k in ( + "started_at", + "completed_at", + "output_data", + "error_message", + "progress", + "progress_message", + "metadata", + ): if isinstance(value, datetime): merge_fields[k] = value.isoformat() else: @@ -340,7 +367,9 @@ return encoded data = json.loads(result) return TaskRecord.from_dict(data) - async def list_tasks(self, status: TaskStatus | None = None, limit: int = 100) -> list[TaskRecord]: + async def list_tasks( + self, status: TaskStatus | None = None, limit: int = 100 + ) -> list[TaskRecord]: """List tasks, optionally filtered by status, sorted by created_at desc.""" redis = await self._get_redis() tasks: list[TaskRecord] = [] @@ -433,7 +462,11 @@ return encoded await redis.zrem(self.ZSET_KEY, tid) continue record = TaskRecord.from_dict(json.loads(raw)) - if record.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED) and record.completed_at is not None: + if ( + record.status + in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED) + and record.completed_at is not None + ): await redis.delete(self._key(tid)) await redis.zrem(self.ZSET_KEY, tid) return True @@ -452,7 +485,11 @@ return encoded if raw is None: continue record = TaskRecord.from_dict(json.loads(raw)) - if record.status in (TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED): + if record.status in ( + TaskStatus.COMPLETED, + TaskStatus.FAILED, + TaskStatus.CANCELLED, + ): tasks.append(record) if cursor == 0: break @@ -505,7 +542,9 @@ def create_task_store( logger.info(f"TaskStore backend: redis ({_sanitize_redis_url(redis_url)})") return store except Exception as exc: - logger.warning(f"Failed to initialise RedisTaskStore ({exc}), falling back to InMemoryTaskStore") + logger.warning( + f"Failed to initialise RedisTaskStore ({exc}), falling back to InMemoryTaskStore" + ) store = InMemoryTaskStore(ttl_seconds=ttl_seconds, max_records=max_records) logger.info("TaskStore backend: memory") diff --git a/src/agentkit/session/store.py b/src/agentkit/session/store.py index 23b5f2e..4628d87 100644 --- a/src/agentkit/session/store.py +++ b/src/agentkit/session/store.py @@ -24,12 +24,18 @@ class SessionStore(Protocol): async def save_session(self, session: Session) -> None: ... async def get_session(self, session_id: str) -> Session | None: ... - async def update_session_status(self, session_id: str, status: SessionStatus) -> Session | None: ... + async def update_session_status( + self, session_id: str, status: SessionStatus + ) -> Session | None: ... async def delete_session(self, session_id: str) -> bool: ... - async def list_sessions(self, agent_name: str | None = None, limit: int = 100) -> list[Session]: ... + async def list_sessions( + self, agent_name: str | None = None, limit: int = 100 + ) -> list[Session]: ... async def append_message(self, message: Message) -> None: ... - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: ... + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: ... async def count_messages(self, session_id: str) -> int: ... async def health_check(self) -> bool: ... @@ -91,7 +97,9 @@ class InMemorySessionStore: del msgs[:excess] msgs.append(message) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: msgs = self._messages.get(session_id, []) sliced = msgs[offset:] if limit is not None: @@ -125,7 +133,13 @@ class RedisSessionStore: if self._redis is None: import redis.asyncio as aioredis - self._redis = aioredis.from_url(self._redis_url, decode_responses=True) + # DEPLOY-2: Helm 注入 REDIS_PASSWORD env var，redis-py from_url 不自动读取， + # 必须显式以 password= 传入。env var 未设置时 password=None 保持向后兼容。 + self._redis = aioredis.from_url( + self._redis_url, + decode_responses=True, + password=os.environ.get("REDIS_PASSWORD") or None, + ) return self._redis def _session_key(self, session_id: str) -> str: @@ -192,7 +206,9 @@ class RedisSessionStore: # Set TTL on message list to match session TTL await redis.expire(key, self._ttl_seconds) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: redis = await self._get_redis() key = self._messages_key(session_id) # Use LRANGE for offset-based pagination @@ -304,7 +320,9 @@ class FileSessionStore: data["session"]["updated_at"] = datetime.now(timezone.utc).isoformat() self._write_session_file(message.session_id, data) - async def get_messages(self, session_id: str, limit: int | None = None, offset: int = 0) -> list[Message]: + async def get_messages( + self, session_id: str, limit: int | None = None, offset: int = 0 + ) -> list[Message]: data = self._read_session_file(session_id) if data is None: return [] @@ -347,10 +365,12 @@ def create_session_store( import redis.asyncio as aioredis # noqa: F401 store = RedisSessionStore(redis_url=redis_url, ttl_seconds=ttl_seconds) - logger.info(f"SessionStore backend: redis") + logger.info("SessionStore backend: redis") return store except (ImportError, OSError, _RedisError, ValueError, KeyError, RuntimeError) as exc: - logger.warning(f"Failed to initialise RedisSessionStore ({exc}), falling back to InMemorySessionStore") + logger.warning( + f"Failed to initialise RedisSessionStore ({exc}), falling back to InMemorySessionStore" + ) store = InMemorySessionStore() logger.info("SessionStore backend: memory") diff --git a/src/agentkit/telemetry/metrics.py b/src/agentkit/telemetry/metrics.py index 0525be7..fde9b1f 100644 --- a/src/agentkit/telemetry/metrics.py +++ b/src/agentkit/telemetry/metrics.py @@ -1,11 +1,15 @@ -"""Metric definitions — no-op when OTel not installed""" +"""Metric definitions — OTel now required (U1). -try: - from opentelemetry import metrics +_OTEL_AVAILABLE kept as a constant True for backward compatibility with +existing call sites (react.py / rewoo.py / reflexion.py / gateway.py) that +guard span creation with `if _OTEL_AVAILABLE:`. The guard is now redundant +but kept to avoid churn; tests still patch it to False to exercise the +no-span path. +""" - _OTEL_AVAILABLE = True -except ImportError: - _OTEL_AVAILABLE = False +from opentelemetry import metrics + +_OTEL_AVAILABLE = True class _NoOpCounter: @@ -42,6 +46,11 @@ _agent_duration_histogram = None _llm_token_histogram = None _tool_duration_histogram = None _pipeline_step_histogram = None +_prompt_cache_hit_counter = None +_prompt_cache_miss_counter = None +# U2 — PII 脱敏覆盖率指标 +_pii_filter_coverage_counter = None +_pii_filter_redacted_counter = None def _get_counter(name: str, description: str, unit: str = "1"): @@ -92,9 +101,7 @@ def tool_duration_histogram(): """Tool call duration.""" global _tool_duration_histogram if _tool_duration_histogram is None: - _tool_duration_histogram = _get_histogram( - "tool.call.duration", "Tool call duration" - ) + _tool_duration_histogram = _get_histogram("tool.call.duration", "Tool call duration") return _tool_duration_histogram @@ -106,3 +113,45 @@ def pipeline_step_histogram(): "pipeline.step.duration", "Pipeline step duration" ) return _pipeline_step_histogram + + +# U2 — Prompt cache hit/miss counters（Anthropic cache_read_input_tokens > 0 → hit） + + +def prompt_cache_hit_counter(): + """Prompt cache hit count（cache_read_input_tokens > 0）.""" + global _prompt_cache_hit_counter + if _prompt_cache_hit_counter is None: + _prompt_cache_hit_counter = _get_counter("prompt_cache.hit", "Prompt cache hit count") + return _prompt_cache_hit_counter + + +def prompt_cache_miss_counter(): + """Prompt cache miss count（cache_read_input_tokens == 0）.""" + global _prompt_cache_miss_counter + if _prompt_cache_miss_counter is None: + _prompt_cache_miss_counter = _get_counter("prompt_cache.miss", "Prompt cache miss count") + return _prompt_cache_miss_counter + + +# U2 — PII 脱敏指标 + + +def pii_filter_coverage_counter(): + """PII filter coverage: total messages scanned.""" + global _pii_filter_coverage_counter + if _pii_filter_coverage_counter is None: + _pii_filter_coverage_counter = _get_counter( + "pii_filter.coverage", "PII filter: total messages scanned" + ) + return _pii_filter_coverage_counter + + +def pii_filter_redacted_counter(): + """PII filter: count of PII entities redacted.""" + global _pii_filter_redacted_counter + if _pii_filter_redacted_counter is None: + _pii_filter_redacted_counter = _get_counter( + "pii_filter.redacted", "PII filter: PII entities redacted" + ) + return _pii_filter_redacted_counter diff --git a/src/agentkit/telemetry/setup.py b/src/agentkit/telemetry/setup.py index 5da9581..38b01d6 100644 --- a/src/agentkit/telemetry/setup.py +++ b/src/agentkit/telemetry/setup.py @@ -1,16 +1,26 @@ -"""OTel initialization — called at app startup""" +"""OTel initialization — called at app startup (U1: OTel now required).""" import logging +from opentelemetry import trace, metrics +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + logger = logging.getLogger(__name__) def setup_telemetry(app, config: dict | None = None): - """Initialize OpenTelemetry if installed and configured. + """Initialize OpenTelemetry if configured. - This is a no-op when: - - config is None or config.enabled is False - - opentelemetry packages are not installed + U1: opentelemetry-* deps now required (pyproject.toml). ImportError + fallbacks removed — a missing package is a hard install error. The + `enabled=False` config switch remains the supported way to disable. Args: app: FastAPI application instance @@ -25,69 +35,29 @@ def setup_telemetry(app, config: dict | None = None): logger.info("Telemetry disabled") return - try: - from opentelemetry import trace, metrics - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader - from opentelemetry.sdk.resources import Resource - except ImportError: - logger.warning( - "OpenTelemetry packages not installed. Telemetry disabled." - ) - return - service_name = config.get("service_name", "fischer-agentkit") resource = Resource.create({"service.name": service_name}) # Tracing setup if config.get("export_traces", True): endpoint = config.get("otlp_endpoint", "http://localhost:4317") - try: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - - provider = TracerProvider(resource=resource) - provider.add_span_processor( - BatchSpanProcessor( - OTLPSpanExporter(endpoint=endpoint, insecure=True) - ) - ) - trace.set_tracer_provider(provider) - logger.info(f"Tracing enabled, exporting to {endpoint}") - except ImportError: - logger.warning( - "OTLP exporter not installed. Tracing disabled." - ) + provider = TracerProvider(resource=resource) + provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True)) + ) + trace.set_tracer_provider(provider) + logger.info(f"Tracing enabled, exporting to {endpoint}") # Metrics setup if config.get("export_metrics", True): endpoint = config.get("otlp_endpoint", "http://localhost:4317") - try: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - - reader = PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=endpoint, insecure=True) - ) - provider = MeterProvider(resource=resource, readers=[reader]) - metrics.set_meter_provider(provider) - logger.info(f"Metrics enabled, exporting to {endpoint}") - except ImportError: - logger.warning( - "OTLP metric exporter not installed. Metrics disabled." - ) + reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint, insecure=True)) + # ponytail: OTel SDK 1.16+ 将 readers kwarg 改名为 metric_readers； + # 旧代码用 readers=[...] 是已存在 bug，被 ImportError fallback 掩盖。 + provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(provider) + logger.info(f"Metrics enabled, exporting to {endpoint}") # FastAPI auto-instrumentation - try: - from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor - - FastAPIInstrumentor.instrument_app(app, excluded_urls="health,metrics") - logger.info("FastAPI auto-instrumentation enabled") - except ImportError: - logger.warning( - "FastAPI instrumentation not installed. Skipping auto-instrumentation." - ) + FastAPIInstrumentor.instrument_app(app, excluded_urls="health,metrics") + logger.info("FastAPI auto-instrumentation enabled") diff --git a/src/agentkit/telemetry/tracer.py b/src/agentkit/telemetry/tracer.py index ad808ef..7893bad 100644 --- a/src/agentkit/telemetry/tracer.py +++ b/src/agentkit/telemetry/tracer.py @@ -1,4 +1,5 @@ """OpenTelemetry tracer integration with no-op fallback.""" + from __future__ import annotations import logging @@ -43,10 +44,14 @@ class NoOpSpan: class NoOpTracer: """No-op tracer when telemetry is disabled.""" - def start_span(self, name: str, attributes: dict[str, object] | None = None) -> ContextManager[NoOpSpan]: + def start_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> ContextManager[NoOpSpan]: return NoOpSpan() - def start_as_current_span(self, name: str, attributes: dict[str, object] | None = None) -> ContextManager[NoOpSpan]: + def start_as_current_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> ContextManager[NoOpSpan]: return NoOpSpan() @@ -86,7 +91,9 @@ class OTelTracer: span = self._tracer.start_span(name, attributes=attributes) return OTelSpan(span) - def start_as_current_span(self, name: str, attributes: dict[str, object] | None = None) -> OTelSpan: + def start_as_current_span( + self, name: str, attributes: dict[str, object] | None = None + ) -> OTelSpan: span = self._tracer.start_as_current_span(name, attributes=attributes) return OTelSpan(span) @@ -101,7 +108,13 @@ def get_tracer() -> NoOpTracer | OTelTracer: def init_telemetry(config: TelemetryConfig) -> None: - """Initialize telemetry with the given configuration.""" + """Initialize telemetry with the given configuration. + + U1: OTel deps now required (pyproject.toml). ImportError fallback removed — + a missing opentelemetry-* package is now a hard install error, not a silent + monitoring blind spot. Runtime errors (network, exporter connect) still + fall back to NoOpTracer so app startup never crashes on transient OTLP issues. + """ global _tracer if not config.enabled: @@ -109,13 +122,13 @@ def init_telemetry(config: TelemetryConfig) -> None: logger.info("Telemetry disabled, using no-op tracer") return - try: - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.resources import Resource - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.sdk.resources import Resource + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + try: resource = Resource.create({"service.name": config.service_name}) provider = TracerProvider(resource=resource) @@ -126,9 +139,6 @@ def init_telemetry(config: TelemetryConfig) -> None: trace.set_tracer_provider(provider) _tracer = OTelTracer(trace.get_tracer(config.service_name)) logger.info(f"Telemetry initialized with endpoint: {config.otlp_endpoint}") - except ImportError: - logger.warning("opentelemetry packages not installed, using no-op tracer") - _tracer = NoOpTracer() except Exception as e: logger.warning(f"Failed to initialize telemetry: {e}, using no-op tracer") _tracer = NoOpTracer() diff --git a/src/agentkit/telemetry/tracing.py b/src/agentkit/telemetry/tracing.py index 4f3c65c..7b10224 100644 --- a/src/agentkit/telemetry/tracing.py +++ b/src/agentkit/telemetry/tracing.py @@ -1,43 +1,33 @@ -"""Tracing helpers — no-op when OTel not installed""" +"""Tracing helpers — OTel now required (U1). + +_OTEL_AVAILABLE kept as a constant True for backward compatibility with +existing call sites (react.py / rewoo.py / reflexion.py / gateway.py) that +guard span creation with `if _OTEL_AVAILABLE:`. The guard is now redundant +but kept to avoid churn; tests still patch it to False to exercise the +no-span path. +""" import logging import time from functools import wraps from typing import Callable +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode + logger = logging.getLogger(__name__) -# Try importing OTel — if not available, provide no-op implementations -try: - from opentelemetry import trace - from opentelemetry.trace import SpanKind, Status, StatusCode - - _OTEL_AVAILABLE = True -except ImportError: - _OTEL_AVAILABLE = False - - # Provide fallback stubs so module-level references work in tests - class _StubEnum: - INTERNAL = "INTERNAL" - CLIENT = "CLIENT" - SERVER = "SERVER" - PRODUCER = "PRODUCER" - CONSUMER = "CONSUMER" - - SpanKind = _StubEnum # type: ignore[misc,assignment] - - class Status: # type: ignore[no-redef] - def __init__(self, *args, **kwargs): - pass - - class StatusCode: # type: ignore[no-redef] - UNSET = "UNSET" - OK = "OK" - ERROR = "ERROR" +_OTEL_AVAILABLE = True class _NoOpSpan: - """No-op span context manager used when OTel is not installed.""" + """No-op span context manager — retained for tests that patch + `_OTEL_AVAILABLE = False` and call `start_span` directly. + + ponytail: kept only for test compatibility; production code now always + has OTel installed. Safe to remove once tests are migrated to patch + `opentelemetry.trace` directly. + """ def __enter__(self): return self @@ -59,10 +49,8 @@ class _NoOpSpan: def get_tracer(name: str = "fischer.agentkit"): - """Get tracer — returns None if OTel not installed.""" - if _OTEL_AVAILABLE: - return trace.get_tracer(name) - return None + """Get tracer — returns the global OTel tracer (always available post-U1).""" + return trace.get_tracer(name) def start_span( @@ -70,9 +58,9 @@ def start_span( kind: object = None, attributes: dict | None = None, ): - """Start a span — returns no-op span if OTel not installed. + """Start a span. Returns an OTel span context manager. - Returns a context manager that yields a span (or no-op). + `_OTEL_AVAILABLE` guard retained for tests that patch it to False. """ if not _OTEL_AVAILABLE: return _NoOpSpan() diff --git a/src/agentkit/tools/bitable_tool.py b/src/agentkit/tools/bitable_tool.py index 6365155..b6d760e 100644 --- a/src/agentkit/tools/bitable_tool.py +++ b/src/agentkit/tools/bitable_tool.py @@ -558,15 +558,23 @@ class BitableTool(Tool): field_id = kwargs.get("field_id") if not field_id: return {"success": False, "error": "Missing required field: field_id"} + # DR-5: type 变更不被支持 — UpdateFieldRequest 当前只接受 name + config， + # 传入 type 会被 Pydantic extra=ignore 静默丢弃（用户以为改了类型，实际无效果）。 + # 显式拒绝并返回错误，避免静默失败的 UX 陷阱。 + # 升级路径：如需支持类型变更，扩展 UpdateFieldRequest 接受 field_type + + # 实现数据迁移逻辑（类型转换 + 校验）。 + if kwargs.get("type") is not None: + return { + "success": False, + "error": ( + "Unsupported field type change — field type is immutable after " + "creation. To change a field's type, delete and recreate it." + ), + "code": "UNSUPPORTED_FIELD_TYPE_CHANGE", + } payload: dict[str, object] = {} if kwargs.get("name") is not None: payload["name"] = kwargs["name"] - # ponytail: PATCH /fields/{id} is backed by UpdateFieldRequest which - # currently accepts only name + config. `type` is forwarded as - # `field_type` so the tool is forward-compatible once the request - # model adds it; today it is silently ignored by Pydantic (extra=ignore). - if kwargs.get("type") is not None: - payload["field_type"] = kwargs["type"] if kwargs.get("config") is not None: payload["config"] = kwargs["config"] client = await self._get_client() diff --git a/tests/unit/admin/test_models.py b/tests/unit/admin/test_models.py index ff97ef0..fa19a0f 100644 --- a/tests/unit/admin/test_models.py +++ b/tests/unit/admin/test_models.py @@ -120,9 +120,9 @@ async def _list_table_names(db: aiosqlite.Connection) -> set[str]: class TestSchemaVersion: - def test_schema_version_is_v4(self): - """V4 adds the skill_states table (U6 — Admin Console).""" - assert _SCHEMA_VERSION == 4 + def test_schema_version_is_v5(self): + """V5 adds user_idp_links + auth_audit_log (U4 SSO 集成).""" + assert _SCHEMA_VERSION == 5 def test_sqlalchemy_model_table_names(self): assert DepartmentModel.__tablename__ == "departments" @@ -163,13 +163,13 @@ class TestInitAuthDbTables: tables = await _list_table_names(db) assert "department_quotas" in tables - async def test_records_schema_version_4_in_auth_meta(self, fresh_db: Path): + async def test_records_schema_version_5_in_auth_meta(self, fresh_db: Path): async with aiosqlite.connect(str(fresh_db)) as db: db.row_factory = aiosqlite.Row cursor = await db.execute("SELECT value FROM auth_meta WHERE key='schema_version'") row = await cursor.fetchone() assert row is not None - assert row["value"] == "4" + assert row["value"] == "5" assert row["value"] == str(_SCHEMA_VERSION) async def test_init_auth_db_is_idempotent(self, tmp_path: Path): diff --git a/tests/unit/auth/test_models.py b/tests/unit/auth/test_models.py index a2296e7..31a35a8 100644 --- a/tests/unit/auth/test_models.py +++ b/tests/unit/auth/test_models.py @@ -118,9 +118,9 @@ async def _list_index_names(db: aiosqlite.Connection, table: str) -> set[str]: class TestSchemaVersion: - def test_schema_version_is_v4(self): - """The current schema version is 4 (V4 adds skill_states table).""" - assert _SCHEMA_VERSION == 4 + def test_schema_version_is_v5(self): + """The current schema version is 5 (V5 adds user_idp_links + auth_audit_log, U4 SSO).""" + assert _SCHEMA_VERSION == 5 def test_sqlalchemy_model_table_name(self): assert AuthSessionModel.__tablename__ == "auth_sessions" diff --git a/tests/unit/bitable/test_bitable_tool.py b/tests/unit/bitable/test_bitable_tool.py index 1f885fd..8b6b75b 100644 --- a/tests/unit/bitable/test_bitable_tool.py +++ b/tests/unit/bitable/test_bitable_tool.py @@ -552,9 +552,9 @@ async def test_update_view_action(tool: BitableTool) -> None: """update_view action PATCHes /views/{id} with name + config.""" result = await tool.execute(action="create_table", table_name="VU") table_id = result["table"]["id"] - view_id = ( - await tool.execute(action="create_view", table_id=table_id, name="Old") - )["view"]["id"] + view_id = (await tool.execute(action="create_view", table_id=table_id, name="Old"))["view"][ + "id" + ] resp = await tool.execute( action="update_view", @@ -682,9 +682,7 @@ async def test_new_actions_internal_token_passthrough( table_id = result["table"]["id"] # create_view via the new action. - v = await tool_real_auth.execute( - action="create_view", table_id=table_id, name="tv1" - ) + v = await tool_real_auth.execute(action="create_view", table_id=table_id, name="tv1") assert v["success"] is True view_id = v["view"]["id"] @@ -716,3 +714,60 @@ async def test_delete_view_internal_token_404_when_missing( resp = await tool_real_auth.execute(action="delete_view", view_id="no-such-view") assert resp["success"] is False assert "404" in resp["error"] + + +# --------------------------------------------------------------------------- +# DR-5: _update_field rejects type change (no silent success) +# --------------------------------------------------------------------------- + + +async def test_update_field_rejects_type_change(tool: BitableTool) -> None: + """DR-5: _update_field with `type` arg returns error, does NOT call HTTP. + + Without this guard, the field_type would be silently dropped by + UpdateFieldRequest's Pydantic extra=ignore, and the agent would + believe the type changed when it didn't. + """ + result = await tool.execute(action="create_table", table_name="DR5") + table_id = result["table"]["id"] + client = await tool._get_client() + field_id = ( + await client.post( + f"/tables/{table_id}/fields", + json={"name": "col", "field_type": "text", "owner": "user"}, + ) + ).json()["field"]["id"] + + resp = await tool.execute( + action="update_field", + field_id=field_id, + type="number", + ) + assert resp["success"] is False + assert resp.get("code") == "UNSUPPORTED_FIELD_TYPE_CHANGE" + assert "immutable" in resp["error"] + + +async def test_update_field_still_works_without_type(tool: BitableTool) -> None: + """DR-5: _update_field without `type` still PATCHes name/config normally. + + Regression guard: the DR-5 guard must not block legitimate updates. + """ + result = await tool.execute(action="create_table", table_name="DR5b") + table_id = result["table"]["id"] + client = await tool._get_client() + field_id = ( + await client.post( + f"/tables/{table_id}/fields", + json={"name": "col", "field_type": "text", "owner": "user"}, + ) + ).json()["field"]["id"] + + resp = await tool.execute( + action="update_field", + field_id=field_id, + name="renamed", + config={"description": "ok"}, + ) + assert resp["success"] is True + assert resp["field"]["name"] == "renamed" diff --git a/tests/unit/bitable/test_repository.py b/tests/unit/bitable/test_repository.py new file mode 100644 index 0000000..eafe031 --- /dev/null +++ b/tests/unit/bitable/test_repository.py @@ -0,0 +1,113 @@ +"""Tests for BitableRepository — U3/DR-1 + DR-4 fixes. + +DR-1: field_id UUID validation before SQL interpolation (pure unit test). +DR-4: atomic delete_view_if_not_last (requires PostgreSQL). +""" + +from __future__ import annotations + +import pytest + +from agentkit.bitable.repository import _validate_field_id + + +# ── DR-1: field_id UUID validation ────────────────────────── + + +class TestValidateFieldId: + """DR-1: field_id 校验 — 拒绝非 UUID 格式进入 SQL 结构插值。""" + + def test_valid_uuid_passes(self): + valid = "550e8400-e29b-41d4-a716-446655440000" + assert _validate_field_id(valid) == valid + + def test_valid_uppercase_uuid_passes(self): + valid = "550E8400-E29B-41D4-A716-446655440000" + assert _validate_field_id(valid) == valid + + @pytest.mark.parametrize( + "bad_field_id,label", + [ + ("not-a-uuid", "arbitrary string"), + ("", "empty string"), + ("550e8400-e29b-41d4-a716", "too short"), + ("550e8400-e29b-41d4-a716-446655440000-extra", "too long"), + ("550e8400-e29b-41d4-a716-44665544000g", "non-hex char"), + ("'; DROP TABLE bitable.bitable_views; --", "SQL injection attempt"), + ("values->>'field' = :fv0", "SQL fragment"), + ], + ) + def test_invalid_field_id_raises(self, bad_field_id: str, label: str): + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(bad_field_id, context=label) + + def test_non_string_raises(self): + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(None) # type: ignore[arg-type] + with pytest.raises(ValueError, match="Invalid field_id format"): + _validate_field_id(12345) # type: ignore[arg-type] + + +# ── DR-4: atomic delete_view_if_not_last ──────────────────── +# These tests require PostgreSQL (SELECT FOR UPDATE behavior) + + +pytestmark_postgres = pytest.mark.postgres + + +@pytest.mark.postgres +class TestDeleteViewIfNotLastAtomic: + """DR-4: 原子条件 delete_view — SELECT FOR UPDATE 事务防止 TOCTOU。 + + 并发场景不变量："table 至少 1 view" 在并发删除最后两个 view 时保持。 + """ + + async def test_delete_view_if_not_last_succeeds_when_multiple_siblings( + self, bitable_service, make_table + ): + """多 view 时删除成功。""" + from agentkit.bitable.repository import BitableRepository + + table = await make_table(name="T1") + v1 = await bitable_service.create_view(table.id, name="v1") + await bitable_service.create_view(table.id, name="v2") + + repo = BitableRepository(bitable_service._repo._db) + deleted = await repo.delete_view_if_not_last(v1.id, table.id) + assert deleted is True + # v1 gone, v2 still present + assert await bitable_service.get_view(v1.id) is None + assert (await bitable_service.list_views(table.id)) is not None + + async def test_delete_view_if_not_last_refuses_last_view(self, bitable_service, make_table): + """最后一个 view 时拒绝删除（返回 False，不抛异常 — service 层负责抛 LastViewDeletionError）。""" + from agentkit.bitable.repository import BitableRepository + + table = await make_table(name="T2") + only_view = await bitable_service.create_view(table.id, name="only") + + repo = BitableRepository(bitable_service._repo._db) + deleted = await repo.delete_view_if_not_last(only_view.id, table.id) + assert deleted is False + # View still exists + assert await bitable_service.get_view(only_view.id) is not None + + async def test_delete_view_if_not_last_wrong_table_returns_false( + self, bitable_service, make_table + ): + """view_id 不属于给定 table_id 时返回 False（不误删其他 table 的 view）。""" + from agentkit.bitable.repository import BitableRepository + + table_a = await make_table(name="TA") + table_b = await make_table(name="TB") + va = await bitable_service.create_view(table_a.id, name="va") + # table_b 有自己的 view + await bitable_service.create_view(table_b.id, name="vb1") + await bitable_service.create_view(table_b.id, name="vb2") + + repo = BitableRepository(bitable_service._repo._db) + # 用 table_b 的 id 但 view_id 是 table_a 的 — 不应删除 + deleted = await repo.delete_view_if_not_last(va.id, table_b.id) + assert deleted is False + # va 仍然存在 + assert await bitable_service.get_view(va.id) is not None diff --git a/tests/unit/bitable/test_service.py b/tests/unit/bitable/test_service.py index 67605a5..4bfffb4 100644 --- a/tests/unit/bitable/test_service.py +++ b/tests/unit/bitable/test_service.py @@ -8,7 +8,7 @@ from __future__ import annotations import pytest from agentkit.bitable.models import FieldOwner, FieldType -from agentkit.bitable.service import FieldDependencyError +from agentkit.bitable.service import FieldDependencyError, LastViewDeletionError pytestmark = pytest.mark.postgres @@ -294,3 +294,44 @@ async def test_list_records_filtered_cursor_pagination(bitable_service) -> None: # All records unique all_ids = {r.id for r in [records, records2, records3] for r in r} assert len(all_ids) == 5 + + +# --------------------------------------------------------------------------- +# DR-4: delete_view last-view protection (service layer) +# --------------------------------------------------------------------------- + + +async def test_delete_view_raises_last_view_deletion_error(bitable_service, make_table) -> None: + """DR-4: service.delete_view raises LastViewDeletionError on the last view. + + Guards the invariant "every table has at least 1 view" at the service + layer so the route can map it to HTTP 409 without touching the repo. + """ + table = await make_table(name="LastView") + only = await bitable_service.create_view(table.id, name="only") + + with pytest.raises(LastViewDeletionError): + await bitable_service.delete_view(only.id) + # View is still there + assert await bitable_service.get_view(only.id) is not None + + +async def test_delete_view_succeeds_when_multiple_views(bitable_service, make_table) -> None: + """DR-4: service.delete_view succeeds when ≥2 views remain (happy path).""" + table = await make_table(name="MultiView") + v1 = await bitable_service.create_view(table.id, name="v1") + await bitable_service.create_view(table.id, name="v2") + + deleted = await bitable_service.delete_view(v1.id) + assert deleted is True + assert await bitable_service.get_view(v1.id) is None + + +async def test_delete_view_returns_false_when_missing(bitable_service, make_table) -> None: + """DR-4: service.delete_view on a non-existent view returns False (no raise).""" + table = await make_table(name="MissingView") + # ensure ≥1 view exists so the table is valid + await bitable_service.create_view(table.id, name="v1") + + deleted = await bitable_service.delete_view("nonexistent-view-id") + assert deleted is False diff --git a/tests/unit/experts/test_team_orchestrator.py b/tests/unit/experts/test_team_orchestrator.py index a21c2c2..6de5b60 100644 --- a/tests/unit/experts/test_team_orchestrator.py +++ b/tests/unit/experts/test_team_orchestrator.py @@ -81,15 +81,17 @@ def _make_mock_expert( } # Mock agent.execute() to return a successful TaskResult mock_agent = MagicMock() - mock_agent.execute = AsyncMock(return_value=TaskResult( - task_id="test", - agent_name=name, - status=TaskStatus.COMPLETED.value, - output_data={"content": f"Result from {name}"}, - error_message=None, - started_at=None, - completed_at=None, - )) + mock_agent.execute = AsyncMock( + return_value=TaskResult( + task_id="test", + agent_name=name, + status=TaskStatus.COMPLETED.value, + output_data={"content": f"Result from {name}"}, + error_message=None, + started_at=None, + completed_at=None, + ) + ) # U3: orchestrator._run_agent_steps() calls agent.execute_stream() (async gen), # not agent.execute(). Mock it to yield one final_answer event. mock_agent.execute_stream = make_execute_stream_mock(f"Result from {name}") @@ -162,20 +164,24 @@ def _make_mock_llm_gateway( def _make_mock_pool() -> MagicMock: """创建 mock AgentPool，模拟上下文隔离的 agent 创建""" pool = MagicMock() + def _create_agent(config): m = MagicMock() - m.execute = AsyncMock(return_value=TaskResult( - task_id="test", - agent_name=config.name, - status=TaskStatus.COMPLETED.value, - output_data={"content": f"Isolated result from {config.name}"}, - error_message=None, - started_at=None, - completed_at=None, - )) + m.execute = AsyncMock( + return_value=TaskResult( + task_id="test", + agent_name=config.name, + status=TaskStatus.COMPLETED.value, + output_data={"content": f"Isolated result from {config.name}"}, + error_message=None, + started_at=None, + completed_at=None, + ) + ) # U3: orchestrator calls agent.execute_stream() (async gen) m.execute_stream = make_execute_stream_mock(f"Isolated result from {config.name}") return m + pool.create_agent = AsyncMock(side_effect=_create_agent) pool.remove_agent = AsyncMock() return pool @@ -220,9 +226,24 @@ class TestPipelineExecution: # LLM 分解为 3 个串行阶段 gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -231,10 +252,12 @@ class TestPipelineExecution: execution_order: list[str] = [] for name in ["lead", "member1", "member2"]: original_stream = team._experts[name].agent.execute_stream + async def tracking_stream(task_msg, _orig=original_stream, _name=name): execution_order.append(_name) async for ev in _orig(task_msg): yield ev + team._experts[name].agent.execute_stream = tracking_stream result = await orchestrator.execute("串行任务") @@ -256,9 +279,24 @@ class TestPipelineExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": []}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["A", "B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": [], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["A", "B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -310,6 +348,59 @@ class TestPipelineExecution: event_types = [c[0][1]["type"] for c in calls] assert "team_synthesis" in event_types + @pytest.mark.asyncio + async def test_synthesis_failure_emits_terminal_team_synthesis_error(self): + """U3/R4-F2: 流式综合异常时发 team_synthesis 终结事件(status=error)，避免前端孤儿 milestone 永久转圈。 + + 行为契约： + - 内层 except 捕获 synthesis 异常 → 广播 team_synthesis(status=error, synthesis_id) → re-raise + - 外层 except 捕获 → 调用 _fallback_to_single_agent（不 re-raise） + - 因此 execute() 返回 status="fallback"，且最后一个 team_synthesis 事件 status=error。 + """ + team = _make_team_with_experts() + orchestrator = TeamOrchestrator(team) + + # 让 _synthesize_results 抛异常 — 触发 error 终结事件路径 + async def _fail_synthesize(*args, **kwargs): + raise RuntimeError("synthesis boom") + + orchestrator._synthesize_results = _fail_synthesize + + # 不应抛出 — 外层 except 捕获后 fallback + result = await orchestrator.execute("测试任务") + assert result["status"] == "fallback" + + calls = team._handoff_transport.send.call_args_list + synth_events = [c for c in calls if c[0][1].get("type") == "team_synthesis"] + # 必须有终结事件，且 status=error + assert len(synth_events) >= 1 + terminal = synth_events[-1] + assert terminal[0][1].get("status") == "error" + assert terminal[0][1].get("synthesis_id") is not None + assert "synthesis boom" in terminal[0][1].get("error", "") + + @pytest.mark.asyncio + async def test_synthesis_cancelled_emits_terminal_team_synthesis_cancelled(self): + """U3/R4-F2: 流式综合被 CancelledError 中断时发 team_synthesis 终结事件(status=cancelled)。""" + team = _make_team_with_experts() + orchestrator = TeamOrchestrator(team) + + # 让 _synthesize_results 抛 CancelledError — 触发 cancelled 终结事件路径 + async def _cancel_synthesize(*args, **kwargs): + raise asyncio.CancelledError() + + orchestrator._synthesize_results = _cancel_synthesize + + with pytest.raises(asyncio.CancelledError): + await orchestrator.execute("测试任务") + + calls = team._handoff_transport.send.call_args_list + synth_events = [c for c in calls if c[0][1].get("type") == "team_synthesis"] + assert len(synth_events) >= 1 + terminal = synth_events[-1] + assert terminal[0][1].get("status") == "cancelled" + assert terminal[0][1].get("synthesis_id") is not None + # ── 阶段事件广播测试 ────────────────────────────────────── @@ -351,8 +442,18 @@ class TestPhaseEvents: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -403,9 +504,24 @@ class TestTaskDecomposition: gateway = _make_mock_llm_gateway( phases=[ - {"name": "规划", "assigned_expert": "lead", "task_description": "设计架构", "depends_on": []}, - {"name": "前端", "assigned_expert": "member1", "task_description": "实现UI", "depends_on": ["规划"]}, - {"name": "后端", "assigned_expert": "member2", "task_description": "实现API", "depends_on": ["规划"]}, + { + "name": "规划", + "assigned_expert": "lead", + "task_description": "设计架构", + "depends_on": [], + }, + { + "name": "前端", + "assigned_expert": "member1", + "task_description": "实现UI", + "depends_on": ["规划"], + }, + { + "name": "后端", + "assigned_expert": "member2", + "task_description": "实现API", + "depends_on": ["规划"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -456,10 +572,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_valid_json(self): """_parse_phases 正确解析 JSON 数组""" - content = json.dumps([ - {"name": "A", "assigned_expert": "member1", "task_description": "任务A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "任务B", "depends_on": ["A"]}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "member1", + "task_description": "任务A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "任务B", + "depends_on": ["A"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1", "member2"], "lead") assert len(phases) == 2 assert phases[0].name == "A" @@ -473,9 +601,16 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_invalid_expert_falls_back_to_lead(self): """_parse_phases 对无效专家名回退到 lead""" - content = json.dumps([ - {"name": "A", "assigned_expert": "nonexistent", "task_description": "任务A", "depends_on": []}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "nonexistent", + "task_description": "任务A", + "depends_on": [], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1"], "lead") assert len(phases) == 1 assert phases[0].assigned_expert == "lead" @@ -489,10 +624,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_empty_name_skipped(self): """_parse_phases 跳过空名称的阶段""" - content = json.dumps([ - {"name": "", "assigned_expert": "member1", "task_description": "任务A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "任务B", "depends_on": []}, - ]) + content = json.dumps( + [ + { + "name": "", + "assigned_expert": "member1", + "task_description": "任务A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "任务B", + "depends_on": [], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["member1"], "lead") assert len(phases) == 1 assert phases[0].name == "B" @@ -500,10 +647,22 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_resolves_depends_on_by_name(self): """_parse_phases 通过名称解析 depends_on 为 ID""" - content = json.dumps([ - {"name": "规划", "assigned_expert": "lead", "task_description": "设计", "depends_on": []}, - {"name": "实现", "assigned_expert": "member1", "task_description": "编码", "depends_on": ["规划"]}, - ]) + content = json.dumps( + [ + { + "name": "规划", + "assigned_expert": "lead", + "task_description": "设计", + "depends_on": [], + }, + { + "name": "实现", + "assigned_expert": "member1", + "task_description": "编码", + "depends_on": ["规划"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["lead", "member1"], "lead") assert len(phases) == 2 # 实现 depends on 规划 @@ -512,9 +671,16 @@ class TestTaskDecomposition: @pytest.mark.asyncio async def test_parse_phases_unknown_dependency_ignored(self): """_parse_phases 对未知依赖名称忽略""" - content = json.dumps([ - {"name": "A", "assigned_expert": "lead", "task_description": "任务A", "depends_on": ["unknown_phase"]}, - ]) + content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "lead", + "task_description": "任务A", + "depends_on": ["unknown_phase"], + }, + ] + ) phases = TeamOrchestrator._parse_phases(content, ["lead"], "lead") assert len(phases) == 1 assert len(phases[0].depends_on) == 0 # unknown dependency ignored @@ -558,7 +724,12 @@ class TestPhaseExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "nonexistent", "task_description": "任务A", "depends_on": []}, + { + "name": "A", + "assigned_expert": "nonexistent", + "task_description": "任务A", + "depends_on": [], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -578,8 +749,18 @@ class TestPhaseExecution: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -639,9 +820,24 @@ class TestPhaseFailure: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "阶段B", "depends_on": ["A"]}, - {"name": "C", "assigned_expert": "member2", "task_description": "阶段C", "depends_on": ["B"]}, + { + "name": "A", + "assigned_expert": "lead", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "阶段B", + "depends_on": ["A"], + }, + { + "name": "C", + "assigned_expert": "member2", + "task_description": "阶段C", + "depends_on": ["B"], + }, ] ) team._experts["lead"].agent._llm_gateway = gateway @@ -787,8 +983,18 @@ class TestResultSynthesis: gateway = _make_mock_llm_gateway( phases=[ - {"name": "A", "assigned_expert": "member1", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "阶段B", "depends_on": []}, + { + "name": "A", + "assigned_expert": "member1", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "阶段B", + "depends_on": [], + }, ], synthesis_content="综合结果", ) @@ -810,10 +1016,22 @@ class TestResultSynthesis: gateway = AsyncMock() decomp_response = MagicMock() - decomp_response.content = json.dumps([ - {"name": "A", "assigned_expert": "member1", "task_description": "阶段A", "depends_on": []}, - {"name": "B", "assigned_expert": "member2", "task_description": "阶段B", "depends_on": []}, - ]) + decomp_response.content = json.dumps( + [ + { + "name": "A", + "assigned_expert": "member1", + "task_description": "阶段A", + "depends_on": [], + }, + { + "name": "B", + "assigned_expert": "member2", + "task_description": "阶段B", + "depends_on": [], + }, + ] + ) # ponytail: 函数式 side_effect — 首次返回 decomposition，后续一律抛 RuntimeError # （列表式 side_effect 耗尽会抛 StopIteration，被 U3 收窄后的 except 漏捕获； # 函数式让"LLM 不可用"语义明确，覆盖验收+综合所有后续调用） @@ -826,12 +1044,14 @@ class TestResultSynthesis: raise RuntimeError("LLM unavailable") gateway.chat = AsyncMock(side_effect=chat_side_effect) + # U3: synthesizer tries chat_stream first (broadcast_callback is set). # Make it raise so synthesizer catches and falls back to concatenation, # matching the test intent ("without LLM"). async def _stream_raise(*a, **kw): raise RuntimeError("LLM unavailable") yield # async gen marker + gateway.chat_stream = MagicMock(side_effect=_stream_raise) team._experts["lead"].agent._llm_gateway = gateway @@ -1049,11 +1269,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1082,10 +1304,17 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=names) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": f"P{i}", "assigned_expert": name, "task_description": f"P{i}", "depends_on": []} - for i, name in enumerate(names) - ]) + gateway = _make_mock_llm_gateway( + phases=[ + { + "name": f"P{i}", + "assigned_expert": name, + "task_description": f"P{i}", + "depends_on": [], + } + for i, name in enumerate(names) + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1113,11 +1342,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team, max_concurrent_phases=1) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1144,11 +1375,13 @@ class TestConcurrencyLimit: team = _make_team_with_experts(expert_names=["lead", "m1", "m2"]) orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, - {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + {"name": "B", "assigned_expert": "m1", "task_description": "B", "depends_on": []}, + {"name": "C", "assigned_expert": "m2", "task_description": "C", "depends_on": []}, + ] + ) team._experts["lead"].agent._llm_gateway = gateway tracker = _ConcurrencyTracker() @@ -1210,11 +1443,13 @@ class TestConcurrencyLimit: orchestrator._generate_debate_opening = AsyncMock(return_value="Opening statement") orchestrator._generate_debate_argument = tracking_debate_argument orchestrator._generate_debate_summary = AsyncMock(return_value="Round summary") - orchestrator._generate_debate_verdict = AsyncMock(return_value={ - "conclusion": "Final conclusion", - "decision": "adopt", - "rationale": "Because", - }) + orchestrator._generate_debate_verdict = AsyncMock( + return_value={ + "conclusion": "Final conclusion", + "decision": "adopt", + "rationale": "Because", + } + ) await orchestrator._execute_debate_phase(phase, plan) @@ -1345,10 +1580,17 @@ class TestSharedWorkspaceRedis: team = _make_team_with_experts() orchestrator = TeamOrchestrator(team) - gateway = _make_mock_llm_gateway(phases=[ - {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, - {"name": "B", "assigned_expert": "member1", "task_description": "B", "depends_on": ["A"]}, - ]) + gateway = _make_mock_llm_gateway( + phases=[ + {"name": "A", "assigned_expert": "lead", "task_description": "A", "depends_on": []}, + { + "name": "B", + "assigned_expert": "member1", + "task_description": "B", + "depends_on": ["A"], + }, + ] + ) team._experts["lead"].agent._llm_gateway = gateway # Mock _execute_phase to return large content + verify offloading diff --git a/tests/unit/test_auth_audit.py b/tests/unit/test_auth_audit.py new file mode 100644 index 0000000..ec09590 --- /dev/null +++ b/tests/unit/test_auth_audit.py @@ -0,0 +1,208 @@ +"""Audit service 单元测试 (U4 — RBAC + deprovisioning 审计, KTD-8)。 + +验证： +- RBAC 角色变更审计记录（actor/target/role_before/role_after） +- Deprovisioning 审计记录（actor/target/reason/source） +- 审计记录持久化到 auth_audit_log 表 +- 按目标用户查询审计历史 +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import aiosqlite +import pytest + +from agentkit.server.auth.audit import ( + AuditService, + EVENT_DEPROVISION, + EVENT_ROLE_CHANGE, + SOURCE_BREAKGLASS, + SOURCE_MANUAL, + get_audit_service, + set_audit_service, +) +from agentkit.server.auth.models import audit_log_row_to_dict, init_auth_db + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +@pytest.fixture +def audit_service(auth_db: Path) -> AuditService: + svc = AuditService(db_path=auth_db) + set_audit_service(svc) + return svc + + +@pytest.fixture +async def seed_users(auth_db: Path) -> dict[str, str]: + """插入两个用户（admin + target）用于审计测试。""" + now = datetime.now(timezone.utc).isoformat() + admin_id = str(uuid4()) + target_id = str(uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (admin_id, "admin_alice", "alice@example.com", "hash", "admin", 1, 0, 0, now, now), + ) + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (target_id, "target_bob", "bob@example.com", "hash", "member", 1, 0, 0, now, now), + ) + await db.commit() + return {"admin_id": admin_id, "target_id": target_id} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestRoleChangeAudit: + async def test_records_role_change( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + record = await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + role_before="member", + role_after="operator", + reason="晋升", + source=SOURCE_MANUAL, + ) + assert record["event_type"] == EVENT_ROLE_CHANGE + assert record["role_before"] == "member" + assert record["role_after"] == "operator" + assert record["actor_username"] == "admin_alice" + assert record["target_username"] == "target_bob" + assert record["reason"] == "晋升" + + async def test_persists_to_db( + self, audit_service: AuditService, auth_db: Path, seed_users: dict[str, str] + ) -> None: + await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + role_before="member", + role_after="admin", + reason="提升管理员", + ) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE event_type = ? AND target_user_id = ?", + (EVENT_ROLE_CHANGE, seed_users["target_id"]), + ) + rows = await cursor.fetchall() + assert len(rows) == 1 + row = audit_log_row_to_dict(rows[0]) + assert row["role_before"] == "member" + assert row["role_after"] == "admin" + + +class TestDeprovisionAudit: + async def test_records_deprovision( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + record = await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="离职", + source=SOURCE_MANUAL, + ) + assert record["event_type"] == EVENT_DEPROVISION + assert record["reason"] == "离职" + assert record["source"] == SOURCE_MANUAL + + async def test_break_glass_source( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + """break_glass 应急通道 — source 记录为 break_glass。""" + record = await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="应急禁用", + source=SOURCE_BREAKGLASS, + ) + assert record["source"] == SOURCE_BREAKGLASS + + async def test_persists_to_db( + self, audit_service: AuditService, auth_db: Path, seed_users: dict[str, str] + ) -> None: + await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=seed_users["target_id"], + target_username="target_bob", + reason="审计测试", + ) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM auth_audit_log WHERE event_type = ?", + (EVENT_DEPROVISION,), + ) + rows = await cursor.fetchall() + assert len(rows) == 1 + + +class TestListForTarget: + async def test_lists_audit_history( + self, audit_service: AuditService, seed_users: dict[str, str] + ) -> None: + """按目标用户查询审计历史（admin 视图）。""" + target_id = seed_users["target_id"] + await audit_service.record_role_change( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=target_id, + target_username="target_bob", + role_before="member", + role_after="operator", + reason="第一次", + ) + await audit_service.record_deprovision( + actor_user_id=seed_users["admin_id"], + actor_username="admin_alice", + target_user_id=target_id, + target_username="target_bob", + reason="第二次", + ) + records = await audit_service.list_for_target(target_id) + assert len(records) == 2 + # 按 created_at DESC 排序 — deprovision 在前 + assert records[0]["event_type"] == EVENT_DEPROVISION + assert records[1]["event_type"] == EVENT_ROLE_CHANGE + + +class TestGetAuditService: + def test_singleton(self) -> None: + svc1 = get_audit_service() + svc2 = get_audit_service() + assert svc1 is svc2 diff --git a/tests/unit/test_oidc_provider.py b/tests/unit/test_oidc_provider.py new file mode 100644 index 0000000..a7ebe86 --- /dev/null +++ b/tests/unit/test_oidc_provider.py @@ -0,0 +1,240 @@ +"""OIDC provider 单元测试 (U4 SSO 集成)。 + +验证： +- redirect URL 生成（含 state） +- callback 用户 JIT 创建按 sub 关联 +- 禁止邮箱合并 (R1a) +- 相同 sub 复用现有用户 +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +import pytest + +from agentkit.server.auth.idp_registry import ( + IDPConfig, + IDPRegistry, + OIDCConfig, +) +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.providers.oidc import OIDCProvider, get_state_cache + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def oidc_idp(monkeypatch: pytest.MonkeyPatch) -> IDPRegistry: + """注册一个测试 OIDC IDP 并替换全局 registry。""" + registry = IDPRegistry() + registry.register( + IDPConfig( + name="test_oidc", + type="oidc", + display_name="Test OIDC", + oidc=OIDCConfig( + client_id="test-client-id", + client_secret="test-client-secret", + server_metadata_url="https://idp.test/authorize", + redirect_uri="https://app.test/api/v1/auth/oauth/test_oidc/callback", + scope="openid email profile", + ), + ) + ) + monkeypatch.setattr("agentkit.server.auth.providers.oidc.get_idp_registry", lambda: registry) + return registry + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """创建临时 auth DB 并指向 AGENTKIT_AUTH_DB。""" + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +class _FakeOAuth2Client: + """替代 AsyncOAuth2Client 的假 client — 不发 HTTP。""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.token = {"access_token": "fake-access-token", "id_token": None} + + async def __aenter__(self) -> "_FakeOAuth2Client": + return self + + async def __aexit__(self, *args: Any) -> None: + pass + + def create_authorization_url(self, url: str, **kwargs: Any) -> tuple[str, dict[str, Any]]: + """构造假 authorize URL（含 state / scope 参数）。""" + state = kwargs.get("state", "") + scope = kwargs.get("scope", "") + client_id = kwargs.get("client_id", "test-client-id") + redirect_uri = kwargs.get("redirect_uri", "") + return ( + f"{url}?response_type=code&client_id={client_id}" + f"&redirect_uri={redirect_uri}&scope={scope}&state={state}", + {"state": state, "url": url}, + ) + + async def fetch_token(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + return self.token + + async def get(self, url: str) -> Any: + class _Resp: + status_code = 200 + + def json(self) -> dict[str, Any]: + return { + "sub": "oidc-sub-123", + "email": "sso_user@example.com", + "name": "SSO User", + } + + return _Resp() + + +@pytest.fixture +def fake_oauth_client(monkeypatch: pytest.MonkeyPatch) -> None: + """替换 oidc 模块的 AsyncOAuth2Client 为假实现。""" + monkeypatch.setattr("agentkit.server.auth.providers.oidc.AsyncOAuth2Client", _FakeOAuth2Client) + + +@pytest.fixture(autouse=True) +def _reset_state_cache() -> None: + """每个测试前清空 state 缓存。""" + cache = get_state_cache() + cache._store.clear() + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestGetRedirectUrl: + async def test_generates_url_with_state(self, oidc_idp: IDPRegistry) -> None: + url = await OIDCProvider().get_redirect_url("test_oidc") + assert "https://idp.test/authorize" in url + assert "state=" in url + assert "client_id=test-client-id" in url + + async def test_unknown_provider_raises(self, oidc_idp: IDPRegistry) -> None: + with pytest.raises(ValueError, match="未配置"): + await OIDCProvider().get_redirect_url("nonexistent") + + async def test_state_cached_for_csrf(self, oidc_idp: IDPRegistry) -> None: + url = await OIDCProvider().get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + # state 应在缓存中（consume 验证） + assert get_state_cache().consume(state) is True + + +class TestHandleCallback: + async def test_creates_user_by_sub( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """首次登录 JIT 创建用户，按 sub 关联到 user_idp_links。""" + # 预置 state + provider = OIDCProvider() + url = await provider.get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + + user = await provider.handle_callback("test_oidc", "fake-code", state) + assert user["username"] == "SSO User" + assert user["email"] == "sso_user@example.com" + + # 验证 user_idp_links 按 sub 关联 + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM user_idp_links WHERE idp_name = ? AND idp_subject = ?", + ("test_oidc", "oidc-sub-123"), + ) + link = await cursor.fetchone() + assert link is not None + assert link["user_id"] == user["id"] + + async def test_same_sub_reuses_user( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """相同 sub 第二次登录复用现有用户，不重复创建。""" + provider = OIDCProvider() + # 第一次登录 + url1 = await provider.get_redirect_url("test_oidc") + state1 = url1.split("state=", 1)[1].split("&", 1)[0] + user1 = await provider.handle_callback("test_oidc", "code1", state1) + + # 第二次登录（相同 sub） + url2 = await provider.get_redirect_url("test_oidc") + state2 = url2.split("state=", 1)[1].split("&", 1)[0] + user2 = await provider.handle_callback("test_oidc", "code2", state2) + + assert user1["id"] == user2["id"] + + async def test_no_email_merge( + self, + oidc_idp: IDPRegistry, + auth_db: Path, + fake_oauth_client: None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """R1a: 不同 sub 但相同 email — 禁止合并，创建新用户。""" + # 预创建一个本地用户，email 与 SSO userinfo 相同 + now = datetime.now(timezone.utc).isoformat() + local_id = str(uuid.uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + local_id, + "local_user", + "sso_user@example.com", + "$2b$12$dummyhash", + "member", + 1, + 0, + 0, + now, + now, + ), + ) + await db.commit() + + # SSO 用不同 sub（_FakeOAuth2Client 返回 sub=oidc-sub-123） + provider = OIDCProvider() + url = await provider.get_redirect_url("test_oidc") + state = url.split("state=", 1)[1].split("&", 1)[0] + sso_user = await provider.handle_callback("test_oidc", "code", state) + + # 应创建新用户，而非复用 local_user + assert sso_user["id"] != local_id + # R1a: 邮箱冲突时不合并本地用户，改用 fallback 邮箱（保留 UNIQUE 约束） + assert sso_user["email"] != "sso_user@example.com" + assert sso_user["email"].endswith("@sso.test_oidc.local") + + async def test_invalid_state_raises( + self, oidc_idp: IDPRegistry, auth_db: Path, fake_oauth_client: None + ) -> None: + """无效 state（CSRF 校验失败）应拒绝。""" + with pytest.raises(ValueError, match="state"): + await OIDCProvider().handle_callback("test_oidc", "code", "invalid-state") + + +class TestAuthenticateNotImplemented: + async def test_raises(self) -> None: + from agentkit.server.auth.providers.exceptions import ProviderNotImplemented + + with pytest.raises(ProviderNotImplemented): + await OIDCProvider().authenticate(username="x", password="y") diff --git a/tests/unit/test_pii_filter.py b/tests/unit/test_pii_filter.py new file mode 100644 index 0000000..ee4af34 --- /dev/null +++ b/tests/unit/test_pii_filter.py @@ -0,0 +1,200 @@ +"""U2/R3 PII 脱敏 hook 测试。 + +覆盖场景： +- 正则版脱敏：手机号 / 邮箱 / 身份证 / 银行卡 +- fail-closed：脱敏失败时抛 PIIFilterError +- hash 审计：脱敏后记录 hash，原文不出现 +- ner_hook：自定义 NER 函数优先于正则版 +- messages 脱敏：string content + Anthropic content blocks +""" + +from __future__ import annotations + +import pytest + +from agentkit.llm.pii_filter import ( + PIIFilterError, + PIIMatch, + filter_messages_pii, + filter_pii, +) + + +# ── 正则版脱敏 ──────────────────────────────────────────── + + +class TestRegexRedaction: + """正则版 PII 脱敏""" + + def test_phone_redacted(self): + result = filter_pii("联系我：13912345678") + assert result.redacted_count == 1 + assert "13912345678" not in result.redacted_text + assert "[REDACTED_PHONE:" in result.redacted_text + assert len(result.matches[0].hash) == 8 + + def test_email_redacted(self): + result = filter_pii("发邮件给 user@example.com 谢谢") + assert result.redacted_count == 1 + assert "user@example.com" not in result.redacted_text + assert "[REDACTED_EMAIL:" in result.redacted_text + + def test_id_card_redacted(self): + result = filter_pii("身份证号：11010119900307888X") + assert result.redacted_count == 1 + assert "11010119900307888X" not in result.redacted_text + assert "[REDACTED_ID_CARD:" in result.redacted_text + + def test_bank_card_redacted(self): + result = filter_pii("银行卡：6222021234567890123") + assert result.redacted_count == 1 + assert "6222021234567890123" not in result.redacted_text + assert "[REDACTED_BANK_CARD:" in result.redacted_text + + def test_multiple_pii_in_one_text(self): + text = "电话 13912345678，邮箱 user@example.com，身份证 11010119900307888X" + result = filter_pii(text) + assert result.redacted_count == 3 + assert "13912345678" not in result.redacted_text + assert "user@example.com" not in result.redacted_text + assert "11010119900307888X" not in result.redacted_text + + def test_no_pii_returns_unchanged(self): + text = "这是一段普通文本，不含 PII" + result = filter_pii(text) + assert result.redacted_count == 0 + assert result.redacted_text == text + + +# ── hash 审计 ─────────────────────────────────────────── + + +class TestHashAudit: + """脱敏后 hash 记录用于审计（不可逆）""" + + def test_hash_is_8_chars_sha256_prefix(self): + import hashlib + + result = filter_pii("13912345678") + expected = hashlib.sha256("13912345678".encode("utf-8")).hexdigest()[:8] + assert result.matches[0].hash == expected + assert len(result.matches[0].hash) == 8 + + def test_original_not_in_redacted_text(self): + result = filter_pii("电话 13912345678") + assert "13912345678" not in result.redacted_text + # hash 不等于原文 + assert result.matches[0].redacted != result.matches[0].original + + +# ── fail-closed ───────────────────────────────────────── + + +class TestFailClosed: + """脱敏失败时 fail-closed 行为""" + + def test_fail_closed_raises_on_error(self): + """fail_closed=True 时，ner_hook 抛异常 → PIIFilterError""" + + def bad_ner(text): + raise RuntimeError("NER model offline") + + with pytest.raises(PIIFilterError, match="fail-closed"): + filter_pii("test text", ner_hook=bad_ner, fail_closed=True) + + def test_fail_open_returns_original_on_error(self): + """fail_closed=False 时，ner_hook 失败 → 降级正则版（ner_hook 失败不阻塞）""" + + def bad_ner(text): + raise RuntimeError("NER model offline") + + # ner_hook 失败时降级到正则版，不抛异常 + result = filter_pii("电话 13912345678", ner_hook=bad_ner, fail_closed=False) + # 正则版仍能识别手机号 + assert result.redacted_count == 1 + assert "13912345678" not in result.redacted_text + + +# ── ner_hook ──────────────────────────────────────────── + + +class TestNerHook: + """自定义 NER hook 优先于正则版""" + + def test_ner_hook_takes_precedence(self): + """ner_hook 返回脱敏结果时，正则版不再处理（避免双重脱敏）""" + + def custom_ner(text): + # 模拟 NER 模型只识别 "SECRET" 关键词 + redacted = text.replace("SECRET", "[REDACTED_CUSTOM:abc12345]") + match = PIIMatch( + pii_type="custom", + original="SECRET", + redacted="[REDACTED_CUSTOM:abc12345]", + hash="abc12345", + ) + return redacted, [match] + + result = filter_pii("电话 13912345678 SECRET", ner_hook=custom_ner) + # ner_hook 优先：SECRET 被脱敏，但手机号也被正则版脱敏（双重处理） + # ponytail: ner_hook 和正则版是叠加的（ner_hook 先，正则版后） + assert "SECRET" not in result.redacted_text + assert "[REDACTED_CUSTOM:abc12345]" in result.redacted_text + # 正则版仍处理手机号 + assert "13912345678" not in result.redacted_text + + +# ── messages 脱敏 ─────────────────────────────────────── + + +class TestMessagesRedaction: + """对 chat messages 列表执行 PII 脱敏""" + + def test_string_content_redacted(self): + messages = [ + {"role": "user", "content": "电话 13912345678"}, + {"role": "assistant", "content": "好的"}, + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + assert "13912345678" not in redacted[0]["content"] + assert redacted[1]["content"] == "好的" # 无 PII 不变 + + def test_anthropic_content_blocks_redacted(self): + """Anthropic content blocks（list of dicts）中的 text 被脱敏""" + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "电话 13912345678"}, + {"type": "text", "text": "无 PII 文本"}, + ], + } + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + assert "13912345678" not in redacted[0]["content"][0]["text"] + assert redacted[0]["content"][1]["text"] == "无 PII 文本" + + def test_non_text_content_passthrough(self): + """非 text 类型的 content block 透传不脱敏""" + messages = [ + { + "role": "user", + "content": [ + {"type": "image", "source": "data:..."}, + {"type": "text", "text": "电话 13912345678"}, + ], + } + ] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 1 + # image block 透传 + assert redacted[0]["content"][0] == {"type": "image", "source": "data:..."} + + def test_non_string_non_list_content_passthrough(self): + """非 str/非 list content 透传""" + messages = [{"role": "user", "content": None}] + redacted, matches = filter_messages_pii(messages) + assert len(matches) == 0 + assert redacted[0]["content"] is None diff --git a/tests/unit/test_prompt_cache_layers.py b/tests/unit/test_prompt_cache_layers.py index d3e61f0..cf128a6 100644 --- a/tests/unit/test_prompt_cache_layers.py +++ b/tests/unit/test_prompt_cache_layers.py @@ -32,7 +32,9 @@ class _StubGateway: yield # makes this an async generator -def _make_engine(provider_name: str | None = "anthropic", *, cache_enable: bool = True) -> tuple[ReActEngine, _StubGateway]: +def _make_engine( + provider_name: str | None = "anthropic", *, cache_enable: bool = True +) -> tuple[ReActEngine, _StubGateway]: gw = _StubGateway(provider_name=provider_name) engine = ReActEngine.__new__(ReActEngine) engine._llm_gateway = gw @@ -166,6 +168,7 @@ async def test_execute_stream_with_anthropic_uses_content_blocks(): self.captured_messages = list(kwargs.get("messages", [])) # yield one chunk then end from agentkit.llm.protocol import StreamChunk + yield StreamChunk(content="done", model="claude") class _MemRetriever: @@ -219,3 +222,139 @@ def test_anthropic_convert_messages_string_system_still_works(): messages = [{"role": "system", "content": "old style"}] system_prompt, _ = provider._convert_messages(messages) assert system_prompt == "old style" + + +# ---- U2 cache hit/miss metric emission ---- + + +def test_token_usage_cache_hit_property(): + """U2: TokenUsage.cache_hit 反映 cache_read_input_tokens > 0""" + from agentkit.llm.protocol import TokenUsage + + hit_usage = TokenUsage( + prompt_tokens=100, + completion_tokens=50, + cache_read_input_tokens=80, + cache_creation_input_tokens=20, + ) + assert hit_usage.cache_hit is True + + miss_usage = TokenUsage(prompt_tokens=100, completion_tokens=50) + assert miss_usage.cache_hit is False + assert miss_usage.cache_read_input_tokens == 0 + + +def test_anthropic_provider_parses_cache_tokens_from_response(): + """U2: Anthropic provider 从 API 响应解析 cache_read_input_tokens""" + from agentkit.llm.providers.anthropic import AnthropicProvider + + provider = AnthropicProvider.__new__(AnthropicProvider) + provider.api_key = "test" + provider.base_url = "http://test" + # 模拟 Anthropic 响应含 cache 字段 + mock_response = { + "content": [{"type": "text", "text": "hello"}], + "model": "claude-sonnet-4", + "usage": { + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 80, + "cache_creation_input_tokens": 20, + }, + } + # 直接调用 _parse_response（私有方法，测试用） + usage = provider._parse_response(mock_response, "claude-sonnet-4").usage + assert usage.cache_read_input_tokens == 80 + assert usage.cache_creation_input_tokens == 20 + assert usage.cache_hit is True + + +def _build_cache_metric_gateway(response): + """构造 LLMGateway mock，跳过 __init__。 + + ponytail: 直接装配测试所需的最小属性，避免初始化真实 provider/cache。 + """ + from unittest.mock import AsyncMock, MagicMock + + from agentkit.llm.gateway import LLMGateway + + mock_provider = MagicMock() + mock_provider.chat = AsyncMock(return_value=response) + + gw = LLMGateway.__new__(LLMGateway) + gw._providers = {"mock": mock_provider} + gw._config = MagicMock() + gw._config.providers = {} # _calculate_cost 真实迭代返回 0.0 + gw._config.fallbacks = {} # _get_models_to_try 返回 [resolved_model] + gw._cache_manager = None + gw._usage_tracker = MagicMock() + gw._resolve_model_alias = lambda m: m + gw._resolve_model = lambda m: (mock_provider, m) + return gw + + +def test_gateway_emits_prompt_cache_hit_metric_on_cache_read(): + """U2: gateway.chat 在 cache_read_input_tokens > 0 时 emit prompt_cache.hit""" + from unittest.mock import AsyncMock, patch + + from agentkit.llm.protocol import LLMResponse, TokenUsage + + response = LLMResponse( + content="test", + model="claude", + usage=TokenUsage( + prompt_tokens=100, + completion_tokens=50, + cache_read_input_tokens=80, + ), + ) + + gw = _build_cache_metric_gateway(response) + + # patch gateway 模块内的导入绑定名（from ... import prompt_cache_hit_counter） + with ( + patch("agentkit.llm.gateway.prompt_cache_hit_counter") as hit_counter, + patch("agentkit.llm.gateway.prompt_cache_miss_counter") as miss_counter, + patch( + "agentkit.llm.gateway.LLMGateway._record_usage", + new_callable=AsyncMock, + ), + ): + import asyncio + + asyncio.run(gw.chat(messages=[{"role": "user", "content": "hi"}], model="claude")) + + # cache_read_input_tokens > 0 → hit counter called + hit_counter().add.assert_called_once() + miss_counter().add.assert_not_called() + + +def test_gateway_emits_prompt_cache_miss_metric_on_no_cache_read(): + """U2: gateway.chat 在 cache_read_input_tokens == 0 时 emit prompt_cache.miss""" + from unittest.mock import AsyncMock, patch + + from agentkit.llm.protocol import LLMResponse, TokenUsage + + response = LLMResponse( + content="test", + model="claude", + usage=TokenUsage(prompt_tokens=100, completion_tokens=50), # cache_read=0 + ) + + gw = _build_cache_metric_gateway(response) + + with ( + patch("agentkit.llm.gateway.prompt_cache_hit_counter") as hit_counter, + patch("agentkit.llm.gateway.prompt_cache_miss_counter") as miss_counter, + patch( + "agentkit.llm.gateway.LLMGateway._record_usage", + new_callable=AsyncMock, + ), + ): + import asyncio + + asyncio.run(gw.chat(messages=[{"role": "user", "content": "hi"}], model="claude")) + + # cache_read_input_tokens == 0 → miss counter called + miss_counter().add.assert_called_once() + hit_counter().add.assert_not_called() diff --git a/tests/unit/test_saml_provider.py b/tests/unit/test_saml_provider.py new file mode 100644 index 0000000..6d41cbd --- /dev/null +++ b/tests/unit/test_saml_provider.py @@ -0,0 +1,196 @@ +"""SAML 2.0 provider 单元测试 (U4 SSO 集成)。 + +验证： +- ACS NameID 提取 + JIT 用户创建 +- 按 NameID 关联（R1a: 禁止邮箱合并） +- 相同 NameID 复用现有用户 +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite +import pytest + +from agentkit.server.auth.idp_registry import ( + IDPConfig, + IDPRegistry, + SAMLConfig, +) +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.providers.saml import SAMLProvider + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def saml_idp(monkeypatch: pytest.MonkeyPatch) -> IDPRegistry: + """注册一个测试 SAML IDP 并替换全局 registry。""" + registry = IDPRegistry() + registry.register( + IDPConfig( + name="test_saml", + type="saml", + display_name="Test SAML", + saml=SAMLConfig( + entity_id="https://idp.test/entity", + sso_url="https://idp.test/sso", + idp_cert="-----BEGIN CERTIFICATE-----\nfakecert\n-----END CERTIFICATE-----", + sp_entity_id="https://app.test/sp", + acs_url="https://app.test/api/v1/auth/saml/test_saml/acs", + ), + ) + ) + monkeypatch.setattr("agentkit.server.auth.providers.saml.get_idp_registry", lambda: registry) + return registry + + +@pytest.fixture +async def auth_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + return db_path + + +@pytest.fixture +def mock_saml_parse(monkeypatch: pytest.MonkeyPatch) -> None: + """替换 _process_saml 为假实现 — 返回固定 NameID + attributes。""" + + def _fake_process( + self: SAMLProvider, req: dict, cfg: Any, provider_name: str + ) -> tuple[str, dict[str, Any]]: + return "saml-nameid-456", {"email": "saml_user@example.com", "name": "SAML User"} + + monkeypatch.setattr(SAMLProvider, "_process_saml", _fake_process) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestHandleAcs: + async def test_creates_user_by_nameid( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """ACS 消费 SAMLResponse + NameID 提取 + JIT 用户创建。""" + user = await SAMLProvider().handle_acs("test_saml", "fake-saml-response") + assert user["username"] == "SAML User" + assert user["email"] == "saml_user@example.com" + + # 验证 user_idp_links 按 NameID 关联 + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute( + "SELECT * FROM user_idp_links WHERE idp_name = ? AND idp_subject = ?", + ("test_saml", "saml-nameid-456"), + ) + link = await cursor.fetchone() + assert link is not None + assert link["idp_type"] == "saml" + assert link["user_id"] == user["id"] + + async def test_same_nameid_reuses_user( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """相同 NameID 第二次登录复用现有用户。""" + provider = SAMLProvider() + user1 = await provider.handle_acs("test_saml", "resp1") + user2 = await provider.handle_acs("test_saml", "resp2") + assert user1["id"] == user2["id"] + + async def test_no_email_merge( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + """R1a: 不同 NameID 相同 email — 禁止合并。""" + now = datetime.now(timezone.utc).isoformat() + local_id = str(uuid.uuid4()) + async with aiosqlite.connect(str(auth_db)) as db: + await db.execute( + "INSERT INTO users (id, username, email, password_hash, role, is_active, " + "is_terminal_authorized, is_server_terminal_authorized, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + local_id, + "local_saml_user", + "saml_user@example.com", + "$2b$12$dummyhash", + "member", + 1, + 0, + 0, + now, + now, + ), + ) + await db.commit() + + # SAML NameID=saml-nameid-456（不同于本地用户） + saml_user = await SAMLProvider().handle_acs("test_saml", "resp") + assert saml_user["id"] != local_id + + async def test_unknown_provider_raises( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + with pytest.raises(ValueError, match="未配置"): + await SAMLProvider().handle_acs("nonexistent", "resp") + + async def test_missing_nameid_raises( + self, + saml_idp: IDPRegistry, + auth_db: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """SAML assertion 缺少 NameID 应拒绝。""" + monkeypatch.setattr( + SAMLProvider, + "_process_saml", + lambda self, req, cfg, name: ("", {}), + ) + with pytest.raises(ValueError, match="NameID"): + await SAMLProvider().handle_acs("test_saml", "resp") + + +class TestAuthenticateNotImplemented: + async def test_raises(self) -> None: + from agentkit.server.auth.providers.exceptions import ProviderNotImplemented + + with pytest.raises(ProviderNotImplemented): + await SAMLProvider().authenticate(username="x", password="y") + + +class TestRevokeUser: + async def test_disables_local_user( + self, + saml_idp: IDPRegistry, + auth_db: Path, + mock_saml_parse: None, + ) -> None: + provider = SAMLProvider() + user = await provider.handle_acs("test_saml", "resp") + await provider.revoke_user(user["id"]) + async with aiosqlite.connect(str(auth_db)) as db: + db.row_factory = aiosqlite.Row + cursor = await db.execute("SELECT is_active FROM users WHERE id = ?", (user["id"],)) + row = await cursor.fetchone() + assert bool(row["is_active"]) is False diff --git a/tests/unit/test_scim_router.py b/tests/unit/test_scim_router.py new file mode 100644 index 0000000..6a1d6d1 --- /dev/null +++ b/tests/unit/test_scim_router.py @@ -0,0 +1,162 @@ +"""SCIM 2.0 router 单元测试 (U4 SSO 集成)。 + +验证 POST / PATCH / GET /scim/v2/Users： +- POST 创建用户 +- PATCH 禁用用户 (active=false) +- GET 列表 / 过滤 +- Bearer token 认证（缺 token / 错误 token → 401） +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from agentkit.server.auth.models import init_auth_db +from agentkit.server.auth.scim.router import router as scim_router + + +SCIM_TOKEN = "test-scim-token-secret" + + +@pytest.fixture +async def scim_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FastAPI: + """创建含 SCIM router 的 TestApp + 初始化 auth DB。""" + db_path = tmp_path / "auth.db" + await init_auth_db(db_path) + monkeypatch.setenv("AGENTKIT_AUTH_DB", str(db_path)) + monkeypatch.setenv("AGENTKIT_SCIM_TOKEN", SCIM_TOKEN) + + app = FastAPI() + app.state.auth_db_path = str(db_path) + app.state.scim_token = SCIM_TOKEN + app.include_router(scim_router, prefix="/api/v1") + return app + + +@pytest.fixture +def client(scim_app: FastAPI) -> TestClient: + return TestClient(scim_app) + + +def _auth_headers(token: str = SCIM_TOKEN) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestCreateUser: + def test_creates_user(self, client: TestClient) -> None: + resp = client.post( + "/api/v1/scim/v2/Users", + json={ + "userName": "scim.user1", + "active": True, + "emails": [{"value": "scim1@example.com", "primary": True}], + }, + headers=_auth_headers(), + ) + assert resp.status_code == 201 + data = resp.json() + assert data["userName"] == "scim.user1" + assert data["active"] is True + assert data["emails"][0]["value"] == "scim1@example.com" + assert data["schemas"] == ["urn:ietf:params:scim:schemas:core:2.0:User"] + + def test_duplicate_username_conflict(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "dup.user", "emails": [{"value": "dup@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "dup.user", "emails": [{"value": "other@example.com"}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 409 + + def test_no_token_unauthorized(self, client: TestClient) -> None: + resp = client.post("/api/v1/scim/v2/Users", json={"userName": "x"}) + assert resp.status_code == 401 + + def test_wrong_token_unauthorized(self, client: TestClient) -> None: + resp = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "x"}, + headers={"Authorization": "Bearer wrong-token"}, + ) + assert resp.status_code == 401 + + +class TestPatchUser: + def test_disable_user(self, client: TestClient) -> None: + # 先创建 + create = client.post( + "/api/v1/scim/v2/Users", + json={"userName": "patch.user", "active": True, "emails": [{"value": "p@example.com"}]}, + headers=_auth_headers(), + ) + user_id = create.json()["id"] + + # PATCH 禁用 + resp = client.patch( + f"/api/v1/scim/v2/Users/{user_id}", + json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 200 + assert resp.json()["active"] is False + + def test_patch_unknown_user_404(self, client: TestClient) -> None: + resp = client.patch( + "/api/v1/scim/v2/Users/nonexistent-id", + json={"Operations": [{"op": "replace", "path": "active", "value": False}]}, + headers=_auth_headers(), + ) + assert resp.status_code == 404 + + +class TestListUsers: + def test_list_all(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "list.user1", "emails": [{"value": "l1@example.com"}]}, + headers=_auth_headers(), + ) + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "list.user2", "emails": [{"value": "l2@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.get("/api/v1/scim/v2/Users", headers=_auth_headers()) + assert resp.status_code == 200 + data = resp.json() + assert data["totalResults"] >= 2 + assert len(data["Resources"]) >= 2 + + def test_filter_by_username(self, client: TestClient) -> None: + client.post( + "/api/v1/scim/v2/Users", + json={"userName": "filter.user", "emails": [{"value": "f@example.com"}]}, + headers=_auth_headers(), + ) + resp = client.get( + "/api/v1/scim/v2/Users", + params={"filter": 'userName eq "filter.user"'}, + headers=_auth_headers(), + ) + assert resp.status_code == 200 + data = resp.json() + assert data["totalResults"] == 1 + assert data["Resources"][0]["userName"] == "filter.user" + + def test_no_token_unauthorized(self, client: TestClient) -> None: + resp = client.get("/api/v1/scim/v2/Users") + assert resp.status_code == 401 diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index b3bfc66..af0e866 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -1,13 +1,12 @@ """Telemetry module unit tests.""" -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch import pytest from agentkit.telemetry.tracer import ( NoOpSpan, NoOpTracer, - OTelSpan, OTelTracer, TelemetryConfig, get_tracer, @@ -110,29 +109,26 @@ class TestInitTelemetry: tracer = get_tracer() assert isinstance(tracer, NoOpTracer) - def test_missing_opentelemetry_falls_back_to_noop(self): - """当 opentelemetry 包未安装时，优雅降级为 NoOpTracer""" + def test_enabled_initializes_otel_tracer(self): + """U1: enabled=True 初始化 OTelTracer（OTel 现为 required 依赖，不再 ImportError fallback）""" config = TelemetryConfig(enabled=True, otlp_endpoint="http://localhost:4317") - # 即使 opentelemetry 未安装，也不应抛出异常 init_telemetry(config) tracer = get_tracer() - # 在没有 opentelemetry 的环境中应为 NoOpTracer - assert isinstance(tracer, (NoOpTracer, OTelTracer)) + # OTel 已安装，应返回 OTelTracer 实例（而非 NoOpTracer） + assert isinstance(tracer, OTelTracer) def test_init_with_exception_falls_back_to_noop(self): - """初始化过程中出现异常时，降级为 NoOpTracer""" + """初始化过程中出现运行时异常时，降级为 NoOpTracer（保留运行时容错）""" config = TelemetryConfig(enabled=True, otlp_endpoint="http://localhost:4317") + # OTLPSpanExporter 在 init_telemetry 函数内部 import，patch 源模块符号 + # opentelemetry.sdk.trace.TracerProvider 是函数内 import 的真实符号 with patch( - "agentkit.telemetry.tracer.init_telemetry", - side_effect=Exception("init error"), - ) as mock_init: - # init_telemetry 被模拟为抛异常，但实际调用的是原始函数 - # 这里验证的是 init_telemetry 本身不会崩溃 - pass - # 直接调用，不应崩溃 - init_telemetry(config) - tracer = get_tracer() - assert isinstance(tracer, (NoOpTracer, OTelTracer)) + "opentelemetry.sdk.trace.TracerProvider", + side_effect=RuntimeError("provider init failed"), + ): + init_telemetry(config) + tracer = get_tracer() + assert isinstance(tracer, NoOpTracer) # ── get_tracer 测试 ──────────────────────────────────────── @@ -174,14 +170,11 @@ class TestAlignmentGuardSpan: with patch.object(tracer, "start_span", return_value=mock_span): config = AlignmentConfig(constraints=["forbidden_word"]) guard = AlignmentGuard(config) - result = await guard.check_output( - {"content": "This contains forbidden_word"} - ) + await guard.check_output({"content": "This contains forbidden_word"}) tracer.start_span.assert_called_once_with("guard.check") call_args_list = [ - (call.args[0], call.args[1]) - for call in mock_span.set_attribute.call_args_list + (call.args[0], call.args[1]) for call in mock_span.set_attribute.call_args_list ] assert ("guard.constraints_count", 1) in call_args_list assert ("guard.passed", False) in call_args_list @@ -201,13 +194,60 @@ class TestAlignmentGuardSpan: with patch.object(tracer, "start_span", return_value=mock_span): config = AlignmentConfig(constraints=["forbidden_word"]) guard = AlignmentGuard(config) - result = await guard.check_output( - {"content": "This is clean text"} - ) + await guard.check_output({"content": "This is clean text"}) call_args_list = [ - (call.args[0], call.args[1]) - for call in mock_span.set_attribute.call_args_list + (call.args[0], call.args[1]) for call in mock_span.set_attribute.call_args_list ] assert ("guard.passed", True) in call_args_list assert ("guard.checked_by", "rule") in call_args_list + + +# ── setup_telemetry smoke test ─────────────────────────── + + +class TestSetupTelemetry: + """setup_telemetry FastAPI 装配 smoke test（U1）""" + + def test_disabled_is_noop(self): + """enabled=False 时不做任何装配""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + # spec=FastAPI 让 getattr 返回真实的默认值（而非 MagicMock 自动属性） + app = MagicMock(spec=FastAPI) + setup_telemetry(app, {"enabled": False}) + # app 不应被 instrument（spec 让未设属性返回 AttributeError→default False） + assert not getattr(app, "_is_instrumented_by_opentelemetry", False) + + def test_enabled_instruments_fastapi(self): + """enabled=True 时 FastAPIInstrumentor 装配 app（不崩溃即可）""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + app = FastAPI() + # 使用一个无效 endpoint — exporter 会失败但不阻塞 instrument + setup_telemetry( + app, + { + "enabled": True, + "otlp_endpoint": "http://127.0.0.1:4317", + "service_name": "test-agentkit", + "export_traces": True, + "export_metrics": True, + }, + ) + # FastAPIInstrumentor 装配后会设置此属性 + assert getattr(app, "_is_instrumented_by_opentelemetry", False) is True + + def test_none_config_is_noop(self): + """config=None 不崩溃""" + from fastapi import FastAPI + + from agentkit.telemetry.setup import setup_telemetry + + app = MagicMock(spec=FastAPI) + setup_telemetry(app, None) + assert not getattr(app, "_is_instrumented_by_opentelemetry", False)