import asyncio import hmac import json import re import logging import os import uuid from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from fastapi import ( APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect, Security, ) from fastapi.security import APIKeyHeader, APIKeyQuery from pydantic import BaseModel from agentkit.core.config_driven import ConfigDrivenAgent from agentkit.core.event_queue import EventQueue from agentkit.core.protocol import Event, TaskEventType, TaskStatus, TurnEventType from agentkit.core.react import ReActEngine from agentkit.chat.skill_routing import ExecutionMode, SkillRoutingResult from agentkit.chat.request_preprocessor import RequestPreprocessor from agentkit.server.routes.evolution_dashboard import ( _experiences as _dashboard_experiences, DashboardExperience, _broadcast_event as _broadcast_dashboard_event, ) from agentkit.core.fallback import EMPTY_LLM_RESPONSE from agentkit.chat.sqlite_conversation_store import SqliteConversationStore from agentkit.server.task_store import InMemoryTaskStore from agentkit.session.models import MessageRole # ponytail: importing module-private helpers from chat.py because the frontend # WS connects to /api/v1/portal/ws (this router), not /api/v1/chat/ws/{session_id}. # Without this, @board/@team prefixes are never intercepted and board/team cards # never render. Upgrade path: extract these into a shared experts dispatch module. from agentkit.server.routes.chat import _execute_board_meeting, _execute_team_collab logger = logging.getLogger(__name__) router = APIRouter(tags=["portal"]) # Use a project-local SQLite file to avoid read-only sandbox restrictions on ~/.agentkit. _PROJECT_ROOT = Path(__file__).parents[4] _CONVERSATIONS_DB_PATH = Path( os.environ.get("AGENTKIT_CONVERSATIONS_DB", _PROJECT_ROOT / "data" / "conversations.db") ) _CONVERSATIONS_DB_PATH.parent.mkdir(parents=True, exist_ok=True) # Track background ReAct tasks so they are not garbage-collected mid-execution. # Tasks are removed automatically via add_done_callback when they complete. _running_background_tasks: set[asyncio.Task] = set() # --------------------------------------------------------------------------- # API Key Authentication # --------------------------------------------------------------------------- _api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False) _api_key_query = APIKeyQuery(name="api_key", auto_error=False) def _ensure_non_empty(text: str | None) -> str: """Ensure response text is never empty or whitespace-only.""" if text and text.strip(): return text return EMPTY_LLM_RESPONSE async def _emit_event_safe( event_queue: EventQueue | None, event_type: str, task_id: str, session_id: str, data: dict | None = None, ) -> None: """Emit an event to the EventQueue without blocking or raising. The EQ is a side-channel: emit failures must never break the WebSocket flow. All exceptions are swallowed and logged at warning level. Args: event_queue: The EventQueue to emit to (no-op if None) event_type: Event type (see TaskEventType / TurnEventType) task_id: Associated task ID session_id: Associated session ID (conversation_id) data: Optional event payload """ if event_queue is None: return try: event = Event.create( event_type=event_type, task_id=task_id, session_id=session_id, data=data or {}, ) await event_queue.emit(event) except (asyncio.QueueFull, RuntimeError, ConnectionError) as e: logger.warning(f"EventQueue emit failed (type={event_type}): {e}", exc_info=True) # P1 #14 fix: TaskStore sync/async compatibility shim. # InMemoryTaskStore methods are sync; RedisTaskStore methods are async. # These helpers detect and await coroutines so portal.py works with both. async def _task_store_create(store, *args, **kwargs): result = store.create(*args, **kwargs) if asyncio.iscoroutine(result): return await result return result async def _task_store_get(store, *args, **kwargs): result = store.get(*args, **kwargs) if asyncio.iscoroutine(result): return await result return result async def _task_store_update_status(store, *args, **kwargs): result = store.update_status(*args, **kwargs) if asyncio.iscoroutine(result): return await result return result async def _task_store_list_tasks(store, *args, **kwargs): result = store.list_tasks(*args, **kwargs) if asyncio.iscoroutine(result): return await result return result async def _verify_api_key( request: Request, api_key_header: str | None = Security(_api_key_header), api_key_query: str | None = Security(_api_key_query), ) -> None: """Verify API key for REST endpoints. Raises HTTPException if invalid.""" configured_api_key: str | None = None if hasattr(request.app.state, "server_config") and request.app.state.server_config: configured_api_key = request.app.state.server_config.api_key if configured_api_key is None and hasattr(request.app.state, "api_key"): configured_api_key = request.app.state.api_key # If no API key is configured, allow all requests (backwards compat) if configured_api_key is None: return provided = api_key_header or api_key_query if not hmac.compare_digest((provided or "").encode(), configured_api_key.encode()): raise HTTPException( status_code=401, detail="Invalid or missing API key. Provide via X-API-Key header or api_key query parameter.", ) # --------------------------------------------------------------------------- # In-memory Conversation Store # --------------------------------------------------------------------------- @dataclass class ChatMessage: role: str # "user" or "assistant" content: str timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) metadata: dict = field(default_factory=dict) @dataclass class Conversation: id: str messages: list[ChatMessage] = field(default_factory=list) created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) updated_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) # Heartbeat timeout in seconds — 0 disables timeout (for testing) _WS_HEARTBEAT_TIMEOUT = float(os.environ.get("AGENTKIT_WS_TIMEOUT", "120")) _conversation_store = SqliteConversationStore(db_path=_CONVERSATIONS_DB_PATH) class _ConvStoreAsSessionManager: """Adapt SqliteConversationStore to the SessionManager.append_message shape used by chat.py's _execute_board_meeting / _execute_team_collab. ponytail: only append_message is implemented — that's all the board/team intercepts call. If future logic needs more SessionManager methods, add them here then. """ def __init__(self, store: SqliteConversationStore) -> None: self._store = store async def append_message( self, session_id: str, role: MessageRole, content: str, tool_call_id: str | None = None, agent_name: str | None = None, metadata: dict[str, object] | None = None, ) -> None: role_str = role.value if hasattr(role, "value") else str(role) await self._store.add_message(session_id, role_str, content, metadata) _sm_adapter = _ConvStoreAsSessionManager(_conversation_store) # --------------------------------------------------------------------------- # Active portal WebSocket connections by user_id # --------------------------------------------------------------------------- class PortalConnectionManager: """Track active portal WebSocket connections by authenticated user_id. Used by the calendar reminder scheduler (and other user-scoped push features) to deliver real-time messages to a user's open chat tab(s). """ # ponytail: per-user connection cap prevents a single client from # exhausting memory via unbounded WS spawns. 16 covers typical # multi-tab usage. Upgrade path: make configurable via server_config. _MAX_CONNECTIONS_PER_USER = 16 def __init__(self) -> None: # user_id -> list of active WebSocket connections self._connections: dict[str, list[WebSocket]] = {} def add(self, user_id: str, ws: WebSocket) -> None: conns = self._connections.setdefault(user_id, []) if len(conns) >= self._MAX_CONNECTIONS_PER_USER: # Close the oldest connection to make room (FIFO eviction). oldest = conns.pop(0) try: # Best-effort close; ignore failures since the socket may # already be dead. import asyncio asyncio.create_task(oldest.close(code=1008, reason="Connection limit exceeded")) except (ConnectionError, RuntimeError): pass conns.append(ws) def remove(self, user_id: str, ws: WebSocket) -> None: conns = self._connections.get(user_id) if conns is None: return self._connections[user_id] = [w for w in conns if w is not ws] if not self._connections[user_id]: del self._connections[user_id] async def send_json(self, user_id: str, message: dict[str, object]) -> None: """Broadcast a JSON message to all connections for *user_id*. Removes stale connections that fail to send. """ conns = list(self._connections.get(user_id, [])) if not conns: return stale: list[WebSocket] = [] for ws in conns: try: await ws.send_json(message) except (ConnectionError, RuntimeError, asyncio.TimeoutError) as e: logger.debug("Portal WS send failed for user %s (marking stale): %s", user_id, e) stale.append(ws) for ws in stale: self.remove(user_id, ws) portal_connection_manager = PortalConnectionManager() async def send_to_user(user_id: str, message: dict[str, object]) -> None: """Public helper to push a message to all portal WebSockets for a user.""" await portal_connection_manager.send_json(user_id, message) # P1 #9 fix: ReAct event type -> TurnEventType mapping for EQ subscribers. # Preserves the original EQ contract so CLI and other subscribers that # filter on TurnEventType constants (e.g. 'turn.thinking') keep working. _REACT_EVENT_TYPE_MAP: dict[str, str] = { "thinking": TurnEventType.THINKING, "tool_call": TurnEventType.TOOL_CALL, "tool_result": TurnEventType.TOOL_RESULT, "token": TurnEventType.TOKEN, "final_answer": TurnEventType.FINAL_ANSWER, "error": TurnEventType.TURN_COMPLETED, # best-effort mapping "confirmation_request": TurnEventType.STEP, } # --------------------------------------------------------------------------- # History injection helper — configurable limit + optional compression # --------------------------------------------------------------------------- # Maximum history messages to inject (can be overridden by server config) _MAX_HISTORY_MESSAGES = 50 async def _build_history_messages( conv_id: str, limit: int = _MAX_HISTORY_MESSAGES, ) -> list[dict]: """Build conversation history messages for LLM context injection. Returns a list of {"role": "user"|"assistant", "content": ...} dicts representing the conversation history (excluding the current user message, which should be appended separately by the caller). """ try: history = await _conversation_store.get_history(conv_id, limit=limit) except (ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError): return [] # The last message in history is the current user message (just added), # so skip it to avoid duplication. messages = [] for hist_msg in history[:-1]: if hist_msg.role in ("user", "assistant"): messages.append({"role": hist_msg.role, "content": hist_msg.content}) return messages # --------------------------------------------------------------------------- # Capability mapping # --------------------------------------------------------------------------- CAPABILITY_CATEGORIES: dict[str, dict[str, str]] = { "chat": { "display_name": "智能对话", "description": "自然语言交互，自动路由到对应能力", "icon": "MessageOutlined", }, "workflow": { "display_name": "工作流编排", "description": "可视化拖拽编排工作流", "icon": "ApartmentOutlined", }, "knowledge": { "display_name": "知识库", "description": "文档摄取、语义检索、多源RAG", "icon": "BookOutlined", }, "skills": { "display_name": "技能管理", "description": "浏览和管理已注册的技能", "icon": "AppstoreOutlined", }, "terminal": { "display_name": "智能终端", "description": "交互式终端会话和命令执行", "icon": "CodeOutlined", }, "computer_use": { "display_name": "Computer Use", "description": "UI自动化操作和截屏识别", "icon": "DesktopOutlined", }, "evolution": { "display_name": "自进化", "description": "经验积累、避坑预警、路径优化", "icon": "RiseOutlined", }, "settings": { "display_name": "系统设置", "description": "配置LLM、技能、知识库连接", "icon": "SettingOutlined", }, } # --------------------------------------------------------------------------- # Request / Response models # --------------------------------------------------------------------------- class ChatRequest(BaseModel): message: str conversation_id: str | None = None sources: list[str] | None = None skill_name: str | None = None class ChatResponse(BaseModel): conversation_id: str message: str timestamp: str matched_skill: str | None = None routing_method: str | None = None confidence: float | None = None task_id: str | None = None status: str = "completed" class CapabilityInfo(BaseModel): name: str display_name: str description: str icon: str enabled: bool skill_count: int class CapabilitiesResponse(BaseModel): capabilities: list[CapabilityInfo] # --------------------------------------------------------------------------- # Helper: resolve agent + skill for a chat request # --------------------------------------------------------------------------- async def _resolve_for_chat( request: ChatRequest, req: Request ) -> tuple[ ConfigDrivenAgent | None, SkillRoutingResult | None, str | None, str | None, float | None ]: """Resolve agent and routing for a chat request via RequestPreprocessor. Returns (agent, routing_result, matched_skill_name, routing_method, confidence). """ pool = req.app.state.agent_pool skill_registry = req.app.state.skill_registry request_preprocessor: RequestPreprocessor = req.app.state.request_preprocessor matched_skill_name: str | None = None routing_method: str | None = None confidence: float | None = None # Get default tools and system prompt default_tools = [] default_system_prompt = None default_agent = pool.get_agent("default") if default_agent is not None: default_tools = default_agent.get_tools() default_system_prompt = ( getattr(default_agent, "_system_prompt", None) or default_agent.get_system_prompt() ) else: all_skills = skill_registry.list_skills() for skill in all_skills: agent = pool.get_agent(skill.name) if agent is not None: default_tools = agent.get_tools() default_system_prompt = ( getattr(agent, "_system_prompt", None) or agent.get_system_prompt() ) break # If skill_name is explicitly provided in the request, use it directly if request.skill_name: routing_result = await request_preprocessor.preprocess( content=f"@skill:{request.skill_name} {request.message}", skill_registry=skill_registry, default_tools=default_tools, default_system_prompt=default_system_prompt, default_model="default", default_agent_name="default", ) else: # Preprocess via RequestPreprocessor (minimal: @skill prefix + greeting regex + REACT) routing_result = await request_preprocessor.preprocess( content=request.message, skill_registry=skill_registry, default_tools=default_tools, default_system_prompt=default_system_prompt, default_model="default", default_agent_name="default", ) matched_skill_name = routing_result.skill_name or routing_result.agent_name routing_method = routing_result.match_method confidence = routing_result.match_confidence # Get or create agent based on routing result if routing_result.matched and routing_result.skill_name: agent = pool.get_agent(routing_result.skill_name) if agent is None: agent = await pool.create_agent_from_skill(routing_result.skill_name) else: agent = pool.get_agent("default") if agent is None: # Fallback: try to create from first available skill all_skills = skill_registry.list_skills() if all_skills: agent = await pool.create_agent_from_skill(all_skills[0].name) return agent, routing_result, matched_skill_name, routing_method, confidence # --------------------------------------------------------------------------- # Endpoints # --------------------------------------------------------------------------- @router.post("/portal/chat", response_model=ChatResponse) async def chat(request: ChatRequest, req: Request, _auth: None = Depends(_verify_api_key)): """Send a chat message and get a response with RequestPreprocessor routing.""" # If skill_name is explicitly requested but not found, return 404 if request.skill_name: skill_registry = req.app.state.skill_registry if not skill_registry.has_skill(request.skill_name): raise HTTPException(status_code=404, detail=f"Skill '{request.skill_name}' not found") agent, routing_result, matched_skill, routing_method, confidence = await _resolve_for_chat( request, req ) # Create or reuse conversation conv = await _conversation_store.get_or_create(request.conversation_id) await _conversation_store.add_message(conv.id, "user", request.message) llm_gateway = req.app.state.llm_gateway task_id = str(uuid.uuid4()) response_text = "" if routing_result is not None and routing_result.execution_mode == ExecutionMode.DIRECT_CHAT: # DIRECT_CHAT: direct LLM call, no ReAct loop (same as WebSocket path) chat_messages = [] if routing_result.system_prompt: chat_messages.append({"role": "system", "content": routing_result.system_prompt}) chat_messages.append({"role": "user", "content": request.message}) # Inject conversation history history_msgs = await _build_history_messages(conv.id) for hm in history_msgs: chat_messages.insert(-1, hm) response = await llm_gateway.chat( messages=chat_messages, model=routing_result.model or "default", agent_name="default", task_type="chat", ) response_text = _ensure_non_empty(response.content) else: # REACT / SKILL_REACT / REWOO / REFLEXION / PLAN_EXEC / TEAM_COLLAB # Advanced modes (REWOO, REFLEXION, PLAN_EXEC, TEAM_COLLAB) currently # fall back to REACT with a warning. Full integration is tracked separately. if routing_result is not None and routing_result.execution_mode not in ( ExecutionMode.REACT, ExecutionMode.SKILL_REACT, ): logger.warning( f"Execution mode {routing_result.execution_mode.value} not yet supported " f"in portal REST, falling back to REACT" ) react_config = agent.get_react_config() react_engine = getattr(agent, "_react_engine", None) if react_engine is None: react_engine = ReActEngine( llm_gateway=llm_gateway, max_steps=react_config["max_steps"], ) else: react_engine.reset() messages = [{"role": "user", "content": request.message}] # Inject conversation history history_msgs = await _build_history_messages(conv.id) for hm in reversed(history_msgs): messages.insert(0, hm) tools = agent.get_tools() model = agent.get_model() system_prompt = getattr(agent, "_system_prompt", None) or agent.get_system_prompt() timeout_seconds = react_config["timeout_seconds"] collected_output: list[str] = [] try: async for event in react_engine.execute_stream( messages=messages, tools=tools, model=model, agent_name=agent.name, system_prompt=system_prompt, timeout_seconds=timeout_seconds, ): if event.event_type == "final_answer": collected_output.append(event.data.get("output", "")) except asyncio.CancelledError: raise except Exception as e: response_text = f"执行出错: {e}" else: response_text = _ensure_non_empty( "".join(collected_output) if collected_output else None ) await _conversation_store.add_message(conv.id, "assistant", response_text) return ChatResponse( conversation_id=conv.id, message=response_text, timestamp=datetime.now(timezone.utc).isoformat(), matched_skill=matched_skill, routing_method=routing_method, confidence=confidence, task_id=task_id, status="completed", ) @router.post("/portal/chat/stream") async def chat_stream(request: ChatRequest, req: Request, _auth: None = Depends(_verify_api_key)): """Stream chat responses via SSE with RequestPreprocessor routing.""" from sse_starlette.sse import EventSourceResponse agent, routing_result, matched_skill, routing_method, confidence = await _resolve_for_chat( request, req ) # Create or reuse conversation conv = await _conversation_store.get_or_create(request.conversation_id) await _conversation_store.add_message(conv.id, "user", request.message) llm_gateway = req.app.state.llm_gateway async def event_generator(): # Send routing info as first event yield { "event": "routing", "data": json.dumps( { "skill": matched_skill, "method": routing_method, "confidence": confidence, } ), } if ( routing_result is not None and routing_result.execution_mode == ExecutionMode.DIRECT_CHAT ): # DIRECT_CHAT: direct LLM call, no ReAct loop chat_messages = [] if routing_result.system_prompt: chat_messages.append({"role": "system", "content": routing_result.system_prompt}) chat_messages.append({"role": "user", "content": request.message}) history_msgs = await _build_history_messages(conv.id) for hm in history_msgs: chat_messages.insert(-1, hm) response = await llm_gateway.chat( messages=chat_messages, model=routing_result.model or "default", agent_name="default", task_type="chat", ) response_text = _ensure_non_empty(response.content) await _conversation_store.add_message(conv.id, "assistant", response_text) yield { "event": "final_answer", "data": json.dumps( { "step": 0, "data": {"output": response_text}, "timestamp": datetime.now(timezone.utc).isoformat(), } ), } else: # REACT / SKILL_REACT / REWOO / REFLEXION / PLAN_EXEC / TEAM_COLLAB # Advanced modes fall back to REACT with a warning. if routing_result is not None and routing_result.execution_mode not in ( ExecutionMode.REACT, ExecutionMode.SKILL_REACT, ): logger.warning( f"Execution mode {routing_result.execution_mode.value} not yet supported " f"in portal SSE, falling back to REACT" ) react_config = agent.get_react_config() react_engine = getattr(agent, "_react_engine", None) if react_engine is None: react_engine = ReActEngine( llm_gateway=llm_gateway, max_steps=react_config["max_steps"], ) else: react_engine.reset() messages = [{"role": "user", "content": request.message}] tools = agent.get_tools() model = agent.get_model() system_prompt = getattr(agent, "_system_prompt", None) or agent.get_system_prompt() timeout_seconds = react_config["timeout_seconds"] collected_output: list[str] = [] try: async for event in react_engine.execute_stream( messages=messages, tools=tools, model=model, agent_name=agent.name, system_prompt=system_prompt, timeout_seconds=timeout_seconds, ): if event.event_type == "final_answer": collected_output.append(event.data.get("output", "")) yield { "event": event.event_type, "data": json.dumps( { "step": event.step, "data": event.data, "timestamp": event.timestamp, } ), } except asyncio.CancelledError: raise except Exception as e: yield { "event": "error", "data": json.dumps({"error": str(e)}), } return response_text = _ensure_non_empty( "".join(collected_output) if collected_output else None ) await _conversation_store.add_message(conv.id, "assistant", response_text) return EventSourceResponse(event_generator()) @router.get("/portal/capabilities", response_model=CapabilitiesResponse) async def get_capabilities(req: Request, _auth: None = Depends(_verify_api_key)): """List all available capabilities with their status.""" skill_registry = req.app.state.skill_registry all_skills = skill_registry.list_skills() # Build a map of capability tag -> skill count cap_skill_counts: dict[str, int] = {} for skill in all_skills: for cap in skill.capabilities: cap_skill_counts[cap.tag] = cap_skill_counts.get(cap.tag, 0) + 1 # Also count the skill itself toward "skills" category cap_skill_counts["skills"] = cap_skill_counts.get("skills", 0) + 1 capabilities: list[CapabilityInfo] = [] for cat_name, cat_info in CAPABILITY_CATEGORIES.items(): skill_count = cap_skill_counts.get(cat_name, 0) capabilities.append( CapabilityInfo( name=cat_name, display_name=cat_info["display_name"], description=cat_info["description"], icon=cat_info["icon"], enabled=True, skill_count=skill_count, ) ) return CapabilitiesResponse(capabilities=capabilities) @router.get("/portal/conversations") async def list_conversations(limit: int = 20, _auth: None = Depends(_verify_api_key)): """List recent conversations. For each conversation, derive the title from the first user message read directly from SQLite (independent of the in-memory cache, which may have an empty `messages` list after a restart). This prevents the regression where titles collapse to the placeholder "对话". Also tags each conversation with ``is_board`` so the sidebar can show a "私董会" badge without having to fetch every conversation's full history. The check is a cheap metadata LIKE query — the list endpoint stays O(limit) even for hundreds of conversations. """ convs = await _conversation_store.list_conversations(limit=limit) result: list[dict] = [] for c in convs: # Re-derive title from the persisted user message so cache misses # after a restart don't surface the default placeholder. first_user = await _conversation_store.get_first_user_message(c.id) title = _derive_conversation_title_from_content(first_user.content if first_user else None) is_board = await _conversation_has_board_started(c.id) result.append( { "id": c.id, "title": title, "created_at": c.created_at.isoformat(), "updated_at": c.updated_at.isoformat(), "message_count": len(c.messages), "is_board": is_board, } ) return result async def _conversation_has_board_started(conversation_id: str) -> bool: """Return True if the conversation contains a persisted board_started event. Used by the sidebar list so it can render the "私董会" badge without fetching the full message history. Reads from SQLite directly — the in-memory cache may not be populated after a server restart. Returns False on any storage error so the badge never blocks the list endpoint. """ try: return await _conversation_store.has_message_with_type( conversation_id, "board_started" ) except (ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError): logger.warning("is_board lookup failed for %s", conversation_id, exc_info=True) return False def _derive_conversation_title(conv: Conversation) -> str: """Derive a human-readable title from the first user message in the conversation object.""" for msg in conv.messages: if msg.role == "user" and msg.content: return msg.content[:20] + ("..." if len(msg.content) > 20 else "") return "对话" _COMMAND_PREFIX_RE = re.compile( r"^@(?:board|team)(?::[^\s]+)?(?:\s+rounds=\d+)?\s*", re.IGNORECASE, ) def _strip_command_prefix(content: str) -> str: """Strip a leading @board/@team command so conversation titles show only the topic. Examples: "@board:warren,charlie 怎么看 AI" → "怎么看 AI" "@team 私董会" → "私董会" "@board rounds=3 软件行业" → "软件行业" """ if not content: return "" return _COMMAND_PREFIX_RE.sub("", content, count=1).strip() def _derive_conversation_title_from_content(content: str | None) -> str: """Derive title from a string content (used when conv.messages is empty).""" if content: cleaned = _strip_command_prefix(content) if cleaned: return cleaned[:20] + ("..." if len(cleaned) > 20 else "") return "对话" @router.get("/portal/conversations/{conversation_id}") async def get_conversation( conversation_id: str, limit: int = 50, _auth: None = Depends(_verify_api_key) ): """Get conversation history from SQLite-backed store. Title is derived from the first user message in `history` (the authoritative source) rather than the in-memory cache, which may have an empty `messages` list after a server restart. This prevents the regression where selecting a conversation collapses the title to "对话". """ history = await _conversation_store.get_history(conversation_id, limit=limit) if not history: raise HTTPException(status_code=404, detail=f"Conversation '{conversation_id}' not found") conv = await _conversation_store.get_or_create(conversation_id) first_user_content = next( (m.content for m in history if m.role == "user" and m.content), None, ) return { "id": conv.id, "title": _derive_conversation_title_from_content(first_user_content), "messages": [_hydrate_persisted_message(conv.id, i, m) for i, m in enumerate(history)], "created_at": conv.created_at.isoformat(), "updated_at": conv.updated_at.isoformat(), "is_board": any( (m.metadata or {}).get("message_type") == "board_started" for m in history ), } # Fields we store inside Message.metadata to reconstruct board_* messages # after a page reload. The Message dataclass only has role/content/timestamp # as first-class columns; everything else (message_type, expert identity, # board round/role, conclusion payload) rides along in metadata. _PERSISTED_MESSAGE_FIELDS = ( "message_type", "expert_id", "expert_name", "expert_avatar", "expert_color", "board_round", "board_role", "board_conclusion", "board_started", "matched_skill", "confidence", "routing_method", "thinking", "tool_calls", ) def _hydrate_persisted_message(conv_id: str, index: int, msg) -> dict: """Build the API response dict for a single persisted message. Promotes well-known rendering fields from ``msg.metadata`` to the top level so the frontend can render board_speech / board_summary / board_conclusion cards after a reload — without this, every restored assistant message would look like a plain chat bubble. """ payload: dict = { "id": f"{conv_id}-{index}", "role": msg.role, "content": msg.content, "timestamp": msg.timestamp.isoformat(), "metadata": getattr(msg, "metadata", None) or {}, } meta = payload["metadata"] if not isinstance(meta, dict): return payload for key in _PERSISTED_MESSAGE_FIELDS: if key in meta and meta[key] is not None: payload[key] = meta[key] return payload @router.delete("/portal/conversations/{conversation_id}") async def delete_conversation(conversation_id: str, _auth: None = Depends(_verify_api_key)): """Delete a conversation and all its messages. ponytail: IDOR note — portal endpoints use API-key auth (single-tenant access model: API key = full access to all conversations). The SQLite store has no user_id column, so per-user ownership cannot be enforced without a schema migration. If API keys become per-user, add a user_id column to conversations + filter DELETE by (id, user_id). Upgrade path: migrate portal endpoints to JWT auth + per-user scoping. """ deleted = await _conversation_store.delete_conversation(conversation_id) if not deleted: raise HTTPException(status_code=404, detail=f"Conversation '{conversation_id}' not found") return {"deleted": True, "id": conversation_id} def _derive_title_from_messages(messages: list) -> str: """Derive title from a list of Message objects (SessionManager format).""" for msg in messages: if msg.role.value == "user" and msg.content: return msg.content[:20] + ("..." if len(msg.content) > 20 else "") return "对话" async def _execute_react_background( react_engine: ReActEngine, messages: list[dict], tools: list, model: str, agent_name: str, system_prompt: str | None, timeout_seconds: float | None, conv_id: str, task_id: str, event_queue: EventQueue, conversation_store: SqliteConversationStore, task_store: InMemoryTaskStore | None = None, ) -> None: """Execute ReAct engine in the background, decoupled from WebSocket lifecycle. Events are emitted to the EventQueue (filtered by task_id) so that any subscriber — including a reconnected WebSocket — can consume them. Results are always persisted to the conversation store, regardless of whether a WebSocket subscriber is active. Task status is tracked in TaskStore when provided. """ collected_output: list[str] = [] try: if task_store is not None: try: await _task_store_update_status( task_store, task_id, TaskStatus.RUNNING, started_at=datetime.now(timezone.utc) ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to update TaskStore RUNNING", exc_info=True) async for event in react_engine.execute_stream( messages=messages, tools=tools, model=model, agent_name=agent_name, system_prompt=system_prompt, timeout_seconds=timeout_seconds, ): if event.event_type == "final_answer": collected_output.append(event.data.get("output", "")) # P1 #8/#9/#10 fix: Preserve TurnEventType mapping, step field, # and original data structure for EQ subscriber compatibility. # Note: Event dataclass has no 'step' field; use getattr for # compatibility with ReActEngine events that may include it. _turn_event_type = _REACT_EVENT_TYPE_MAP.get(event.event_type) if _turn_event_type is not None: await _emit_event_safe( event_queue, _turn_event_type, task_id=task_id, session_id=conv_id, data={ **event.data, "step": getattr(event, "step", 0), "timestamp": event.timestamp, }, ) # Normal completion: persist result response_text = _ensure_non_empty("".join(collected_output) if collected_output else None) await conversation_store.add_message(conv_id, "assistant", response_text) if task_store is not None: try: await _task_store_update_status( task_store, task_id, TaskStatus.COMPLETED, output_data={"output": response_text}, completed_at=datetime.now(timezone.utc), progress=1.0, progress_message="Completed", ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to update TaskStore COMPLETED", exc_info=True) # Emit task.completed so subscribers know the task is done await _emit_event_safe( event_queue, TaskEventType.TASK_COMPLETED, task_id=task_id, session_id=conv_id, data={"output": response_text, "timestamp": datetime.now(timezone.utc).isoformat()}, ) except asyncio.CancelledError: # Application shutdown or explicit cancel — persist partial output # and mark task as FAILED so resume does not block forever. # P0 #1/#2 fix: ALL persistence operations must use asyncio.shield # and the async TaskStore shim. Without shield, a re-entrant # cancellation kills the cleanup itself; without the shim, # RedisTaskStore (async) silently drops the coroutine. if collected_output: partial = _ensure_non_empty("".join(collected_output)) try: await asyncio.shield(conversation_store.add_message(conv_id, "assistant", partial)) except ( asyncio.CancelledError, ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to persist partial output on cancel") if task_store is not None: try: await asyncio.shield( _task_store_update_status( task_store, task_id, TaskStatus.FAILED, error_message="Task cancelled", completed_at=datetime.now(timezone.utc), ) ) except ( asyncio.CancelledError, ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to update TaskStore on cancel", exc_info=True) # P0 #2 fix: _emit_event_safe is async (it awaits event_queue.emit). # Shield it so a re-entrant CancelledError doesn't kill the emit # and leave subscribers blocked until timeout. try: await asyncio.shield( _emit_event_safe( event_queue, TaskEventType.TASK_FAILED, task_id=task_id, session_id=conv_id, data={ "error": "Task cancelled", "timestamp": datetime.now(timezone.utc).isoformat(), }, ) ) except (asyncio.CancelledError, asyncio.QueueFull, RuntimeError, ConnectionError): logger.warning("Failed to emit TASK_FAILED on cancel") raise # Propagate cancellation except Exception as e: # Persist any partial output collected before the error if collected_output: partial = _ensure_non_empty("".join(collected_output)) try: await conversation_store.add_message(conv_id, "assistant", partial) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to persist partial output in background task") if task_store is not None: try: await _task_store_update_status( task_store, task_id, TaskStatus.FAILED, error_message=str(e), completed_at=datetime.now(timezone.utc), ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to update TaskStore FAILED", exc_info=True) # Emit task.failed so subscribers know the task failed await _emit_event_safe( event_queue, TaskEventType.TASK_FAILED, task_id=task_id, session_id=conv_id, data={"error": str(e), "timestamp": datetime.now(timezone.utc).isoformat()}, ) @router.websocket("/portal/ws") async def portal_websocket(websocket: WebSocket): """Real-time chat WebSocket endpoint.""" await websocket.accept() # ponytail: ws_user_id must be initialized before any early return — the # finally block below references it. Previously the api_key reject path # returned before assignment, causing UnboundLocalError that masked the # original auth error. Upgrade path: refactor auth into a decorator. ws_user_id: str | None = None # Authentication (after accept, since FastAPI requires accept before close) configured_api_key: str | None = None if hasattr(websocket.app.state, "server_config") and websocket.app.state.server_config: configured_api_key = websocket.app.state.server_config.api_key if configured_api_key is None and hasattr(websocket.app.state, "api_key"): configured_api_key = websocket.app.state.api_key # Check api_key query param if configured_api_key: provided = websocket.query_params.get("api_key") if not hmac.compare_digest((provided or "").encode(), configured_api_key.encode()): await websocket.send_json( {"type": "error", "data": {"message": "Invalid or missing api_key"}} ) await websocket.close(code=4001, reason="Invalid or missing api_key") return # Track authenticated portal connections for user-scoped push (calendar # reminders, etc.). user_id is None for API-key / dev-mode clients. current_user = getattr(websocket.state, "current_user", None) or {} ws_user_id = current_user.get("user_id") if ws_user_id: portal_connection_manager.add(ws_user_id, websocket) # Wait for first chat message before creating conversation conv: Conversation | None = None # task_id is per-user-message; tracked here so the outer except can emit task.failed task_id: str | None = None # Track the active background task so cancel can propagate to it. active_bg_task: asyncio.Task | None = None try: while True: try: timeout = _WS_HEARTBEAT_TIMEOUT if _WS_HEARTBEAT_TIMEOUT > 0 else None raw = await asyncio.wait_for(websocket.receive_text(), timeout=timeout) except asyncio.TimeoutError: await websocket.close(code=1000, reason="Heartbeat timeout") return try: msg = json.loads(raw) except json.JSONDecodeError: continue msg_type = msg.get("type") if msg_type == "cancel": # Cancel the active background task if still running if active_bg_task is not None and not active_bg_task.done(): active_bg_task.cancel() active_bg_task = None await websocket.send_json( { "type": "result", "data": { "status": "cancelled", "timestamp": datetime.now(timezone.utc).isoformat(), }, } ) return if msg_type == "ping": await websocket.send_json({"type": "pong"}) continue if msg_type == "resume": # Frontend reconnected and wants to resume a running task resume_task_id = msg.get("task_id", "") if not resume_task_id: continue # P1 #3/#4 fix: Fail-closed ownership verification. # Require conversation_id and TaskStore — reject if either # is missing, to prevent cross-conversation task hijacking # via empty conversation_id or unconfigured TaskStore. resume_conv_id = msg.get("conversation_id", "") if not resume_conv_id: await websocket.send_json( { "type": "error", "data": { "message": "Resume requires conversation_id.", "task_id": resume_task_id, }, } ) continue resume_task_store: InMemoryTaskStore | None = getattr( websocket.app.state, "task_store", None ) resume_eq: EventQueue | None = getattr(websocket.app.state, "event_queue", None) # P1 #4: Fail-closed if TaskStore is unavailable — cannot # verify ownership without it. if resume_task_store is None: await websocket.send_json( { "type": "error", "data": { "message": "Resume not supported (TaskStore unavailable). Please retry your request.", "task_id": resume_task_id, }, } ) continue try: record = await _task_store_get(resume_task_store, resume_task_id) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("TaskStore.get failed during resume", exc_info=True) record = None if record is not None: # P1 #3: Fail-closed ownership check — reject if # conversation_id is missing from task metadata OR # does not match the request. task_conv_id = (record.metadata or {}).get("conversation_id", "") if not task_conv_id or resume_conv_id != task_conv_id: logger.warning( "Resume rejected: conversation_id mismatch " "(task=%s, request=%s, task_id=%s)", task_conv_id, resume_conv_id, resume_task_id, ) await websocket.send_json( { "type": "error", "data": { "message": "Task does not belong to this conversation.", "task_id": resume_task_id, }, } ) continue if record.status == TaskStatus.COMPLETED: # Task already finished — send result immediately output = (record.output_data or {}).get("output", "") await websocket.send_json( { "type": "result", "data": { "message": output, "timestamp": record.completed_at.isoformat() if record.completed_at else datetime.now(timezone.utc).isoformat(), }, } ) continue elif record.status == TaskStatus.FAILED: await websocket.send_json( { "type": "error", "data": { "message": record.error_message or "Task failed", }, } ) continue else: # Task not found in store — cannot resume await websocket.send_json( { "type": "error", "data": { "message": "Task not found or has expired. Please retry your request.", "task_id": resume_task_id, }, } ) continue # Task is still running — subscribe to EventQueue for remaining events. # H6: if EventQueue is unavailable, inform the client instead of # silently continuing (which would leave the UI loading forever). if resume_eq is None: await websocket.send_json( { "type": "error", "data": { "message": "Resume not supported (EventQueue unavailable). Please retry your request.", }, } ) continue # C2: bound the subscribe loop with a timeout so a dead # background task cannot block resume forever. resume_timeout = _WS_HEARTBEAT_TIMEOUT * 10 if _WS_HEARTBEAT_TIMEOUT > 0 else 600 try: async with asyncio.timeout(resume_timeout): async for event in resume_eq.subscribe(task_id=resume_task_id): if event.event_type == TaskEventType.TASK_COMPLETED: response_text = event.data.get("output", EMPTY_LLM_RESPONSE) await websocket.send_json( { "type": "result", "data": { "message": response_text, "timestamp": event.data.get( "timestamp", datetime.now(timezone.utc).isoformat(), ), }, } ) break elif event.event_type == TaskEventType.TASK_FAILED: await websocket.send_json( { "type": "error", "data": { "message": event.data.get("error", "Unknown error"), }, } ) break else: # P1 #8/#10 fix: step and data are now # top-level fields in event.data. await websocket.send_json( { "type": "step", "data": { "event_type": event.event_type, "step": event.data.get("step", 0), "data": { k: v for k, v in event.data.items() if k not in ("step", "timestamp") }, "timestamp": event.data.get("timestamp", ""), }, } ) except TimeoutError: logger.warning(f"Resume subscribe timed out for task {resume_task_id}") await websocket.send_json( { "type": "error", "data": { "message": "Task resume timed out. Please retry your request.", "task_id": resume_task_id, }, } ) except RuntimeError as exc: # P1 #5: subscriber limit reached or EQ closed — send # a friendly error instead of terminating the connection. logger.warning("Resume subscribe failed for task %s: %s", resume_task_id, exc) await websocket.send_json( { "type": "error", "data": { "message": "Server busy, please retry shortly.", "task_id": resume_task_id, }, } ) continue if msg_type != "chat": continue message_text = msg.get("message", "") model_override = msg.get("model") # Frontend model selector if not message_text: continue # Create or switch conversation based on conversation_id from frontend conv_id = msg.get("conversation_id") if conv_id: if conv is None or conv.id != conv_id: conv = await _conversation_store.get_or_create(conv_id) await websocket.send_json({"type": "connected", "conversation_id": conv.id}) elif conv is None: conv = await _conversation_store.get_or_create(conv_id) await websocket.send_json({"type": "connected", "conversation_id": conv.id}) # @board / @team intercept — mirror chat.py:1076-1081. # Frontend WS connects to /api/v1/portal/ws (this router), not # /api/v1/chat/ws/{session_id} (chat.py). Without this intercept # @board/@team messages are treated as plain text and no # board_started/team_formed events are broadcast, so the cards # never render. Placed before task_id emit so intercepted # messages don't leave orphan tasks in the EQ side-channel. if await _execute_board_meeting(websocket, conv.id, message_text, _sm_adapter): continue if await _execute_team_collab(websocket, conv.id, message_text, _sm_adapter): continue # Generate task_id for this user message and emit task.created to EQ # (EQ is a side-channel: emit failures never break the WebSocket flow) task_id = str(uuid.uuid4()) event_queue: EventQueue | None = getattr(websocket.app.state, "event_queue", None) task_store: InMemoryTaskStore | None = getattr(websocket.app.state, "task_store", None) await _emit_event_safe( event_queue, TaskEventType.TASK_CREATED, task_id=task_id, session_id=conv.id, data={"message": message_text}, ) # Add user message to conversation await _conversation_store.add_message(conv.id, "user", message_text) start_time = datetime.now(timezone.utc) async def _record_experience( task_type: str, goal: str, outcome: str, duration_seconds: float ) -> None: """Record experience to dashboard after chat completion.""" try: exp = DashboardExperience( id=str(uuid.uuid4()), task_type=task_type, goal=goal[:200], outcome=outcome, duration_seconds=duration_seconds, created_at=datetime.now(timezone.utc), ) _dashboard_experiences.append(exp) await _broadcast_dashboard_event( "experience_added", { "id": exp.id, "task_type": exp.task_type, "goal": exp.goal, "outcome": exp.outcome, }, ) await _broadcast_dashboard_event("metrics_updated", {"period": "7d"}) except ( asyncio.QueueFull, RuntimeError, ConnectionError, ValueError, KeyError, ) as e: logger.warning(f"Failed to record experience: {e}") # Unified preprocessing via RequestPreprocessor (minimal: @skill prefix + greeting regex + REACT) pool = websocket.app.state.agent_pool skill_registry = websocket.app.state.skill_registry llm_gateway = websocket.app.state.llm_gateway request_preprocessor: RequestPreprocessor = websocket.app.state.request_preprocessor all_skills = skill_registry.list_skills() # Get default tools for RequestPreprocessor default_tools = [] default_system_prompt = None default_agent = pool.get_agent("default") if default_agent is not None: default_tools = default_agent.get_tools() default_system_prompt = ( getattr(default_agent, "_system_prompt", None) or default_agent.get_system_prompt() ) else: for skill in all_skills: agent = pool.get_agent(skill.name) if agent is not None: default_tools = agent.get_tools() default_system_prompt = ( getattr(agent, "_system_prompt", None) or agent.get_system_prompt() ) break # Preprocess via RequestPreprocessor (minimal: @skill prefix + greeting regex + REACT) routing_result = await request_preprocessor.preprocess( content=message_text, skill_registry=skill_registry, default_tools=default_tools, default_system_prompt=default_system_prompt, default_model=model_override or "default", default_agent_name="default", ) await websocket.send_json( { "type": "routing", "skill": routing_result.agent_name or "default", "method": routing_result.match_method or "intent", "confidence": routing_result.match_confidence, } ) # Emit task.started to EQ (execution begins after routing) await _emit_event_safe( event_queue, TaskEventType.TASK_STARTED, task_id=task_id, session_id=conv.id, data={ "agent_name": routing_result.agent_name or "default", "execution_mode": routing_result.execution_mode.value if hasattr(routing_result.execution_mode, "value") else str(routing_result.execution_mode), }, ) # Register task in TaskStore for status tracking and recovery if task_store is not None: try: await _task_store_create( task_store, task_id=task_id, agent_name=routing_result.agent_name or "default", input_data={"message": message_text}, skill_name=routing_result.skill_name, ) # Store conversation_id in metadata for frontend recovery await _task_store_update_status( task_store, task_id, TaskStatus.PENDING, metadata={"conversation_id": conv.id}, ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to register task in TaskStore", exc_info=True) # Execute based on routing result's execution_mode # This is the single source of truth for path selection, # replacing fragile string-matching on match_method. if routing_result.execution_mode == ExecutionMode.DIRECT_CHAT: # Zero-cost path: direct LLM call, no ReAct loop chat_messages = [] # Inject system prompt (contains SOUL/USER/MEMORY/DAILY) for identity continuity if routing_result.system_prompt: chat_messages.append( {"role": "system", "content": routing_result.system_prompt} ) chat_messages.append({"role": "user", "content": message_text}) # Inject conversation history for context continuity history_msgs = await _build_history_messages(conv.id) for hm in history_msgs: chat_messages.insert(-1, hm) response = await llm_gateway.chat( messages=chat_messages, model=model_override or "default", agent_name="default", task_type="chat", ) # Store assistant reply for multi-turn context continuity response_content = _ensure_non_empty(response.content) await _conversation_store.add_message(conv.id, "assistant", response_content) # Update TaskStore status to COMPLETED if task_store is not None: try: await _task_store_update_status( task_store, task_id, TaskStatus.COMPLETED, output_data={"output": response_content}, completed_at=datetime.now(timezone.utc), progress=1.0, progress_message="Completed", ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): logger.warning("Failed to update TaskStore for DIRECT_CHAT", exc_info=True) # Emit turn.final_answer and task.completed to EQ await _emit_event_safe( event_queue, TurnEventType.FINAL_ANSWER, task_id=task_id, session_id=conv.id, data={"output": response_content}, ) await _emit_event_safe( event_queue, TaskEventType.TASK_COMPLETED, task_id=task_id, session_id=conv.id, data={"output": response_content}, ) await websocket.send_json( { "type": "result", "data": { "message": response_content, "timestamp": datetime.now(timezone.utc).isoformat(), }, } ) await _record_experience( "chat", message_text, "success", (datetime.now(timezone.utc) - start_time).total_seconds(), ) continue # REACT / SKILL_REACT / REWOO / REFLEXION / PLAN_EXEC / TEAM_COLLAB # Advanced modes fall back to REACT with a warning. if routing_result.execution_mode not in ( ExecutionMode.REACT, ExecutionMode.SKILL_REACT, ): logger.warning( f"Execution mode {routing_result.execution_mode.value} not yet supported " f"in portal WebSocket, falling back to REACT" ) agent_name = routing_result.agent_name or "default" agent = pool.get_agent(agent_name) if agent is None: # Agent not in pool — fall back to direct chat. # This handles the case where routing returned an agent_name # that doesn't exist in the pool (e.g. "default" or a # skill that hasn't been instantiated yet). logger.info( f"Session {conv.id}: agent '{agent_name}' not in pool, falling back to direct chat" ) chat_messages = [] # Inject system prompt (contains SOUL/USER/MEMORY/DAILY) for identity continuity if routing_result.system_prompt: chat_messages.append( {"role": "system", "content": routing_result.system_prompt} ) chat_messages.append({"role": "user", "content": message_text}) try: history = await _conversation_store.get_history(conv.id, limit=20) for hist_msg in history[:-1]: if hist_msg.role in ("user", "assistant"): chat_messages.insert( -1, {"role": hist_msg.role, "content": hist_msg.content} ) except ( ConnectionError, OSError, asyncio.TimeoutError, ValueError, KeyError, RuntimeError, ): pass response = await llm_gateway.chat( messages=chat_messages, model=model_override or "default", agent_name="default", task_type="chat", ) # Store assistant reply for multi-turn context continuity response_content = _ensure_non_empty(response.content) await _conversation_store.add_message(conv.id, "assistant", response_content) # Emit turn.final_answer and task.completed to EQ (fallback path) await _emit_event_safe( event_queue, TurnEventType.FINAL_ANSWER, task_id=task_id, session_id=conv.id, data={"output": response_content}, ) await _emit_event_safe( event_queue, TaskEventType.TASK_COMPLETED, task_id=task_id, session_id=conv.id, data={"output": response_content}, ) await websocket.send_json( { "type": "result", "data": { "status": "completed", "content": response_content, "timestamp": datetime.now(timezone.utc).isoformat(), }, } ) await _record_experience( "chat", message_text, "success", (datetime.now(timezone.utc) - start_time).total_seconds(), ) continue # Execute via ReAct stream react_config = agent.get_react_config() # Reuse agent's ReActEngine if available (aligned with chat.py pattern) react_engine = getattr(agent, "_react_engine", None) if react_engine is None: react_engine = ReActEngine( llm_gateway=llm_gateway, max_steps=react_config["max_steps"], ) else: react_engine.reset() messages = [{"role": "user", "content": message_text}] # Inject conversation history for context continuity history_msgs = await _build_history_messages(conv.id) for hm in reversed(history_msgs): messages.insert(0, hm) tools = agent.get_tools() model = model_override or agent.get_model() system_prompt = getattr(agent, "_system_prompt", None) or agent.get_system_prompt() timeout_seconds = react_config["timeout_seconds"] logger.info( f"[portal] agent='{agent_name}' tools={len(tools)} " f"[{', '.join(t.name for t in tools)}] model={model}" ) # Start ReAct execution as a background task, decoupled from # WebSocket lifecycle. When the WebSocket disconnects, the # background task continues running and persists the result. bg_task = asyncio.create_task( _execute_react_background( react_engine=react_engine, messages=messages, tools=tools, model=model, agent_name=agent.name, system_prompt=system_prompt, timeout_seconds=timeout_seconds, conv_id=conv.id, task_id=task_id, event_queue=event_queue, conversation_store=_conversation_store, task_store=task_store, ) ) _running_background_tasks.add(bg_task) bg_task.add_done_callback(_running_background_tasks.discard) active_bg_task = bg_task # C1 guard: EventQueue is required for subscribe; fall back to # awaiting the background task directly if unavailable. if event_queue is None: logger.warning("EventQueue not configured; awaiting background task directly") try: await bg_task except (RuntimeError, ConnectionError, asyncio.TimeoutError): pass # errors handled inside _execute_react_background active_bg_task = None continue # Subscribe to EventQueue (filtered by task_id) and forward # events to the WebSocket. When the WebSocket disconnects, # this loop exits but the background task continues. # P1 #7 fix: bound the subscribe loop with a timeout so a # hung background task cannot block the WebSocket forever. # Matches the resume path's timeout strategy. _subscribe_timeout = _WS_HEARTBEAT_TIMEOUT * 10 if _WS_HEARTBEAT_TIMEOUT > 0 else 600 try: async with asyncio.timeout(_subscribe_timeout): async for event in event_queue.subscribe(task_id=task_id): if event.event_type == TaskEventType.TASK_COMPLETED: response_text = event.data.get("output", EMPTY_LLM_RESPONSE) await websocket.send_json( { "type": "result", "data": { "message": response_text, "timestamp": event.data.get( "timestamp", datetime.now(timezone.utc).isoformat(), ), }, } ) await _record_experience( routing_result.skill_name or "agent", message_text, "success" if response_text != EMPTY_LLM_RESPONSE else "failure", (datetime.now(timezone.utc) - start_time).total_seconds(), ) break elif event.event_type == TaskEventType.TASK_FAILED: await websocket.send_json( { "type": "error", "data": { "message": event.data.get("error", "Unknown error"), }, } ) await _record_experience( routing_result.skill_name or "agent", message_text, "failure", (datetime.now(timezone.utc) - start_time).total_seconds(), ) break else: # Forward ReAct events as step messages. # P1 #8/#10 fix: step and data are now top-level # fields in event.data (no longer nested). await websocket.send_json( { "type": "step", "data": { "event_type": event.event_type, "step": event.data.get("step", 0), "data": { k: v for k, v in event.data.items() if k not in ("step", "timestamp") }, "timestamp": event.data.get("timestamp", ""), }, } ) except TimeoutError: logger.warning(f"Subscribe loop timed out for task {task_id}") if active_bg_task is not None and not active_bg_task.done(): active_bg_task.cancel() await websocket.send_json( { "type": "error", "data": { "message": "Task timed out. Please retry your request.", "task_id": task_id, }, } ) except RuntimeError as exc: # P1 #5: subscriber limit reached or EQ closed — send # a friendly error instead of terminating the connection. logger.warning("Subscribe failed for task %s: %s", task_id, exc) await websocket.send_json( { "type": "error", "data": { "message": "Server busy, please retry shortly.", "task_id": task_id, }, } ) except WebSocketDisconnect: logger.debug(f"Portal WebSocket disconnected for conversation {conv.id if conv else 'N/A'}") # P0 fix: Do NOT cancel the background task on disconnect. # The entire purpose of the three-layer defense is to let the # background task continue running and persist the result so the # frontend can resume it after reconnection. Cancelling here would # kill the task, lose the full output, and mark it FAILED — # defeating layers 2 and 3. The task is only cancelled on explicit # user cancel (msg_type == 'cancel') or application shutdown. except asyncio.CancelledError: raise except Exception as e: logger.error(f"Portal WebSocket error: {e}") # P1 #6 fix: Do NOT cancel the background task on connection-level # errors (ConnectionResetError, BrokenPipeError, etc.). These are # functionally equivalent to WebSocketDisconnect — the client dropped # — and the background task must survive to persist its result. # Only cancel on truly unexpected errors that may have corrupted # state needed by the background task. if not isinstance(e, (ConnectionResetError, BrokenPipeError, ConnectionError)): if active_bg_task is not None and not active_bg_task.done(): active_bg_task.cancel() # Emit task.failed to EQ if a task was in progress # (task_id is set when a user message is received; None before that) if task_id is not None and conv is not None: event_queue = getattr(websocket.app.state, "event_queue", None) await _emit_event_safe( event_queue, TaskEventType.TASK_FAILED, task_id=task_id, session_id=conv.id, data={"error": str(e)}, ) try: await websocket.send_json({"type": "error", "data": {"message": str(e)}}) except (ConnectionError, RuntimeError, asyncio.TimeoutError): pass finally: # Remove from user-scoped push tracking on any disconnect/error/return. if ws_user_id: portal_connection_manager.remove(ws_user_id, websocket)