From 8a62e72fd52d10e77d21d2b0059dda419fc03f42 Mon Sep 17 00:00:00 2001 From: kris Date: Fri, 3 Apr 2026 00:21:19 +0800 Subject: [PATCH] refactor: add execution backend selection --- .../2026-04-02-boss-execution-foundation.md | 15 +- ...-04-02-boss-execution-foundation-design.md | 5 + src/lib/boss-data.ts | 33 + src/lib/boss-master-agent.ts | 727 ++++++++++-------- src/lib/execution/backend-selector.ts | 97 +++ .../execution/backends/aliyun-qwen-backend.ts | 22 + .../backends/master-codex-node-backend.ts | 22 + src/lib/execution/backends/openai-backend.ts | 19 + tests/execution-backend-selector.test.ts | 75 ++ tests/master-agent-message-queue.test.ts | 162 +++- tests/master-agent-openai-fallback.test.ts | 208 ++++- 11 files changed, 1067 insertions(+), 318 deletions(-) create mode 100644 src/lib/execution/backend-selector.ts create mode 100644 src/lib/execution/backends/aliyun-qwen-backend.ts create mode 100644 src/lib/execution/backends/master-codex-node-backend.ts create mode 100644 src/lib/execution/backends/openai-backend.ts create mode 100644 tests/execution-backend-selector.test.ts diff --git a/docs/superpowers/plans/2026-04-02-boss-execution-foundation.md b/docs/superpowers/plans/2026-04-02-boss-execution-foundation.md index 87eaa4f..31701ad 100644 --- a/docs/superpowers/plans/2026-04-02-boss-execution-foundation.md +++ b/docs/superpowers/plans/2026-04-02-boss-execution-foundation.md @@ -610,8 +610,9 @@ export async function selectExecutionBackend(input: { primary: { provider: string; status: string }; backups: Array<{ provider: string; status: string }>; }) { - if (input.primary.provider === "master_codex_node" && input.primary.status === "ready") { - return { backendId: "master-codex-node" }; + const primaryBackend = resolveBackendByProvider(input.primary.provider); + if (input.primary.status === "ready") { + return primaryBackend; } const qwen = input.backups.find((item) => item.provider === "aliyun_qwen_api" && item.status === "ready"); if (qwen) { @@ -621,12 +622,20 @@ export async function selectExecutionBackend(input: { if (openai) { return { backendId: "openai-api" }; } - return { backendId: "master-codex-node" }; + const master = input.backups.find((item) => item.provider === "master_codex_node" && item.status === "ready"); + if (master) { + return { backendId: "master-codex-node" }; + } + return primaryBackend; } export const selectExecutionBackendForTesting = selectExecutionBackend; ``` +补充说明： +- selector 的运行时语义以“`ready primary` 优先，否则按 `aliyun_qwen -> openai -> master_codex_node` 顺序回退，最后再回 primary 兜底”为准。 +- 如果同一 provider 存在多个账号，只要其中任何一个是 `ready`，该 backend 就视为可选。 + 在 `/Users/kris/code/boss/src/lib/boss-master-agent.ts` 中，把 provider fallback 的选择逻辑收进一个单独调用： ```ts diff --git a/docs/superpowers/specs/2026-04-02-boss-execution-foundation-design.md b/docs/superpowers/specs/2026-04-02-boss-execution-foundation-design.md index 92b1711..6ee804f 100644 --- a/docs/superpowers/specs/2026-04-02-boss-execution-foundation-design.md +++ b/docs/superpowers/specs/2026-04-02-boss-execution-foundation-design.md @@ -138,6 +138,11 @@ interface ExecutionBackend { 说明： - 这层只负责“选择谁执行”，不负责组装 prompt，也不负责审批和业务决策 +- 运行时选择语义应明确为： + - `ready primary` 优先 + - 否则按 `aliyun_qwen -> openai -> master_codex_node` 顺序回退 + - 如果没有任何 ready backend，再回 primary 兜底 +- 同一 provider 下如果存在多个账号，只要其中任一账号 `ready`，该 backend 就视为可选 ### 3. `SessionRuntime` diff --git a/src/lib/boss-data.ts b/src/lib/boss-data.ts index e71152d..e72a460 100644 --- a/src/lib/boss-data.ts +++ b/src/lib/boss-data.ts @@ -5560,6 +5560,39 @@ export async function getMasterAgentTask(taskId: string) { return state.masterAgentTasks.find((item) => item.taskId === taskId) ?? null; } +export async function reassignMasterAgentTaskExecution(payload: { + taskId: string; + deviceId: string; + accountId?: string; + accountLabel?: string; + executionPrompt?: string; +}) { + const task = await mutateState((state) => { + const next = state.masterAgentTasks.find((item) => item.taskId === payload.taskId); + if (!next) { + throw new Error("MASTER_AGENT_TASK_NOT_FOUND"); + } + if (next.status !== "queued") { + return { ...next }; + } + next.deviceId = payload.deviceId; + next.accountId = payload.accountId; + next.accountLabel = payload.accountLabel; + if (payload.executionPrompt?.trim()) { + next.executionPrompt = payload.executionPrompt.trim(); + } + return { ...next }; + }); + + publishBossEvent("master_agent.task.updated", { + taskId: task.taskId, + deviceId: task.deviceId, + status: task.status, + }); + + return task; +} + export async function claimNextMasterAgentTask(deviceId: string) { let attachmentProjectId: string | undefined; let dispatchExecutionProjectId: string | undefined; diff --git a/src/lib/boss-master-agent.ts b/src/lib/boss-master-agent.ts index 14b5a8a..dcd1d21 100644 --- a/src/lib/boss-master-agent.ts +++ b/src/lib/boss-master-agent.ts @@ -14,13 +14,22 @@ import { getMasterAgentTask, queueMasterAgentTask, readState, + reassignMasterAgentTaskExecution, isDispatchableThreadProject, touchUserMasterMemories, updateAttachmentAnalysisResult, updateAiAccountHealth, } from "@/lib/boss-data"; -import type { AiProvider, DispatchPlanTarget, Project, ProjectAgentControls, ReasoningEffort } from "@/lib/boss-data"; +import type { + AiAccount, + AiProvider, + DispatchPlanTarget, + Project, + ProjectAgentControls, + ReasoningEffort, +} from "@/lib/boss-data"; import { canInlineAttachmentText, extractAttachmentTextExcerpt } from "@/lib/boss-attachments"; +import { listExecutionBackendChoices, selectExecutionBackend } from "@/lib/execution/backend-selector"; import { resolveRuntimeRelevantMemories } from "@/lib/execution/memory-resolver"; import type { RelevantMemory } from "@/lib/execution/memory-resolver"; import { buildExecutionPrompt } from "@/lib/execution/prompt-assembler"; @@ -363,38 +372,140 @@ function normalizeApiProviderFetchFailure(provider: ApiCompatibleProvider, error return normalizeApiProviderError(provider, String(error)); } -function fallbackAiRolePriority(role: "primary" | "backup" | "api_fallback") { - switch (role) { - case "primary": - return 0; - case "backup": - return 1; - case "api_fallback": - return 2; - default: - return 9; - } +function isUsableApiAccount(account: AiAccount, provider: ApiCompatibleProvider) { + return ( + account.enabled && + account.provider === provider && + (account.status === "ready" || account.status === "degraded") && + Boolean(account.apiKey?.trim()) + ); } -async function findFallbackApiAccount(excludedAccountId?: string) { - const state = await readState(); - return [...state.aiAccounts] - .filter( - (account) => - account.accountId !== excludedAccountId && - account.enabled && - isApiCompatibleProvider(account.provider) && - Boolean(account.apiKey?.trim()), - ) - .sort((left, right) => { - const roleDelta = fallbackAiRolePriority(left.role) - fallbackAiRolePriority(right.role); - if (roleDelta !== 0) return roleDelta; - return (right.updatedAt ?? "").localeCompare(left.updatedAt ?? ""); - })[0]; +function isUsableMasterNodeAccount(account: AiAccount) { + return ( + account.enabled && + account.provider === "master_codex_node" && + account.status === "ready" && + Boolean(account.nodeId?.trim()) + ); +} + +function isOnlineMasterNodeAccount( + state: Awaited>, + account: AiAccount, +) { + if (!isUsableMasterNodeAccount(account)) { + return false; + } + const deviceId = account.nodeId?.trim(); + if (!deviceId) { + return false; + } + const device = state.devices.find((item) => item.id === deviceId); + return Boolean(device && device.status === "online"); +} + +function sortSelectableAccounts(left: AiAccount, right: AiAccount) { + if (left.isActive !== right.isActive) { + return left.isActive ? -1 : 1; + } + return (right.updatedAt ?? "").localeCompare(left.updatedAt ?? ""); +} + +function sortApiSelectableAccounts(left: AiAccount, right: AiAccount) { + if (left.status !== right.status) { + return left.status === "ready" ? -1 : 1; + } + return sortSelectableAccounts(left, right); +} + +async function resolveAccountForSelectedBackend( + selectedBackendProvider: AiProvider, + runtimeAccount: AiAccount, +) { + if (selectedBackendProvider === "master_codex_node") { + const state = await readState(); + if (isOnlineMasterNodeAccount(state, runtimeAccount)) { + return runtimeAccount; + } + + return state.aiAccounts + .filter((account) => isOnlineMasterNodeAccount(state, account)) + .sort(sortSelectableAccounts)[0]; + } + + if (selectedBackendProvider === "openai_api" || selectedBackendProvider === "aliyun_qwen_api") { + const state = await readState(); + const candidates = [ + ...(isUsableApiAccount(runtimeAccount, selectedBackendProvider) ? [runtimeAccount] : []), + ...state.aiAccounts.filter((account): account is AiAccount => + account.accountId !== runtimeAccount.accountId && isUsableApiAccount(account, selectedBackendProvider), + ), + ]; + return candidates.sort(sortApiSelectableAccounts)[0]; + } + + return null; +} + +interface ApiExecutionCandidate { + provider: ApiCompatibleProvider; + account: AiAccount; + deviceId: string; + model: string; +} + +async function buildApiExecutionCandidates(params: { + backendChoices: Array<{ provider: AiProvider }>; + runtimeAccount: AiAccount; + agentControls?: ProjectAgentControls | null; +}) { + const candidates: ApiExecutionCandidate[] = []; + const seenAccountIds = new Set(); + + for (const backend of params.backendChoices) { + if (!isApiCompatibleProvider(backend.provider)) { + continue; + } + + const account = await resolveAccountForSelectedBackend(backend.provider, params.runtimeAccount); + if (!account || !isApiCompatibleProvider(account.provider) || !account.apiKey?.trim()) { + continue; + } + if (seenAccountIds.has(account.accountId)) { + continue; + } + + seenAccountIds.add(account.accountId); + candidates.push({ + provider: account.provider, + account, + deviceId: account.provider === "aliyun_qwen_api" ? ALIYUN_QWEN_DEVICE_ID : OPENAI_MASTER_AGENT_DEVICE_ID, + model: + params.agentControls?.modelOverride || + account.model || + apiProviderConfig(account.provider).defaultModel, + }); + } + + return candidates; +} + +async function resolveMasterNodeExecutionCandidate(params: { + backendChoices: Array<{ backendId: string; provider: AiProvider }>; + runtimeAccount: AiAccount; +}) { + const wantsMasterNode = params.backendChoices.some((backend) => backend.backendId === "master-codex-node"); + if (!wantsMasterNode) { + return null; + } + + const account = await resolveAccountForSelectedBackend("master_codex_node", params.runtimeAccount); + return account && account.provider === "master_codex_node" ? account : null; } async function replyViaOpenAiAccount(params: { - account: Awaited>; + account: AiAccount; requestText: string; currentSessionExpiresAt?: string; senderLabel: string; @@ -431,6 +542,10 @@ async function replyViaOpenAiAccount(params: { status: "ready", lastValidatedAt: new Date().toISOString(), lastUsedAt: new Date().toISOString(), + activate: !params.account.isActive, + switchReason: params.account.isActive + ? params.account.switchReason + : `主 Agent 回复时自动切换到 ${params.account.label}`, }); return { @@ -554,80 +669,146 @@ function buildMasterOpenAiReplyPrompt( } async function queueAndStartOpenAiMasterAgentReply(params: { - provider: ApiCompatibleProvider; + candidates: ApiExecutionCandidate[]; taskId: string; - deviceId: string; requestText: string; currentSessionExpiresAt?: string; - apiKey: string; - model: string; reasoningEffort: ReasoningEffort; agentControls?: ProjectAgentControls | null; promptPolicy?: Awaited>; userPrompt?: Awaited>; projectMemories?: RelevantMemory[]; userMemories?: RelevantMemory[]; + masterFallback?: { + account: AiAccount; + executionPrompt: string; + } | null; }) { + const completeTaskSafely = async (payload: Parameters[0]) => { + try { + await completeMasterAgentTask(payload); + } catch (error) { + if (error instanceof Error && error.message === "MASTER_AGENT_TASK_NOT_FOUND") { + return; + } + throw error; + } + }; + const timer = setTimeout(() => { void (async () => { - const task = await getMasterAgentTask(params.taskId); - if (!task || task.status !== "queued") { + let lastErrorMessage = "主 Agent 当前调用模型失败。"; + + for (const candidate of params.candidates) { + const task = await getMasterAgentTask(params.taskId); + if (!task || task.status !== "queued") { + return; + } + + if (task.accountId !== candidate.account.accountId || task.deviceId !== candidate.deviceId) { + await reassignMasterAgentTaskExecution({ + taskId: params.taskId, + deviceId: candidate.deviceId, + accountId: candidate.account.accountId, + accountLabel: candidate.account.label, + }); + } + + try { + const generated = await generateApiProviderReply({ + provider: candidate.provider, + apiKey: candidate.account.apiKey ?? "", + model: candidate.model, + reasoningEffort: params.reasoningEffort, + requestText: params.requestText, + currentSessionExpiresAt: params.currentSessionExpiresAt, + agentControls: params.agentControls, + promptPolicy: params.promptPolicy, + userPrompt: params.userPrompt, + projectMemories: params.projectMemories, + userMemories: params.userMemories, + }); + + await completeTaskSafely({ + taskId: params.taskId, + deviceId: candidate.deviceId, + status: "completed", + replyBody: generated.content, + requestId: generated.requestId, + }); + await updateAiAccountHealth({ + accountId: candidate.account.accountId, + status: "ready", + lastValidatedAt: new Date().toISOString(), + lastUsedAt: new Date().toISOString(), + activate: !candidate.account.isActive, + switchReason: candidate.account.isActive + ? candidate.account.switchReason + : `主 Agent 回复时自动切换到 ${candidate.account.label}`, + }); + return; + } catch (error) { + if (error instanceof Error && error.message === "MASTER_AGENT_TASK_NOT_FOUND") { + return; + } + lastErrorMessage = error instanceof Error ? error.message : "主 Agent 当前调用模型失败。"; + await updateAiAccountHealth({ + accountId: candidate.account.accountId, + status: "degraded", + lastError: lastErrorMessage, + lastValidatedAt: new Date().toISOString(), + }); + } + } + + if (params.masterFallback) { + const fallbackTask = await getMasterAgentTask(params.taskId); + if (!fallbackTask || fallbackTask.status !== "queued") { + return; + } + await reassignMasterAgentTaskExecution({ + taskId: params.taskId, + deviceId: params.masterFallback.account.nodeId || "mac-studio", + accountId: params.masterFallback.account.accountId, + accountLabel: params.masterFallback.account.label, + executionPrompt: params.masterFallback.executionPrompt, + }); return; } - try { - const generated = await generateApiProviderReply({ - provider: params.provider, - apiKey: params.apiKey, - model: params.model, - reasoningEffort: params.reasoningEffort, - requestText: params.requestText, - currentSessionExpiresAt: params.currentSessionExpiresAt, - agentControls: params.agentControls, - promptPolicy: params.promptPolicy, - userPrompt: params.userPrompt, - projectMemories: params.projectMemories, - userMemories: params.userMemories, - }); - - await completeMasterAgentTask({ - taskId: params.taskId, - deviceId: params.deviceId, - status: "completed", - replyBody: generated.content, - requestId: generated.requestId, - }); - } catch (error) { - await completeMasterAgentTask({ - taskId: params.taskId, - deviceId: params.deviceId, - status: "failed", - errorMessage: error instanceof Error ? error.message : "主 Agent 当前调用模型失败。", - }); - } + await completeTaskSafely({ + taskId: params.taskId, + deviceId: params.candidates[0]?.deviceId ?? OPENAI_MASTER_AGENT_DEVICE_ID, + status: "failed", + errorMessage: lastErrorMessage, + }); })(); }, 0); timer.unref?.(); } async function enqueueOpenAiMasterAgentReply(params: { - provider: ApiCompatibleProvider; - accountId: string; - accountLabel: string; + candidates: ApiExecutionCandidate[]; requestMessageId?: string; requestText: string; requestedBy: string; requestedByAccount: string; currentSessionExpiresAt?: string; - apiKey: string; - model: string; reasoningEffort: ReasoningEffort; agentControls?: ProjectAgentControls | null; promptPolicy?: Awaited>; userPrompt?: Awaited>; projectMemories?: RelevantMemory[]; userMemories?: RelevantMemory[]; + masterFallback?: { + account: AiAccount; + executionPrompt: string; + } | null; }) { + const primaryCandidate = params.candidates[0]; + if (!primaryCandidate) { + throw new Error("MASTER_AGENT_API_BACKEND_NOT_AVAILABLE"); + } const state = await readState(); const task = await queueMasterAgentTask({ requestMessageId: params.requestMessageId ?? "master-agent-manual", @@ -644,29 +825,27 @@ async function enqueueOpenAiMasterAgentReply(params: { ), requestedBy: params.requestedBy, requestedByAccount: params.requestedByAccount, - deviceId: params.provider === "aliyun_qwen_api" ? ALIYUN_QWEN_DEVICE_ID : OPENAI_MASTER_AGENT_DEVICE_ID, - accountId: params.accountId, - accountLabel: params.accountLabel, + deviceId: primaryCandidate.deviceId, + accountId: primaryCandidate.account.accountId, + accountLabel: primaryCandidate.account.label, }); void queueAndStartOpenAiMasterAgentReply({ - provider: params.provider, + candidates: params.candidates, taskId: task.taskId, - deviceId: params.provider === "aliyun_qwen_api" ? ALIYUN_QWEN_DEVICE_ID : OPENAI_MASTER_AGENT_DEVICE_ID, requestText: params.requestText, currentSessionExpiresAt: params.currentSessionExpiresAt, - apiKey: params.apiKey, - model: params.model, reasoningEffort: params.reasoningEffort, agentControls: params.agentControls, promptPolicy: params.promptPolicy, userPrompt: params.userPrompt, projectMemories: params.projectMemories, userMemories: params.userMemories, + masterFallback: params.masterFallback, }); const queuedReply: QueuedMasterAgentReplyEnvelope = { ok: true as const, - accountId: params.accountId, + accountId: primaryCandidate.account.accountId, taskId: task.taskId, masterReplyState: "queued" as const, task: { @@ -1390,82 +1569,98 @@ export async function replyToMasterAgentUserMessage(params: { params.requestedByAccount, params.requestText, ); + const state = await readState(); + const primaryDeviceId = runtime.account.nodeId || state.user.boundDeviceId || "mac-studio"; + const primaryDevice = state.devices.find((device) => device.id === primaryDeviceId); + const primaryBackendStatus = + runtime.account.provider === "master_codex_node" && (!primaryDevice || primaryDevice.status !== "online") + ? "degraded" + : runtime.account.status; + const backendSelectionInput = { + primary: { + provider: runtime.account.provider, + status: primaryBackendStatus, + }, + backups: state.aiAccounts + .filter((account) => account.accountId !== runtime.account.accountId) + .map((account) => ({ + provider: account.provider, + status: account.status, + })), + }; + const selectedBackend = await selectExecutionBackend(backendSelectionInput); + const backendChoices = listExecutionBackendChoices(backendSelectionInput); const agentControls = executionConfig.agentControls; + const masterExecutionPrompt = buildMasterCodexNodePrompt( + state, + params.requestText, + params.currentSessionExpiresAt, + agentControls, + executionConfig.promptPolicy, + executionConfig.userPrompt, + executionConfig.projectMemories, + executionConfig.userMemories, + ); + const selectedMasterAccount = await resolveMasterNodeExecutionCandidate({ + backendChoices, + runtimeAccount: runtime.account, + }); + const apiExecutionCandidates = await buildApiExecutionCandidates({ + backendChoices, + runtimeAccount: runtime.account, + agentControls, + }); + const hasMasterFallback = backendChoices.some((backend) => backend.backendId === "master-codex-node"); - if (params.mode === "enqueue") { - if (runtime.account.provider === "master_codex_node") { - const state = await readState(); - const deviceId = runtime.account.nodeId || state.user.boundDeviceId || "mac-studio"; - const boundDevice = state.devices.find((device) => device.id === deviceId); - const boundNodeLabel = - runtime.account.nodeLabel?.trim() || - boundDevice?.name || - state.user.boundCodexNodeLabel || - deviceId; + const runMasterNodeExecution = async () => { + if (!selectedMasterAccount) { + await appendMasterAgentSystemReply( + [ + `当前主控身份是 ${runtime.summary.roleLabel}，目标后端是 Master Codex Node，但当前没有可用的 master 节点账号。`, + "请先把可用的 Master Codex Node 重新接回，再重试。", + ].join(""), + `主 Agent · ${runtime.summary.roleLabel}`, + ); + return { ok: false as const, reason: "MASTER_NODE_NOT_CONNECTED" }; + } - if (!boundDevice || boundDevice.status !== "online") { - await updateAiAccountHealth({ - accountId: runtime.account.accountId, - status: "degraded", - lastError: !boundDevice ? "MASTER_CODEX_NODE_DEVICE_NOT_FOUND" : "MASTER_CODEX_NODE_DEVICE_OFFLINE", - lastValidatedAt: new Date().toISOString(), - }); + const deviceId = selectedMasterAccount.nodeId || state.user.boundDeviceId || "mac-studio"; + const boundDevice = state.devices.find((device) => device.id === deviceId); + const boundNodeLabel = + selectedMasterAccount.nodeLabel?.trim() || + boundDevice?.name || + state.user.boundCodexNodeLabel || + deviceId; - const fallbackAccount = await findFallbackApiAccount(runtime.account.accountId); - if (fallbackAccount?.apiKey?.trim() && isApiCompatibleProvider(fallbackAccount.provider)) { - return enqueueOpenAiMasterAgentReply({ - provider: fallbackAccount.provider, - accountId: fallbackAccount.accountId, - accountLabel: fallbackAccount.label || aiRoleLabel(fallbackAccount.role), - requestMessageId: params.requestMessageId, - requestText: params.requestText, - requestedBy: params.requestedBy, - requestedByAccount: params.requestedByAccount, - currentSessionExpiresAt: params.currentSessionExpiresAt, - apiKey: fallbackAccount.apiKey, - model: - agentControls?.modelOverride || - fallbackAccount.model || - apiProviderConfig(fallbackAccount.provider).defaultModel, - reasoningEffort: agentControls?.reasoningEffortOverride || "medium", - agentControls, - promptPolicy: executionConfig.promptPolicy, - userPrompt: executionConfig.userPrompt, - projectMemories: executionConfig.projectMemories, - userMemories: executionConfig.userMemories, - }); - } - - await appendMasterAgentSystemReply( - `主 GPT 不在手机里直接登录。当前绑定设备 ${boundNodeLabel}${boundDevice ? " 不在线" : " 未找到"}，主 Agent 暂时无法通过这台设备对话。请先在该设备上登录 Codex / ChatGPT Plus，并确保 local-agent 在线后再重试。`, - `主 Agent · ${runtime.summary.roleLabel}`, - ); - return { ok: false as const, reason: "MASTER_NODE_OFFLINE" }; - } - - const task = await queueMasterAgentTask({ - requestMessageId: params.requestMessageId ?? "master-agent-manual", - requestText: params.requestText, - executionPrompt: buildMasterCodexNodePrompt( - state, - params.requestText, - params.currentSessionExpiresAt, - agentControls, - executionConfig.promptPolicy, - executionConfig.userPrompt, - executionConfig.projectMemories, - executionConfig.userMemories, - ), - requestedBy: params.requestedBy, - requestedByAccount: params.requestedByAccount, - deviceId, - accountId: runtime.account.accountId, - accountLabel: runtime.account.label || runtime.summary.roleLabel, + if (!boundDevice || boundDevice.status !== "online") { + await updateAiAccountHealth({ + accountId: selectedMasterAccount.accountId, + status: "degraded", + lastError: !boundDevice ? "MASTER_CODEX_NODE_DEVICE_NOT_FOUND" : "MASTER_CODEX_NODE_DEVICE_OFFLINE", + lastValidatedAt: new Date().toISOString(), }); + await appendMasterAgentSystemReply( + `主 GPT 不在手机里直接登录。当前绑定设备 ${boundNodeLabel}${boundDevice ? " 不在线" : " 未找到"}，主 Agent 暂时无法通过这台设备对话。请先在该设备上登录 Codex / ChatGPT Plus，并确保 local-agent 在线后再重试。`, + `主 Agent · ${selectedMasterAccount.label || runtime.summary.roleLabel}`, + ); + return { ok: false as const, reason: "MASTER_NODE_OFFLINE" }; + } + const task = await queueMasterAgentTask({ + requestMessageId: params.requestMessageId ?? "master-agent-manual", + requestText: params.requestText, + executionPrompt: masterExecutionPrompt, + requestedBy: params.requestedBy, + requestedByAccount: params.requestedByAccount, + deviceId, + accountId: selectedMasterAccount.accountId, + accountLabel: selectedMasterAccount.label || runtime.summary.roleLabel, + }); + + if (params.mode === "enqueue") { const queuedReply: QueuedMasterAgentReplyEnvelope = { ok: true as const, - accountId: runtime.account.accountId, + accountId: selectedMasterAccount.accountId, taskId: task.taskId, masterReplyState: "queued" as const, task: { @@ -1477,117 +1672,16 @@ export async function replyToMasterAgentUserMessage(params: { return queuedReply; } - if (isApiCompatibleProvider(runtime.account.provider) && runtime.account.apiKey?.trim()) { - return enqueueOpenAiMasterAgentReply({ - provider: runtime.account.provider, - accountId: runtime.account.accountId, - accountLabel: runtime.account.label || runtime.summary.roleLabel, - requestMessageId: params.requestMessageId, - requestText: params.requestText, - requestedBy: params.requestedBy, - requestedByAccount: params.requestedByAccount, - currentSessionExpiresAt: params.currentSessionExpiresAt, - apiKey: runtime.account.apiKey, - model: executionConfig.model, - reasoningEffort: executionConfig.reasoningEffort, - agentControls, - promptPolicy: executionConfig.promptPolicy, - userPrompt: executionConfig.userPrompt, - projectMemories: executionConfig.projectMemories, - userMemories: executionConfig.userMemories, - }); - } - } - - if (runtime.account.provider === "master_codex_node") { - const state = await readState(); - const deviceId = runtime.account.nodeId || state.user.boundDeviceId || "mac-studio"; - const boundDevice = state.devices.find((device) => device.id === deviceId); - const boundNodeLabel = - runtime.account.nodeLabel?.trim() || - boundDevice?.name || - state.user.boundCodexNodeLabel || - deviceId; - - if (!boundDevice || boundDevice.status !== "online") { - await updateAiAccountHealth({ - accountId: runtime.account.accountId, - status: "degraded", - lastError: !boundDevice ? "MASTER_CODEX_NODE_DEVICE_NOT_FOUND" : "MASTER_CODEX_NODE_DEVICE_OFFLINE", - lastValidatedAt: new Date().toISOString(), - }); - const fallbackAccount = await findFallbackApiAccount(runtime.account.accountId); - if (fallbackAccount) { - try { - return await replyViaOpenAiAccount({ - account: fallbackAccount, - requestText: params.requestText, - currentSessionExpiresAt: params.currentSessionExpiresAt, - senderLabel: `主 Agent · ${fallbackAccount.label || aiRoleLabel(fallbackAccount.role)}`, - agentControls, - promptPolicy: executionConfig.promptPolicy, - userPrompt: executionConfig.userPrompt, - projectMemories: executionConfig.projectMemories, - userMemories: executionConfig.userMemories, - }); - } catch { - // Fall through to the original offline guidance when the fallback API account cannot respond. - } - } - await appendMasterAgentSystemReply( - `主 GPT 不在手机里直接登录。当前绑定设备 ${boundNodeLabel}${boundDevice ? " 不在线" : " 未找到"}，主 Agent 暂时无法通过这台设备对话。请先在该设备上登录 Codex / ChatGPT Plus，并确保 local-agent 在线后再重试。`, - `主 Agent · ${runtime.summary.roleLabel}`, - ); - return { ok: false as const, reason: "MASTER_NODE_OFFLINE" }; - } - - const task = await queueMasterAgentTask({ - requestMessageId: params.requestMessageId ?? "master-agent-manual", - requestText: params.requestText, - executionPrompt: buildMasterCodexNodePrompt( - state, - params.requestText, - params.currentSessionExpiresAt, - agentControls, - executionConfig.promptPolicy, - executionConfig.userPrompt, - executionConfig.projectMemories, - executionConfig.userMemories, - ), - requestedBy: params.requestedBy, - requestedByAccount: params.requestedByAccount, - deviceId, - accountId: runtime.account.accountId, - accountLabel: runtime.summary.roleLabel, - }); const completedTask = await waitForMasterAgentTaskCompletion(task.taskId); if (completedTask?.status === "completed") { return { ok: true as const, - accountId: runtime.account.accountId, + accountId: selectedMasterAccount.accountId, taskId: task.taskId, requestId: completedTask.requestId, }; } if (completedTask?.status === "failed") { - const fallbackAccount = await findFallbackApiAccount(runtime.account.accountId); - if (fallbackAccount) { - try { - return await replyViaOpenAiAccount({ - account: fallbackAccount, - requestText: params.requestText, - currentSessionExpiresAt: params.currentSessionExpiresAt, - senderLabel: `主 Agent · ${fallbackAccount.label || aiRoleLabel(fallbackAccount.role)}`, - agentControls, - promptPolicy: executionConfig.promptPolicy, - userPrompt: executionConfig.userPrompt, - projectMemories: executionConfig.projectMemories, - userMemories: executionConfig.userMemories, - }); - } catch { - // Preserve the original execution failure below if the fallback account also fails. - } - } return { ok: false as const, reason: "MASTER_NODE_EXEC_FAILED", @@ -1601,9 +1695,86 @@ export async function replyToMasterAgentUserMessage(params: { `当前主控身份是 ${runtime.summary.roleLabel}，任务已经转交到 ${boundNodeLabel} 的 Master Codex Node。`, "如果本机 Codex 节点在线，回复会在稍后自动回写到当前会话。", ].join(""), - `主 Agent · ${runtime.summary.roleLabel}`, + `主 Agent · ${selectedMasterAccount.label || runtime.summary.roleLabel}`, ); - return { ok: true as const, accountId: runtime.account.accountId, taskId: task.taskId }; + return { ok: true as const, accountId: selectedMasterAccount.accountId, taskId: task.taskId }; + }; + + if (params.mode === "enqueue") { + if (selectedBackend.backendId === "master-codex-node") { + return runMasterNodeExecution(); + } + + if (apiExecutionCandidates.length > 0) { + return enqueueOpenAiMasterAgentReply({ + candidates: apiExecutionCandidates, + requestMessageId: params.requestMessageId, + requestText: params.requestText, + requestedBy: params.requestedBy, + requestedByAccount: params.requestedByAccount, + currentSessionExpiresAt: params.currentSessionExpiresAt, + reasoningEffort: executionConfig.reasoningEffort, + agentControls, + promptPolicy: executionConfig.promptPolicy, + userPrompt: executionConfig.userPrompt, + projectMemories: executionConfig.projectMemories, + userMemories: executionConfig.userMemories, + masterFallback: hasMasterFallback && selectedMasterAccount + ? { + account: selectedMasterAccount, + executionPrompt: masterExecutionPrompt, + } + : null, + }); + } + } + + if (selectedBackend.backendId === "master-codex-node") { + return runMasterNodeExecution(); + } + + let lastApiFailureMessage: string | null = null; + let lastFailedAccount: AiAccount | null = null; + for (const candidate of apiExecutionCandidates) { + try { + return await replyViaOpenAiAccount({ + account: candidate.account, + requestText: params.requestText, + currentSessionExpiresAt: params.currentSessionExpiresAt, + senderLabel: `主 Agent · ${candidate.account.label || aiRoleLabel(candidate.account.role)}`, + agentControls, + promptPolicy: executionConfig.promptPolicy, + userPrompt: executionConfig.userPrompt, + projectMemories: executionConfig.projectMemories, + userMemories: executionConfig.userMemories, + }); + } catch (error) { + lastApiFailureMessage = error instanceof Error ? error.message : "主 Agent 当前调用模型失败。"; + lastFailedAccount = candidate.account; + if (!runtime.isEnvironmentFallback) { + await updateAiAccountHealth({ + accountId: candidate.account.accountId, + status: "degraded", + lastError: lastApiFailureMessage, + lastValidatedAt: new Date().toISOString(), + }); + } + } + } + + if (hasMasterFallback && selectedMasterAccount) { + return runMasterNodeExecution(); + } + + if (lastApiFailureMessage) { + await appendMasterAgentSystemReply( + [ + `我已经收到你的消息，但当前 AI 账号调用失败：${lastApiFailureMessage}。`, + "请到“我的 > AI 账号”检查 API Key、模型名或切换到其他 AI 账号后重试。", + ].join(""), + `主 Agent · ${lastFailedAccount?.label || runtime.summary.roleLabel}`, + ); + return { ok: false as const, reason: "MODEL_CALL_FAILED", message: lastApiFailureMessage }; } if (!isApiCompatibleProvider(runtime.account.provider) || !runtime.account.apiKey?.trim()) { @@ -1618,61 +1789,5 @@ export async function replyToMasterAgentUserMessage(params: { return { ok: false as const, reason: "MASTER_NODE_NOT_CONNECTED" }; } - try { - const generated = await generateApiProviderReply({ - provider: runtime.account.provider, - apiKey: runtime.account.apiKey, - model: executionConfig.model, - reasoningEffort: executionConfig.reasoningEffort, - requestText: params.requestText, - currentSessionExpiresAt: params.currentSessionExpiresAt, - agentControls, - promptPolicy: executionConfig.promptPolicy, - userPrompt: executionConfig.userPrompt, - projectMemories: executionConfig.projectMemories, - userMemories: executionConfig.userMemories, - }); - - await appendMasterAgentSystemReply( - generated.content, - `主 Agent · ${runtime.summary.roleLabel}`, - ); - - if (!runtime.isEnvironmentFallback) { - await updateAiAccountHealth({ - accountId: runtime.account.accountId, - status: "ready", - lastValidatedAt: new Date().toISOString(), - lastUsedAt: new Date().toISOString(), - activate: !runtime.account.isActive, - switchReason: runtime.account.isActive - ? runtime.account.switchReason - : `主 Agent 回复时自动切换到 ${runtime.account.label}`, - }); - } - - return { - ok: true as const, - accountId: runtime.account.accountId, - requestId: generated.requestId, - }; - } catch (error) { - const message = error instanceof Error ? error.message : "主 Agent 当前调用模型失败。"; - if (!runtime.isEnvironmentFallback) { - await updateAiAccountHealth({ - accountId: runtime.account.accountId, - status: "degraded", - lastError: message, - lastValidatedAt: new Date().toISOString(), - }); - } - await appendMasterAgentSystemReply( - [ - `我已经收到你的消息，但当前 AI 账号调用失败：${message}。`, - "请到“我的 > AI 账号”检查 API Key、模型名或切换到其他 AI 账号后重试。", - ].join(""), - `主 Agent · ${runtime.summary.roleLabel}`, - ); - return { ok: false as const, reason: "MODEL_CALL_FAILED", message }; - } + return { ok: false as const, reason: "MASTER_NODE_NOT_CONNECTED" }; } diff --git a/src/lib/execution/backend-selector.ts b/src/lib/execution/backend-selector.ts new file mode 100644 index 0000000..e9fa7f2 --- /dev/null +++ b/src/lib/execution/backend-selector.ts @@ -0,0 +1,97 @@ +import type { AiAccountStatus, AiProvider } from "@/lib/boss-data"; +import { + ALIYUN_QWEN_BACKEND, + isReadyAliyunQwenBackend, +} from "@/lib/execution/backends/aliyun-qwen-backend"; +import { + MASTER_CODEX_NODE_BACKEND, + isReadyMasterCodexNodeBackend, +} from "@/lib/execution/backends/master-codex-node-backend"; +import { OPENAI_BACKEND, isReadyOpenAiBackend } from "@/lib/execution/backends/openai-backend"; + +export interface ExecutionBackendSelectionInput { + primary: { + provider: AiProvider; + status: AiAccountStatus; + }; + backups: Array<{ + provider: AiProvider; + status: AiAccountStatus; + }>; +} + +export type ExecutionBackendChoice = + | typeof MASTER_CODEX_NODE_BACKEND + | typeof OPENAI_BACKEND + | typeof ALIYUN_QWEN_BACKEND; + +function resolveBackendByProvider(provider: AiProvider): ExecutionBackendChoice { + switch (provider) { + case "master_codex_node": + return MASTER_CODEX_NODE_BACKEND; + case "openai_api": + return OPENAI_BACKEND; + case "aliyun_qwen_api": + return ALIYUN_QWEN_BACKEND; + default: + return MASTER_CODEX_NODE_BACKEND; + } +} + +function isReadyBackend(choice: ExecutionBackendChoice, input: ExecutionBackendSelectionInput) { + const candidates = [ + ...(input.primary.provider === choice.provider ? [input.primary] : []), + ...input.backups.filter((item) => item.provider === choice.provider), + ]; + + return candidates.some((candidate) => { + switch (choice.backendId) { + case "aliyun-qwen": + return isReadyAliyunQwenBackend(candidate); + case "openai-api": + return isReadyOpenAiBackend(candidate); + case "master-codex-node": + return isReadyMasterCodexNodeBackend(candidate); + default: + return false; + } + }); +} + +export async function selectExecutionBackend( + input: ExecutionBackendSelectionInput, +): Promise { + return listExecutionBackendChoices(input)[0] ?? resolveBackendByProvider(input.primary.provider); +} + +export function listExecutionBackendChoices( + input: ExecutionBackendSelectionInput, +): ExecutionBackendChoice[] { + const primaryBackend = resolveBackendByProvider(input.primary.provider); + const ordered: ExecutionBackendChoice[] = []; + const seen = new Set(); + + const pushBackend = (backend: ExecutionBackendChoice) => { + if (seen.has(backend.backendId)) { + return; + } + ordered.push(backend); + seen.add(backend.backendId); + }; + + if (input.primary.status === "ready") { + pushBackend(primaryBackend); + } + + const orderedBackends: ExecutionBackendChoice[] = [ALIYUN_QWEN_BACKEND, OPENAI_BACKEND, MASTER_CODEX_NODE_BACKEND]; + for (const backend of orderedBackends) { + if (isReadyBackend(backend, input)) { + pushBackend(backend); + } + } + + pushBackend(primaryBackend); + return ordered; +} + +export const selectExecutionBackendForTesting = selectExecutionBackend; diff --git a/src/lib/execution/backends/aliyun-qwen-backend.ts b/src/lib/execution/backends/aliyun-qwen-backend.ts new file mode 100644 index 0000000..ed5900b --- /dev/null +++ b/src/lib/execution/backends/aliyun-qwen-backend.ts @@ -0,0 +1,22 @@ +import type { AiAccountStatus, AiProvider } from "@/lib/boss-data"; + +export interface ExecutionBackendDescriptor { + backendId: string; + provider: AiProvider; + label: string; + mode: "local" | "api"; +} + +export const ALIYUN_QWEN_BACKEND = { + backendId: "aliyun-qwen", + provider: "aliyun_qwen_api", + label: "阿里百炼 Qwen", + mode: "api", +} as const satisfies ExecutionBackendDescriptor; + +export function isReadyAliyunQwenBackend(input: { + provider: AiProvider; + status: AiAccountStatus; +}) { + return input.provider === ALIYUN_QWEN_BACKEND.provider && input.status === "ready"; +} diff --git a/src/lib/execution/backends/master-codex-node-backend.ts b/src/lib/execution/backends/master-codex-node-backend.ts new file mode 100644 index 0000000..120da27 --- /dev/null +++ b/src/lib/execution/backends/master-codex-node-backend.ts @@ -0,0 +1,22 @@ +import type { AiAccountStatus, AiProvider } from "@/lib/boss-data"; + +export interface ExecutionBackendDescriptor { + backendId: string; + provider: AiProvider; + label: string; + mode: "local" | "api"; +} + +export const MASTER_CODEX_NODE_BACKEND = { + backendId: "master-codex-node", + provider: "master_codex_node", + label: "Master Codex Node", + mode: "local", +} as const satisfies ExecutionBackendDescriptor; + +export function isReadyMasterCodexNodeBackend(input: { + provider: AiProvider; + status: AiAccountStatus; +}) { + return input.provider === MASTER_CODEX_NODE_BACKEND.provider && input.status === "ready"; +} diff --git a/src/lib/execution/backends/openai-backend.ts b/src/lib/execution/backends/openai-backend.ts new file mode 100644 index 0000000..f9c9700 --- /dev/null +++ b/src/lib/execution/backends/openai-backend.ts @@ -0,0 +1,19 @@ +import type { AiAccountStatus, AiProvider } from "@/lib/boss-data"; + +export interface ExecutionBackendDescriptor { + backendId: string; + provider: AiProvider; + label: string; + mode: "local" | "api"; +} + +export const OPENAI_BACKEND = { + backendId: "openai-api", + provider: "openai_api", + label: "OpenAI API", + mode: "api", +} as const satisfies ExecutionBackendDescriptor; + +export function isReadyOpenAiBackend(input: { provider: AiProvider; status: AiAccountStatus }) { + return input.provider === OPENAI_BACKEND.provider && input.status === "ready"; +} diff --git a/tests/execution-backend-selector.test.ts b/tests/execution-backend-selector.test.ts new file mode 100644 index 0000000..e8d554b --- /dev/null +++ b/tests/execution-backend-selector.test.ts @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { selectExecutionBackendForTesting } from "@/lib/execution/backend-selector"; + +test("selectExecutionBackendForTesting prefers the ready primary master codex node", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "master_codex_node", status: "ready" }, + backups: [ + { provider: "aliyun_qwen_api", status: "ready" }, + { provider: "openai_api", status: "ready" }, + ], + }); + + assert.equal(backend.backendId, "master-codex-node"); +}); + +test("selectExecutionBackendForTesting falls back to ready aliyun qwen before openai", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "master_codex_node", status: "degraded" }, + backups: [ + { provider: "openai_api", status: "ready" }, + { provider: "aliyun_qwen_api", status: "ready" }, + ], + }); + + assert.equal(backend.backendId, "aliyun-qwen"); +}); + +test("selectExecutionBackendForTesting falls back to ready openai when aliyun qwen is unavailable", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "master_codex_node", status: "degraded" }, + backups: [ + { provider: "openai_api", status: "ready" }, + { provider: "aliyun_qwen_api", status: "disabled" }, + ], + }); + + assert.equal(backend.backendId, "openai-api"); +}); + +test("selectExecutionBackendForTesting uses fixed backend order when an API primary is not ready", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "openai_api", status: "degraded" }, + backups: [ + { provider: "master_codex_node", status: "ready" }, + { provider: "aliyun_qwen_api", status: "ready" }, + ], + }); + + assert.equal(backend.backendId, "aliyun-qwen"); +}); + +test("selectExecutionBackendForTesting does not let an earlier disabled backup hide a later ready account", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "master_codex_node", status: "degraded" }, + backups: [ + { provider: "openai_api", status: "disabled" }, + { provider: "openai_api", status: "ready" }, + ], + }); + + assert.equal(backend.backendId, "openai-api"); +}); + +test("selectExecutionBackendForTesting falls back to master node last when higher-priority API backends are unavailable", async () => { + const backend = await selectExecutionBackendForTesting({ + primary: { provider: "openai_api", status: "degraded" }, + backups: [ + { provider: "aliyun_qwen_api", status: "disabled" }, + { provider: "master_codex_node", status: "ready" }, + ], + }); + + assert.equal(backend.backendId, "master-codex-node"); +}); diff --git a/tests/master-agent-message-queue.test.ts b/tests/master-agent-message-queue.test.ts index 915c623..59d4370 100644 --- a/tests/master-agent-message-queue.test.ts +++ b/tests/master-agent-message-queue.test.ts @@ -2,7 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import os from "node:os"; import path from "node:path"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { NextRequest } from "next/server"; let runtimeRoot = ""; @@ -71,9 +71,13 @@ test.after(async () => { } }); -test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异步实际回复时继承当前会话覆盖", async () => { +test.beforeEach(async () => { await setup(); + await rm(runtimeRoot, { recursive: true, force: true }); + await mkdir(runtimeRoot, { recursive: true }); +}); +test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异步实际回复时继承当前会话覆盖", async () => { await saveAiAccount({ accountId: "openai-master-agent-queue", label: "API 容灾", @@ -120,10 +124,11 @@ test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异 ok: boolean; task?: { taskId: string; taskType: string; status: string } | null; masterReplyState?: "queued" | "running" | "completed"; - masterReply?: unknown; + masterReply?: { accountId?: string } | null; }; assert.equal(payload.ok, true); + assert.equal(payload.masterReply?.accountId, "openai-master-agent-queue"); assert.equal(payload.masterReplyState, "queued"); assert.ok(payload.task, "expected master-agent message to return a task envelope"); assert.equal(payload.task?.taskType, "conversation_reply"); @@ -161,8 +166,6 @@ test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异 }); test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队列而不是挂到本机设备队列", async () => { - await setup(); - await saveAiAccount({ accountId: "master-codex-primary-offline", label: "主 GPT", @@ -214,8 +217,10 @@ test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队 ok: boolean; task?: { taskId: string; taskType: string; status: string } | null; masterReplyState?: "queued" | "running" | "completed"; + masterReply?: { accountId?: string } | null; }; assert.equal(payload.ok, true); + assert.equal(payload.masterReply?.accountId, "openai-backup-queue"); assert.equal(payload.masterReplyState, "queued"); assert.equal(payload.task?.taskType, "conversation_reply"); @@ -234,3 +239,150 @@ test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队 globalThis.fetch = originalFetch; } }); + +test("master-agent enqueue 在首选主节点离线时会回退到可用的备用主节点并返回实际账号", async () => { + await saveAiAccount({ + accountId: "master-codex-primary-offline", + label: "主 GPT", + role: "primary", + provider: "master_codex_node", + displayName: "离线 Master Codex Node", + nodeId: "offline-node", + nodeLabel: "离线节点", + model: "gpt-5.4", + enabled: true, + setActive: true, + loginStatusNote: "离线主节点", + }); + + await saveAiAccount({ + accountId: "master-codex-backup-online", + label: "备用主节点", + role: "backup", + provider: "master_codex_node", + displayName: "在线备用 Master Codex Node", + nodeId: "mac-studio", + nodeLabel: "Mac Studio", + model: "gpt-5.4", + enabled: true, + setActive: false, + loginStatusNote: "在线备用主节点", + }); + + const response = await POST( + await createAuthedRequest("master-agent", { + body: "请走备用主节点队列", + }), + { params: Promise.resolve({ projectId: "master-agent" }) }, + ); + + assert.equal(response.status, 200); + const payload = (await response.json()) as { + ok: boolean; + task?: { taskId: string; taskType: string; status: string } | null; + masterReplyState?: "queued" | "running" | "completed"; + masterReply?: { accountId?: string } | null; + }; + + assert.equal(payload.ok, true); + assert.equal(payload.masterReply?.accountId, "master-codex-backup-online"); + assert.equal(payload.masterReplyState, "queued"); + assert.equal(payload.task?.taskType, "conversation_reply"); + assert.equal(payload.task?.status, "queued"); + + const state = await readState(); + const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId); + assert.ok(task, "expected queued master-agent task"); + assert.equal(task?.accountId, "master-codex-backup-online"); + assert.equal(task?.deviceId, "mac-studio"); +}); + +test("master-agent enqueue 会在首个 API 候选失败后切到下一条备用链并重写任务账号", async () => { + await saveAiAccount({ + accountId: "openai-primary-queue", + label: "OpenAI 主控", + role: "primary", + provider: "openai_api", + displayName: "OpenAI 主账号", + model: "gpt-5.4", + apiKey: "sk-openai-primary-queue", + enabled: true, + setActive: true, + loginStatusNote: "OpenAI 主控", + }); + + await saveAiAccount({ + accountId: "aliyun-qwen-backup-queue", + label: "阿里备用", + role: "backup", + provider: "aliyun_qwen_api", + displayName: "阿里百炼备用账号", + model: "qwen3.5-plus", + apiKey: "sk-aliyun-backup-queue", + enabled: true, + setActive: false, + loginStatusNote: "阿里备用账号", + }); + + const fetchCalls: string[] = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input) => { + fetchCalls.push(String(input)); + if (typeof input === "string" && input === "https://api.openai.com/v1/responses") { + return new Response(JSON.stringify({ error: { message: "openai queue failure" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + } + if (typeof input === "string" && input === "https://dashscope.aliyuncs.com/compatible-mode/v1/responses") { + return new Response(JSON.stringify({ output_text: "后台队列已切到阿里备用。" }), { + status: 200, + headers: { + "content-type": "application/json", + "x-request-id": "req-master-agent-queue-fallback-chain", + }, + }); + } + throw new Error(`unexpected fetch: ${String(input)}`); + }) as typeof fetch; + + try { + const response = await POST( + await createAuthedRequest("master-agent", { + body: "请让后台队列自动切备用链", + }), + { params: Promise.resolve({ projectId: "master-agent" }) }, + ); + + assert.equal(response.status, 200); + const payload = (await response.json()) as { + ok: boolean; + task?: { taskId: string; taskType: string; status: string } | null; + masterReplyState?: "queued" | "running" | "completed"; + masterReply?: { accountId?: string } | null; + }; + + assert.equal(payload.ok, true); + assert.equal(payload.masterReply?.accountId, "openai-primary-queue"); + assert.equal(payload.masterReplyState, "queued"); + + await waitFor(async () => { + const state = await readState(); + const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId); + return task?.status === "completed"; + }); + + const state = await readState(); + const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId); + assert.ok(task, "expected queued task to remain in state"); + assert.equal(task?.status, "completed"); + assert.equal(task?.accountId, "aliyun-qwen-backup-queue"); + assert.equal(task?.deviceId, "master-agent-aliyun-qwen"); + const aliyunAccount = state.aiAccounts.find((item) => item.accountId === "aliyun-qwen-backup-queue"); + assert.equal(aliyunAccount?.isActive, true); + assert.equal(fetchCalls[0], "https://api.openai.com/v1/responses"); + assert.equal(fetchCalls[1], "https://dashscope.aliyuncs.com/compatible-mode/v1/responses"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/master-agent-openai-fallback.test.ts b/tests/master-agent-openai-fallback.test.ts index d69033b..5cd7920 100644 --- a/tests/master-agent-openai-fallback.test.ts +++ b/tests/master-agent-openai-fallback.test.ts @@ -2,12 +2,13 @@ import test from "node:test"; import assert from "node:assert/strict"; import os from "node:os"; import path from "node:path"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; let runtimeRoot = ""; let replyToMasterAgentUserMessage: (typeof import("../src/lib/boss-master-agent"))["replyToMasterAgentUserMessage"]; let saveAiAccount: (typeof import("../src/lib/boss-data"))["saveAiAccount"]; let readState: (typeof import("../src/lib/boss-data"))["readState"]; +let updateAiAccountHealth: (typeof import("../src/lib/boss-data"))["updateAiAccountHealth"]; async function setup() { if (runtimeRoot) return; @@ -24,6 +25,7 @@ async function setup() { replyToMasterAgentUserMessage = masterAgent.replyToMasterAgentUserMessage; saveAiAccount = data.saveAiAccount; readState = data.readState; + updateAiAccountHealth = data.updateAiAccountHealth; } test.after(async () => { @@ -32,9 +34,13 @@ test.after(async () => { } }); -test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account when the master node is offline", async () => { +test.beforeEach(async () => { await setup(); + await rm(runtimeRoot, { recursive: true, force: true }); + await mkdir(runtimeRoot, { recursive: true }); +}); +test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account when the master node is offline", async () => { await saveAiAccount({ accountId: "master-codex-primary", label: "主 GPT", @@ -101,9 +107,81 @@ test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account } }); -test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup account when the master node is offline", async () => { - await setup(); +test("replyToMasterAgentUserMessage can retry the same degraded API account when it is the only available backend", async () => { + await saveAiAccount({ + accountId: "master-codex-primary", + label: "主 GPT", + role: "primary", + provider: "master_codex_node", + displayName: "Mac 上的 Master Codex Node", + nodeId: "offline-node", + nodeLabel: "离线节点", + model: "gpt-5.4", + enabled: true, + setActive: false, + loginStatusNote: "测试中显式模拟默认主节点离线。", + }); + await updateAiAccountHealth({ + accountId: "master-codex-primary", + status: "degraded", + lastError: "MASTER_CODEX_NODE_DEVICE_OFFLINE", + lastValidatedAt: new Date().toISOString(), + }); + await saveAiAccount({ + accountId: "openai-primary-degraded", + label: "OpenAI 主控", + role: "primary", + provider: "openai_api", + displayName: "OpenAI 主账号", + model: "gpt-5.4", + apiKey: "sk-openai-only", + enabled: true, + setActive: true, + loginStatusNote: "唯一可用的 OpenAI 账号。", + }); + await updateAiAccountHealth({ + accountId: "openai-primary-degraded", + status: "degraded", + lastError: "temporary failure", + lastValidatedAt: new Date().toISOString(), + }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input) => { + if (typeof input === "string" && input === "https://api.openai.com/v1/responses") { + return new Response(JSON.stringify({ output_text: "仍然可以重试同一个 API 账号。" }), { + status: 200, + headers: { + "content-type": "application/json", + "x-request-id": "req-openai-degraded-retry", + }, + }); + } + throw new Error(`unexpected fetch: ${String(input)}`); + }) as typeof fetch; + + try { + const result = await replyToMasterAgentUserMessage({ + requestMessageId: "msg-openai-degraded-retry", + requestText: "请只回复：仍然可以重试同一个 API 账号。", + requestedBy: "Boss 超级管理员", + requestedByAccount: "17600003315", + }); + + assert.equal(result.ok, true); + assert.equal(result.accountId, "openai-primary-degraded"); + + const state = await readState(); + const account = state.aiAccounts.find((item) => item.accountId === "openai-primary-degraded"); + assert.equal(account?.status, "ready"); + assert.equal(account?.isActive, true); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup account when the master node is offline", async () => { await saveAiAccount({ accountId: "master-codex-primary", label: "主 GPT", @@ -169,3 +247,125 @@ test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup globalThis.fetch = originalFetch; } }); + +test("replyToMasterAgentUserMessage retries the next ready API backup when the first API backend call fails", async () => { + await saveAiAccount({ + accountId: "openai-primary-ready", + label: "OpenAI 主控", + role: "primary", + provider: "openai_api", + displayName: "OpenAI 主账号", + model: "gpt-5.4", + apiKey: "sk-openai-primary", + enabled: true, + setActive: true, + loginStatusNote: "主 OpenAI 账号。", + }); + + await saveAiAccount({ + accountId: "aliyun-qwen-backup", + label: "阿里备用", + role: "backup", + provider: "aliyun_qwen_api", + displayName: "阿里百炼备用账号", + accountIdentifier: "dashscope-demo", + model: "qwen3.5-plus", + apiKey: "sk-aliyun-demo-123456", + enabled: true, + setActive: false, + loginStatusNote: "阿里百炼 Qwen 备用账号。", + }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input) => { + if (typeof input === "string" && input === "https://api.openai.com/v1/responses") { + return new Response(JSON.stringify({ error: { message: "openai temporary failure" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + } + if (typeof input === "string" && input === "https://dashscope.aliyuncs.com/compatible-mode/v1/responses") { + return new Response(JSON.stringify({ output_text: "阿里备用接管成功。" }), { + status: 200, + headers: { + "content-type": "application/json", + "x-request-id": "req-master-api-chain", + }, + }); + } + throw new Error(`unexpected fetch: ${String(input)}`); + }) as typeof fetch; + + try { + const result = await replyToMasterAgentUserMessage({ + requestMessageId: "msg-master-api-chain", + requestText: "请只回复：阿里备用接管成功。", + requestedBy: "Boss 超级管理员", + requestedByAccount: "17600003315", + }); + + assert.equal(result.ok, true); + assert.equal(result.accountId, "aliyun-qwen-backup"); + assert.equal(result.requestId, "req-master-api-chain"); + + const state = await readState(); + const openaiAccount = state.aiAccounts.find((item) => item.accountId === "openai-primary-ready"); + assert.equal(openaiAccount?.status, "degraded"); + const aliyunAccount = state.aiAccounts.find((item) => item.accountId === "aliyun-qwen-backup"); + assert.equal(aliyunAccount?.isActive, true); + const masterProject = state.projects.find((project) => project.id === "master-agent"); + const reply = masterProject?.messages.at(-1); + assert.ok(reply, "expected a fallback reply to be appended"); + assert.match(reply?.body ?? "", /阿里备用接管成功/); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("replyToMasterAgentUserMessage falls back to a ready backup master node account when API backends are unavailable", async () => { + await saveAiAccount({ + accountId: "master-codex-primary-offline", + label: "主 GPT", + role: "primary", + provider: "master_codex_node", + displayName: "离线主节点", + nodeId: "offline-node", + nodeLabel: "离线节点", + model: "gpt-5.4", + enabled: true, + setActive: true, + loginStatusNote: "离线主节点。", + }); + + await saveAiAccount({ + accountId: "master-codex-backup-ready", + label: "备用主节点", + role: "backup", + provider: "master_codex_node", + displayName: "在线备用 Master Codex Node", + nodeId: "mac-studio", + nodeLabel: "Mac Studio", + model: "gpt-5.4", + enabled: true, + setActive: false, + loginStatusNote: "在线备用主节点。", + }); + + const result = await replyToMasterAgentUserMessage({ + requestMessageId: "msg-master-node-backup-fallback", + requestText: "请切到备用主节点。", + requestedBy: "Boss 超级管理员", + requestedByAccount: "17600003315", + mode: "enqueue", + }); + + assert.equal(result.ok, true); + assert.equal(result.accountId, "master-codex-backup-ready"); + assert.ok(result.taskId, "expected a queued master-agent task"); + + const state = await readState(); + const task = state.masterAgentTasks.find((item) => item.taskId === result.taskId); + assert.ok(task, "expected queued task to be written into state"); + assert.equal(task?.accountId, "master-codex-backup-ready"); + assert.equal(task?.deviceId, "mac-studio"); +});