refactor: add execution backend selection

2026-04-03 00:21:19 +08:00
parent a3a4f3e980
commit 8a62e72fd5
11 changed files with 1067 additions and 318 deletions
--- a/tests/execution-backend-selector.test.ts
+++ b/tests/execution-backend-selector.test.ts
@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { selectExecutionBackendForTesting } from "@/lib/execution/backend-selector";
+
+test("selectExecutionBackendForTesting prefers the ready primary master codex node", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "master_codex_node", status: "ready" },
+    backups: [
+      { provider: "aliyun_qwen_api", status: "ready" },
+      { provider: "openai_api", status: "ready" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "master-codex-node");
+});
+
+test("selectExecutionBackendForTesting falls back to ready aliyun qwen before openai", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "master_codex_node", status: "degraded" },
+    backups: [
+      { provider: "openai_api", status: "ready" },
+      { provider: "aliyun_qwen_api", status: "ready" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "aliyun-qwen");
+});
+
+test("selectExecutionBackendForTesting falls back to ready openai when aliyun qwen is unavailable", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "master_codex_node", status: "degraded" },
+    backups: [
+      { provider: "openai_api", status: "ready" },
+      { provider: "aliyun_qwen_api", status: "disabled" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "openai-api");
+});
+
+test("selectExecutionBackendForTesting uses fixed backend order when an API primary is not ready", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "openai_api", status: "degraded" },
+    backups: [
+      { provider: "master_codex_node", status: "ready" },
+      { provider: "aliyun_qwen_api", status: "ready" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "aliyun-qwen");
+});
+
+test("selectExecutionBackendForTesting does not let an earlier disabled backup hide a later ready account", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "master_codex_node", status: "degraded" },
+    backups: [
+      { provider: "openai_api", status: "disabled" },
+      { provider: "openai_api", status: "ready" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "openai-api");
+});
+
+test("selectExecutionBackendForTesting falls back to master node last when higher-priority API backends are unavailable", async () => {
+  const backend = await selectExecutionBackendForTesting({
+    primary: { provider: "openai_api", status: "degraded" },
+    backups: [
+      { provider: "aliyun_qwen_api", status: "disabled" },
+      { provider: "master_codex_node", status: "ready" },
+    ],
+  });
+
+  assert.equal(backend.backendId, "master-codex-node");
+});
--- a/tests/master-agent-message-queue.test.ts
+++ b/tests/master-agent-message-queue.test.ts
@@ -2,7 +2,7 @@ import test from "node:test";
 import assert from "node:assert/strict";
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { NextRequest } from "next/server";

 let runtimeRoot = "";
@@ -71,9 +71,13 @@ test.after(async () => {
  }
 });

-test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异步实际回复时继承当前会话覆盖", async () => {
+test.beforeEach(async () => {
  await setup();
+  await rm(runtimeRoot, { recursive: true, force: true });
+  await mkdir(runtimeRoot, { recursive: true });
+});

+test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异步实际回复时继承当前会话覆盖", async () => {
  await saveAiAccount({
    accountId: "openai-master-agent-queue",
    label: "API 容灾",
@@ -120,10 +124,11 @@ test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异
      ok: boolean;
      task?: { taskId: string; taskType: string; status: string } | null;
      masterReplyState?: "queued" | "running" | "completed";
-      masterReply?: unknown;
+      masterReply?: { accountId?: string } | null;
    };

    assert.equal(payload.ok, true);
+    assert.equal(payload.masterReply?.accountId, "openai-master-agent-queue");
    assert.equal(payload.masterReplyState, "queued");
    assert.ok(payload.task, "expected master-agent message to return a task envelope");
    assert.equal(payload.task?.taskType, "conversation_reply");
@@ -161,8 +166,6 @@ test("POST /api/v1/projects/master-agent/messages 快速返回队列态并在异
 });

 test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队列而不是挂到本机设备队列", async () => {
-  await setup();
-
  await saveAiAccount({
    accountId: "master-codex-primary-offline",
    label: "主 GPT",
@@ -214,8 +217,10 @@ test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队
      ok: boolean;
      task?: { taskId: string; taskType: string; status: string } | null;
      masterReplyState?: "queued" | "running" | "completed";
+      masterReply?: { accountId?: string } | null;
    };
    assert.equal(payload.ok, true);
+    assert.equal(payload.masterReply?.accountId, "openai-backup-queue");
    assert.equal(payload.masterReplyState, "queued");
    assert.equal(payload.task?.taskType, "conversation_reply");

@@ -234,3 +239,150 @@ test("master-agent enqueue 在主节点离线时会自动切到 OpenAI 后台队
    globalThis.fetch = originalFetch;
  }
 });
+
+test("master-agent enqueue 在首选主节点离线时会回退到可用的备用主节点并返回实际账号", async () => {
+  await saveAiAccount({
+    accountId: "master-codex-primary-offline",
+    label: "主 GPT",
+    role: "primary",
+    provider: "master_codex_node",
+    displayName: "离线 Master Codex Node",
+    nodeId: "offline-node",
+    nodeLabel: "离线节点",
+    model: "gpt-5.4",
+    enabled: true,
+    setActive: true,
+    loginStatusNote: "离线主节点",
+  });
+
+  await saveAiAccount({
+    accountId: "master-codex-backup-online",
+    label: "备用主节点",
+    role: "backup",
+    provider: "master_codex_node",
+    displayName: "在线备用 Master Codex Node",
+    nodeId: "mac-studio",
+    nodeLabel: "Mac Studio",
+    model: "gpt-5.4",
+    enabled: true,
+    setActive: false,
+    loginStatusNote: "在线备用主节点",
+  });
+
+  const response = await POST(
+    await createAuthedRequest("master-agent", {
+      body: "请走备用主节点队列",
+    }),
+    { params: Promise.resolve({ projectId: "master-agent" }) },
+  );
+
+  assert.equal(response.status, 200);
+  const payload = (await response.json()) as {
+    ok: boolean;
+    task?: { taskId: string; taskType: string; status: string } | null;
+    masterReplyState?: "queued" | "running" | "completed";
+    masterReply?: { accountId?: string } | null;
+  };
+
+  assert.equal(payload.ok, true);
+  assert.equal(payload.masterReply?.accountId, "master-codex-backup-online");
+  assert.equal(payload.masterReplyState, "queued");
+  assert.equal(payload.task?.taskType, "conversation_reply");
+  assert.equal(payload.task?.status, "queued");
+
+  const state = await readState();
+  const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId);
+  assert.ok(task, "expected queued master-agent task");
+  assert.equal(task?.accountId, "master-codex-backup-online");
+  assert.equal(task?.deviceId, "mac-studio");
+});
+
+test("master-agent enqueue 会在首个 API 候选失败后切到下一条备用链并重写任务账号", async () => {
+  await saveAiAccount({
+    accountId: "openai-primary-queue",
+    label: "OpenAI 主控",
+    role: "primary",
+    provider: "openai_api",
+    displayName: "OpenAI 主账号",
+    model: "gpt-5.4",
+    apiKey: "sk-openai-primary-queue",
+    enabled: true,
+    setActive: true,
+    loginStatusNote: "OpenAI 主控",
+  });
+
+  await saveAiAccount({
+    accountId: "aliyun-qwen-backup-queue",
+    label: "阿里备用",
+    role: "backup",
+    provider: "aliyun_qwen_api",
+    displayName: "阿里百炼备用账号",
+    model: "qwen3.5-plus",
+    apiKey: "sk-aliyun-backup-queue",
+    enabled: true,
+    setActive: false,
+    loginStatusNote: "阿里备用账号",
+  });
+
+  const fetchCalls: string[] = [];
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async (input) => {
+    fetchCalls.push(String(input));
+    if (typeof input === "string" && input === "https://api.openai.com/v1/responses") {
+      return new Response(JSON.stringify({ error: { message: "openai queue failure" } }), {
+        status: 500,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    if (typeof input === "string" && input === "https://dashscope.aliyuncs.com/compatible-mode/v1/responses") {
+      return new Response(JSON.stringify({ output_text: "后台队列已切到阿里备用。" }), {
+        status: 200,
+        headers: {
+          "content-type": "application/json",
+          "x-request-id": "req-master-agent-queue-fallback-chain",
+        },
+      });
+    }
+    throw new Error(`unexpected fetch: ${String(input)}`);
+  }) as typeof fetch;
+
+  try {
+    const response = await POST(
+      await createAuthedRequest("master-agent", {
+        body: "请让后台队列自动切备用链",
+      }),
+      { params: Promise.resolve({ projectId: "master-agent" }) },
+    );
+
+    assert.equal(response.status, 200);
+    const payload = (await response.json()) as {
+      ok: boolean;
+      task?: { taskId: string; taskType: string; status: string } | null;
+      masterReplyState?: "queued" | "running" | "completed";
+      masterReply?: { accountId?: string } | null;
+    };
+
+    assert.equal(payload.ok, true);
+    assert.equal(payload.masterReply?.accountId, "openai-primary-queue");
+    assert.equal(payload.masterReplyState, "queued");
+
+    await waitFor(async () => {
+      const state = await readState();
+      const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId);
+      return task?.status === "completed";
+    });
+
+    const state = await readState();
+    const task = state.masterAgentTasks.find((item) => item.taskId === payload.task?.taskId);
+    assert.ok(task, "expected queued task to remain in state");
+    assert.equal(task?.status, "completed");
+    assert.equal(task?.accountId, "aliyun-qwen-backup-queue");
+    assert.equal(task?.deviceId, "master-agent-aliyun-qwen");
+    const aliyunAccount = state.aiAccounts.find((item) => item.accountId === "aliyun-qwen-backup-queue");
+    assert.equal(aliyunAccount?.isActive, true);
+    assert.equal(fetchCalls[0], "https://api.openai.com/v1/responses");
+    assert.equal(fetchCalls[1], "https://dashscope.aliyuncs.com/compatible-mode/v1/responses");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
--- a/tests/master-agent-openai-fallback.test.ts
+++ b/tests/master-agent-openai-fallback.test.ts
@@ -2,12 +2,13 @@ import test from "node:test";
 import assert from "node:assert/strict";
 import os from "node:os";
 import path from "node:path";
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";

 let runtimeRoot = "";
 let replyToMasterAgentUserMessage: (typeof import("../src/lib/boss-master-agent"))["replyToMasterAgentUserMessage"];
 let saveAiAccount: (typeof import("../src/lib/boss-data"))["saveAiAccount"];
 let readState: (typeof import("../src/lib/boss-data"))["readState"];
+let updateAiAccountHealth: (typeof import("../src/lib/boss-data"))["updateAiAccountHealth"];

 async function setup() {
  if (runtimeRoot) return;
@@ -24,6 +25,7 @@ async function setup() {
  replyToMasterAgentUserMessage = masterAgent.replyToMasterAgentUserMessage;
  saveAiAccount = data.saveAiAccount;
  readState = data.readState;
+  updateAiAccountHealth = data.updateAiAccountHealth;
 }

 test.after(async () => {
@@ -32,9 +34,13 @@ test.after(async () => {
  }
 });

-test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account when the master node is offline", async () => {
+test.beforeEach(async () => {
  await setup();
+  await rm(runtimeRoot, { recursive: true, force: true });
+  await mkdir(runtimeRoot, { recursive: true });
+});

+test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account when the master node is offline", async () => {
  await saveAiAccount({
    accountId: "master-codex-primary",
    label: "主 GPT",
@@ -101,9 +107,81 @@ test("replyToMasterAgentUserMessage falls back to a runnable OpenAI API account
  }
 });

-test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup account when the master node is offline", async () => {
-  await setup();
+test("replyToMasterAgentUserMessage can retry the same degraded API account when it is the only available backend", async () => {
+  await saveAiAccount({
+    accountId: "master-codex-primary",
+    label: "主 GPT",
+    role: "primary",
+    provider: "master_codex_node",
+    displayName: "Mac 上的 Master Codex Node",
+    nodeId: "offline-node",
+    nodeLabel: "离线节点",
+    model: "gpt-5.4",
+    enabled: true,
+    setActive: false,
+    loginStatusNote: "测试中显式模拟默认主节点离线。",
+  });
+  await updateAiAccountHealth({
+    accountId: "master-codex-primary",
+    status: "degraded",
+    lastError: "MASTER_CODEX_NODE_DEVICE_OFFLINE",
+    lastValidatedAt: new Date().toISOString(),
+  });

+  await saveAiAccount({
+    accountId: "openai-primary-degraded",
+    label: "OpenAI 主控",
+    role: "primary",
+    provider: "openai_api",
+    displayName: "OpenAI 主账号",
+    model: "gpt-5.4",
+    apiKey: "sk-openai-only",
+    enabled: true,
+    setActive: true,
+    loginStatusNote: "唯一可用的 OpenAI 账号。",
+  });
+  await updateAiAccountHealth({
+    accountId: "openai-primary-degraded",
+    status: "degraded",
+    lastError: "temporary failure",
+    lastValidatedAt: new Date().toISOString(),
+  });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async (input) => {
+    if (typeof input === "string" && input === "https://api.openai.com/v1/responses") {
+      return new Response(JSON.stringify({ output_text: "仍然可以重试同一个 API 账号。" }), {
+        status: 200,
+        headers: {
+          "content-type": "application/json",
+          "x-request-id": "req-openai-degraded-retry",
+        },
+      });
+    }
+    throw new Error(`unexpected fetch: ${String(input)}`);
+  }) as typeof fetch;
+
+  try {
+    const result = await replyToMasterAgentUserMessage({
+      requestMessageId: "msg-openai-degraded-retry",
+      requestText: "请只回复：仍然可以重试同一个 API 账号。",
+      requestedBy: "Boss 超级管理员",
+      requestedByAccount: "17600003315",
+    });
+
+    assert.equal(result.ok, true);
+    assert.equal(result.accountId, "openai-primary-degraded");
+
+    const state = await readState();
+    const account = state.aiAccounts.find((item) => item.accountId === "openai-primary-degraded");
+    assert.equal(account?.status, "ready");
+    assert.equal(account?.isActive, true);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup account when the master node is offline", async () => {
  await saveAiAccount({
    accountId: "master-codex-primary",
    label: "主 GPT",
@@ -169,3 +247,125 @@ test("replyToMasterAgentUserMessage falls back to a runnable aliyun qwen backup
    globalThis.fetch = originalFetch;
  }
 });
+
+test("replyToMasterAgentUserMessage retries the next ready API backup when the first API backend call fails", async () => {
+  await saveAiAccount({
+    accountId: "openai-primary-ready",
+    label: "OpenAI 主控",
+    role: "primary",
+    provider: "openai_api",
+    displayName: "OpenAI 主账号",
+    model: "gpt-5.4",
+    apiKey: "sk-openai-primary",
+    enabled: true,
+    setActive: true,
+    loginStatusNote: "主 OpenAI 账号。",
+  });
+
+  await saveAiAccount({
+    accountId: "aliyun-qwen-backup",
+    label: "阿里备用",
+    role: "backup",
+    provider: "aliyun_qwen_api",
+    displayName: "阿里百炼备用账号",
+    accountIdentifier: "dashscope-demo",
+    model: "qwen3.5-plus",
+    apiKey: "sk-aliyun-demo-123456",
+    enabled: true,
+    setActive: false,
+    loginStatusNote: "阿里百炼 Qwen 备用账号。",
+  });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = (async (input) => {
+    if (typeof input === "string" && input === "https://api.openai.com/v1/responses") {
+      return new Response(JSON.stringify({ error: { message: "openai temporary failure" } }), {
+        status: 500,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    if (typeof input === "string" && input === "https://dashscope.aliyuncs.com/compatible-mode/v1/responses") {
+      return new Response(JSON.stringify({ output_text: "阿里备用接管成功。" }), {
+        status: 200,
+        headers: {
+          "content-type": "application/json",
+          "x-request-id": "req-master-api-chain",
+        },
+      });
+    }
+    throw new Error(`unexpected fetch: ${String(input)}`);
+  }) as typeof fetch;
+
+  try {
+    const result = await replyToMasterAgentUserMessage({
+      requestMessageId: "msg-master-api-chain",
+      requestText: "请只回复：阿里备用接管成功。",
+      requestedBy: "Boss 超级管理员",
+      requestedByAccount: "17600003315",
+    });
+
+    assert.equal(result.ok, true);
+    assert.equal(result.accountId, "aliyun-qwen-backup");
+    assert.equal(result.requestId, "req-master-api-chain");
+
+    const state = await readState();
+    const openaiAccount = state.aiAccounts.find((item) => item.accountId === "openai-primary-ready");
+    assert.equal(openaiAccount?.status, "degraded");
+    const aliyunAccount = state.aiAccounts.find((item) => item.accountId === "aliyun-qwen-backup");
+    assert.equal(aliyunAccount?.isActive, true);
+    const masterProject = state.projects.find((project) => project.id === "master-agent");
+    const reply = masterProject?.messages.at(-1);
+    assert.ok(reply, "expected a fallback reply to be appended");
+    assert.match(reply?.body ?? "", /阿里备用接管成功/);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test("replyToMasterAgentUserMessage falls back to a ready backup master node account when API backends are unavailable", async () => {
+  await saveAiAccount({
+    accountId: "master-codex-primary-offline",
+    label: "主 GPT",
+    role: "primary",
+    provider: "master_codex_node",
+    displayName: "离线主节点",
+    nodeId: "offline-node",
+    nodeLabel: "离线节点",
+    model: "gpt-5.4",
+    enabled: true,
+    setActive: true,
+    loginStatusNote: "离线主节点。",
+  });
+
+  await saveAiAccount({
+    accountId: "master-codex-backup-ready",
+    label: "备用主节点",
+    role: "backup",
+    provider: "master_codex_node",
+    displayName: "在线备用 Master Codex Node",
+    nodeId: "mac-studio",
+    nodeLabel: "Mac Studio",
+    model: "gpt-5.4",
+    enabled: true,
+    setActive: false,
+    loginStatusNote: "在线备用主节点。",
+  });
+
+  const result = await replyToMasterAgentUserMessage({
+    requestMessageId: "msg-master-node-backup-fallback",
+    requestText: "请切到备用主节点。",
+    requestedBy: "Boss 超级管理员",
+    requestedByAccount: "17600003315",
+    mode: "enqueue",
+  });
+
+  assert.equal(result.ok, true);
+  assert.equal(result.accountId, "master-codex-backup-ready");
+  assert.ok(result.taskId, "expected a queued master-agent task");
+
+  const state = await readState();
+  const task = state.masterAgentTasks.find((item) => item.taskId === result.taskId);
+  assert.ok(task, "expected queued task to be written into state");
+  assert.equal(task?.accountId, "master-codex-backup-ready");
+  assert.equal(task?.deviceId, "mac-studio");
+});