feat: route desktop control to authorized devices

.gitignore

@@ -29,6 +29,7 @@ apps/boss-admin-web/node_modules/
 .playwright-mcp/
 .superpowers/
 output/
+outputs/
 admin-redesign*.png
 main-*.js
 android/.project

eslint.config.mjs

@@ -11,6 +11,7 @@ const eslintConfig = defineConfig([
     ".next/**",
     "out/**",
     "build/**",
+    "outputs/**",
     "main-*.js",
     "android/.gradle/**",
     "android/**/build/**",

scripts/deploy-server.sh

@@ -77,6 +77,7 @@ RSYNC_EXCLUDES=(
   --exclude ".git"
   --exclude "node_modules"
   --exclude "data/"
+  --exclude "outputs/"
   --exclude "android/app/build"
   --exclude ".project"
   --exclude ".classpath"

scripts/ssh-computer-use-smoke.mjs

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+
+function writeJson(payload) {
+  process.stdout.write(`${JSON.stringify(payload)}\n`);
+}
+
+async function readStdin() {
+  const chunks = [];
+  for await (const chunk of process.stdin) {
+    chunks.push(typeof chunk === "string" ? chunk : chunk.toString("utf8"));
+  }
+  return chunks.join("").trim();
+}
+
+function parsePayload(raw) {
+  try {
+    const parsed = JSON.parse(raw || "{}");
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error("expected object");
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function envString(name) {
+  return String(process.env[name] || "").trim();
+}
+
+function envBoolean(name) {
+  return envString(name).toLowerCase() === "true";
+}
+
+function detectTargetApp(objective) {
+  const text = String(objective || "").toLowerCase();
+  const candidates = [
+    ["Chrome", ["chrome", "谷歌浏览器"]],
+    ["Safari", ["safari"]],
+    ["Finder", ["finder", "访达"]],
+    ["System Settings", ["system settings", "系统设置", "设置"]],
+    ["QQ", ["qq"]],
+    ["WeChat", ["wechat", "微信"]],
+    ["Telegram", ["telegram"]],
+  ];
+  for (const [name, aliases] of candidates) {
+    if (aliases.some((alias) => text.includes(alias.toLowerCase()))) {
+      return name;
+    }
+  }
+  return "Finder";
+}
+
+function extractQuotedText(objective) {
+  const text = String(objective || "");
+  const patterns = [
+    /[“"]([^“”"]+)[”"]/,
+    /[「『]([^」』]+)[」』]/,
+    /输入[:：]\s*([^\n。；;]+)/,
+    /打字[:：]\s*([^\n。；;]+)/,
+  ];
+  for (const pattern of patterns) {
+    const match = text.match(pattern);
+    const value = match?.[1]?.trim();
+    if (value) return value;
+  }
+  return undefined;
+}
+
+function shouldSubmitAfterTyping(objective) {
+  const text = String(objective || "").toLowerCase();
+  return text.includes("发送") || text.includes("提交") || text.includes("回车") || text.includes("enter");
+}
+
+function escapeAppleScriptString(value) {
+  return String(value || "").replaceAll("\\", "\\\\").replaceAll('"', '\\"');
+}
+
+function buildAppleScript(targetApp, objective) {
+  const lines = [
+    `tell application "${escapeAppleScriptString(targetApp)}"`,
+    "activate",
+    "end tell",
+  ];
+  const typedText = extractQuotedText(objective);
+  if (typedText) {
+    lines.push("delay 0.2");
+    lines.push('tell application "System Events"');
+    lines.push(`keystroke "${escapeAppleScriptString(typedText)}"`);
+    if (shouldSubmitAfterTyping(objective)) {
+      lines.push("key code 36");
+    }
+    lines.push("end tell");
+  }
+  return lines.join("\n");
+}
+
+function runCommand(command, args, env = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      env: {
+        ...process.env,
+        ...env,
+      },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (code) => {
+      if (code !== 0) {
+        reject(new Error(stderr.trim() || `ssh computer use exited with ${code}`));
+        return;
+      }
+      resolve({ stdout: stdout.trim(), stderr: stderr.trim() });
+    });
+  });
+}
+
+async function runRemoteAppleScript(script) {
+  const host = envString("BOSS_SSH_CONTROL_HOST");
+  const user = envString("BOSS_SSH_CONTROL_USER");
+  const port = envString("BOSS_SSH_CONTROL_PORT") || "22";
+  const password = envString("BOSS_SSH_CONTROL_PASSWORD");
+  if (!host) throw new Error("SSH_CONTROL_HOST_REQUIRED");
+  if (!user) throw new Error("SSH_CONTROL_USER_REQUIRED");
+
+  const sshTarget = `${user}@${host}`;
+  const encodedScript = Buffer.from(script, "utf8").toString("base64");
+  const remoteCommand = `printf '%s' '${encodedScript}' | base64 -D | osascript`;
+  const sshArgs = [
+    "-o",
+    "StrictHostKeyChecking=no",
+    "-o",
+    "ConnectTimeout=8",
+    "-o",
+    "PreferredAuthentications=password",
+    "-o",
+    "PubkeyAuthentication=no",
+    "-o",
+    "NumberOfPasswordPrompts=1",
+    "-p",
+    port,
+    sshTarget,
+    remoteCommand,
+  ];
+  if (password) {
+    await runCommand("sshpass", ["-e", "ssh", ...sshArgs], { SSHPASS: password });
+    return;
+  }
+  await runCommand("ssh", sshArgs);
+}
+
+const payload = parsePayload(await readStdin());
+if (!payload) {
+  writeJson({ status: "failed", error: "INVALID_SSH_COMPUTER_USE_PAYLOAD" });
+  process.exit(0);
+}
+
+const requestId = typeof payload.requestId === "string" ? payload.requestId : undefined;
+const objective = typeof payload.objective === "string" && payload.objective.trim()
+  ? payload.objective.trim()
+  : "远程桌面控制 smoke 链路测试";
+const targetApp = detectTargetApp(objective);
+const typedText = extractQuotedText(objective);
+
+try {
+  if (!envString("BOSS_SSH_CONTROL_HOST")) {
+    throw new Error("SSH_CONTROL_HOST_REQUIRED");
+  }
+  if (!envString("BOSS_SSH_CONTROL_USER")) {
+    throw new Error("SSH_CONTROL_USER_REQUIRED");
+  }
+  const appleScript = buildAppleScript(targetApp, objective);
+  if (!envBoolean("BOSS_SSH_CONTROL_DRY_RUN")) {
+    await runRemoteAppleScript(appleScript);
+  }
+  writeJson({
+    status: "completed",
+    requestId,
+    replyBody: `SSH 桌面控制已完成：${objective}`,
+    executionSummary: `ssh osascript ${envBoolean("BOSS_SSH_CONTROL_DRY_RUN") ? "dry-run" : "executed"} (${targetApp})`,
+    targetApp,
+    typedText,
+  });
+} catch (error) {
+  writeJson({
+    status: "failed",
+    requestId,
+    error: error instanceof Error ? error.message : "SSH_COMPUTER_USE_FAILED",
+  });
+}

src/lib/boss-master-agent.ts

@@ -308,6 +308,62 @@ export function classifyMasterAgentControlIntent(
 
 export const classifyMasterAgentControlIntentForTesting = classifyMasterAgentControlIntent;
 
+type ControlTargetDeviceInput = {
+  replyProjectId: string;
+  intentCategory: "browser_control" | "desktop_control";
+  preferredDeviceId?: string;
+  authorizedDeviceIds: string[];
+  devices: Array<{
+    id: string;
+    status?: string;
+    capabilities?: {
+      browserAutomation?: { connected?: boolean };
+      computerUse?: { connected?: boolean };
+    };
+  }>;
+  projects: Array<{
+    id: string;
+    deviceIds?: string[];
+  }>;
+};
+
+function isControlDeviceCapable(
+  device: ControlTargetDeviceInput["devices"][number] | undefined,
+  intentCategory: ControlTargetDeviceInput["intentCategory"],
+) {
+  if (!device || device.status !== "online") return false;
+  if (intentCategory === "browser_control") {
+    return device.capabilities?.browserAutomation?.connected === true;
+  }
+  return device.capabilities?.computerUse?.connected === true;
+}
+
+function resolveMasterAgentControlTargetDeviceId(input: ControlTargetDeviceInput) {
+  const authorized = new Set(input.authorizedDeviceIds);
+  const deviceById = new Map(input.devices.map((device) => [device.id, device]));
+  const project = input.projects.find((item) => item.id === input.replyProjectId);
+  const projectDevice = project?.deviceIds
+    ?.find((deviceId) => authorized.has(deviceId) && isControlDeviceCapable(deviceById.get(deviceId), input.intentCategory));
+  if (projectDevice) return projectDevice;
+
+  const authorizedCapableDevices = input.authorizedDeviceIds.filter((deviceId) =>
+    isControlDeviceCapable(deviceById.get(deviceId), input.intentCategory),
+  );
+  if (authorizedCapableDevices.length === 1) return authorizedCapableDevices[0];
+
+  if (
+    input.preferredDeviceId &&
+    authorized.has(input.preferredDeviceId) &&
+    isControlDeviceCapable(deviceById.get(input.preferredDeviceId), input.intentCategory)
+  ) {
+    return input.preferredDeviceId;
+  }
+
+  return authorizedCapableDevices[0] ?? input.preferredDeviceId ?? input.authorizedDeviceIds[0] ?? "mac-studio";
+}
+
+export const resolveMasterAgentControlTargetDeviceIdForTesting = resolveMasterAgentControlTargetDeviceId;
+
 const GENERIC_COMPATIBLE_MODEL_OPTIONS = ["gpt-5.4-mini", "gpt-5.4", "gpt-5.1", "gpt-4.1"];
 
 type QueuedMasterAgentReplyEnvelope = {
@@ -3709,7 +3765,14 @@ export async function replyToMasterAgentUserMessage(params: {
   if (
     controlIntent.intentCategory === "browser_control" || controlIntent.intentCategory === "desktop_control"
   ) {
-    const deviceId = runtime.account.nodeId || state.user.boundDeviceId || "mac-studio";
+    const deviceId = resolveMasterAgentControlTargetDeviceId({
+      replyProjectId,
+      intentCategory: controlIntent.intentCategory,
+      preferredDeviceId: runtime.account.nodeId || state.user.boundDeviceId || "mac-studio",
+      authorizedDeviceIds: authorizedScope.authorizedDeviceIds,
+      devices: state.devices,
+      projects: state.projects,
+    });
     const taskType = controlIntent.intentCategory;
     const task = await queueMasterAgentTask({
       projectId: replyProjectId,
@@ -3803,6 +3866,14 @@ export async function replyToMasterAgentUserMessage(params: {
           ? "browser-automation-runtime"
           : "computer-use-runtime";
       const taskType = controlIntent.intentCategory;
+      const controlDeviceId = resolveMasterAgentControlTargetDeviceId({
+        replyProjectId,
+        intentCategory: controlIntent.intentCategory,
+        preferredDeviceId: deviceId,
+        authorizedDeviceIds: authorizedScope.authorizedDeviceIds,
+        devices: state.devices,
+        projects: state.projects,
+      });
       const task = await queueMasterAgentTask({
         projectId: replyProjectId,
         taskType,
@@ -3811,7 +3882,7 @@ export async function replyToMasterAgentUserMessage(params: {
         executionPrompt: masterExecutionPrompt,
         requestedBy: params.requestedBy,
         requestedByAccount: params.requestedByAccount,
-        deviceId,
+        deviceId: controlDeviceId,
         accountId: selectedMasterAccount.accountId,
         accountLabel: selectedMasterAccount.label || runtime.summary.roleLabel,
         ...masterTaskAuthorization(["master_agent.ask", "computer.control"]),
@@ -3820,7 +3891,7 @@ export async function replyToMasterAgentUserMessage(params: {
         riskLevel: controlIntent.riskLevel,
         confirmationPolicy: controlIntent.riskLevel === "high" ? "strong_confirm" : "light_confirm",
         requiresUserConfirmation: false,
-        confirmationScopeKey: `${deviceId}:${replyProjectId}`,
+        confirmationScopeKey: `${controlDeviceId}:${replyProjectId}`,
         externalReplyTarget: params.externalReplyTarget,
       });
 

tests/master-agent-control-intent-routing.test.ts

@@ -1,6 +1,9 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import { classifyMasterAgentControlIntentForTesting } from "@/lib/boss-master-agent";
+import {
+  classifyMasterAgentControlIntentForTesting,
+  resolveMasterAgentControlTargetDeviceIdForTesting,
+} from "@/lib/boss-master-agent";
 
 test("routes ordinary product discussion to discussion_only", () => {
   const result = classifyMasterAgentControlIntentForTesting("帮我总结一下这个项目当前目标");
@@ -27,3 +30,56 @@ test("routes desktop gui asks to desktop_control", () => {
   assert.equal(result.executionMode, "desktop");
   assert.equal(result.riskLevel, "medium");
 });
+
+test("routes desktop control to the current project device before global master node", () => {
+  const deviceId = resolveMasterAgentControlTargetDeviceIdForTesting({
+    replyProjectId: "macbook-project",
+    intentCategory: "desktop_control",
+    preferredDeviceId: "mac-studio",
+    authorizedDeviceIds: ["macbook-air"],
+    devices: [
+      {
+        id: "mac-studio",
+        status: "online",
+        capabilities: { computerUse: { connected: true } },
+      },
+      {
+        id: "macbook-air",
+        status: "online",
+        capabilities: { computerUse: { connected: true } },
+      },
+    ],
+    projects: [
+      {
+        id: "macbook-project",
+        deviceIds: ["macbook-air"],
+      },
+    ],
+  });
+
+  assert.equal(deviceId, "macbook-air");
+});
+
+test("routes desktop control to the only authorized capable device for a subaccount", () => {
+  const deviceId = resolveMasterAgentControlTargetDeviceIdForTesting({
+    replyProjectId: "master-agent",
+    intentCategory: "desktop_control",
+    preferredDeviceId: "mac-studio",
+    authorizedDeviceIds: ["macbook-air"],
+    devices: [
+      {
+        id: "mac-studio",
+        status: "online",
+        capabilities: { computerUse: { connected: true } },
+      },
+      {
+        id: "macbook-air",
+        status: "online",
+        capabilities: { computerUse: { connected: true } },
+      },
+    ],
+    projects: [{ id: "master-agent", deviceIds: [] }],
+  });
+
+  assert.equal(deviceId, "macbook-air");
+});

tests/ssh-computer-use-smoke.test.mjs

@@ -0,0 +1,70 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import path from "node:path";
+
+const repoRoot = path.resolve(import.meta.dirname, "..");
+
+function runRuntime(payload, env = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, ["scripts/ssh-computer-use-smoke.mjs"], {
+      cwd: repoRoot,
+      env: {
+        ...process.env,
+        ...env,
+      },
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("close", (status) => {
+      resolve({ status, stdout, stderr });
+    });
+    child.stdin.end(JSON.stringify(payload));
+  });
+}
+
+test("ssh computer use runtime returns a dry-run desktop control summary", async () => {
+  const result = await runRuntime(
+    {
+      requestKind: "desktop_control",
+      requestId: "ssh-dry-run-1",
+      objective: "打开 Chrome 输入“Boss 远程控制测试”",
+    },
+    {
+      BOSS_SSH_CONTROL_HOST: "192.168.31.114",
+      BOSS_SSH_CONTROL_USER: "jas",
+      BOSS_SSH_CONTROL_DRY_RUN: "true",
+    },
+  );
+
+  assert.equal(result.status, 0, result.stderr);
+  const payload = JSON.parse(result.stdout);
+  assert.equal(payload.status, "completed");
+  assert.equal(payload.requestId, "ssh-dry-run-1");
+  assert.equal(payload.targetApp, "Chrome");
+  assert.equal(payload.typedText, "Boss 远程控制测试");
+  assert.match(payload.replyBody, /SSH 桌面控制已完成/);
+});
+
+test("ssh computer use runtime fails closed when host is missing", async () => {
+  const result = await runRuntime({
+    requestKind: "desktop_control",
+    requestId: "ssh-missing-host",
+    objective: "打开 Finder",
+  });
+
+  assert.equal(result.status, 0, result.stderr);
+  const payload = JSON.parse(result.stdout);
+  assert.equal(payload.status, "failed");
+  assert.equal(payload.requestId, "ssh-missing-host");
+  assert.equal(payload.error, "SSH_CONTROL_HOST_REQUIRED");
+});

tsconfig.json

@@ -30,5 +30,5 @@
     ".next/dev/types/**/*.ts",
     "**/*.mts"
   ],
-  "exclude": ["node_modules", "apps/boss-admin-web/**"]
+  "exclude": ["node_modules", "apps/boss-admin-web/**", "outputs/**"]
 }