From 4de64ac01cd1a09bb1d4771d1c80d2f069c956d4 Mon Sep 17 00:00:00 2001 From: AI Bot Date: Tue, 12 May 2026 12:15:43 +0800 Subject: [PATCH] feat: route desktop control to authorized devices --- .gitignore | 1 + eslint.config.mjs | 1 + scripts/deploy-server.sh | 1 + scripts/ssh-computer-use-smoke.mjs | 202 ++++++++++++++++++ src/lib/boss-master-agent.ts | 77 ++++++- ...aster-agent-control-intent-routing.test.ts | 58 ++++- tests/ssh-computer-use-smoke.test.mjs | 70 ++++++ tsconfig.json | 2 +- 8 files changed, 407 insertions(+), 5 deletions(-) create mode 100755 scripts/ssh-computer-use-smoke.mjs create mode 100644 tests/ssh-computer-use-smoke.test.mjs diff --git a/.gitignore b/.gitignore index a2fb2af..809df85 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ apps/boss-admin-web/node_modules/ .playwright-mcp/ .superpowers/ output/ +outputs/ admin-redesign*.png main-*.js android/.project diff --git a/eslint.config.mjs b/eslint.config.mjs index c8a7bd3..065bd84 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -11,6 +11,7 @@ const eslintConfig = defineConfig([ ".next/**", "out/**", "build/**", + "outputs/**", "main-*.js", "android/.gradle/**", "android/**/build/**", diff --git a/scripts/deploy-server.sh b/scripts/deploy-server.sh index 68f3121..c6975ff 100755 --- a/scripts/deploy-server.sh +++ b/scripts/deploy-server.sh @@ -77,6 +77,7 @@ RSYNC_EXCLUDES=( --exclude ".git" --exclude "node_modules" --exclude "data/" + --exclude "outputs/" --exclude "android/app/build" --exclude ".project" --exclude ".classpath" diff --git a/scripts/ssh-computer-use-smoke.mjs b/scripts/ssh-computer-use-smoke.mjs new file mode 100755 index 0000000..158ee7c --- /dev/null +++ b/scripts/ssh-computer-use-smoke.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; + +function writeJson(payload) { + process.stdout.write(`${JSON.stringify(payload)}\n`); +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(typeof chunk === "string" ? chunk : chunk.toString("utf8")); + } + return chunks.join("").trim(); +} + +function parsePayload(raw) { + try { + const parsed = JSON.parse(raw || "{}"); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("expected object"); + } + return parsed; + } catch { + return null; + } +} + +function envString(name) { + return String(process.env[name] || "").trim(); +} + +function envBoolean(name) { + return envString(name).toLowerCase() === "true"; +} + +function detectTargetApp(objective) { + const text = String(objective || "").toLowerCase(); + const candidates = [ + ["Chrome", ["chrome", "谷歌浏览器"]], + ["Safari", ["safari"]], + ["Finder", ["finder", "访达"]], + ["System Settings", ["system settings", "系统设置", "设置"]], + ["QQ", ["qq"]], + ["WeChat", ["wechat", "微信"]], + ["Telegram", ["telegram"]], + ]; + for (const [name, aliases] of candidates) { + if (aliases.some((alias) => text.includes(alias.toLowerCase()))) { + return name; + } + } + return "Finder"; +} + +function extractQuotedText(objective) { + const text = String(objective || ""); + const patterns = [ + /[“"]([^“”"]+)[”"]/, + /[「『]([^」』]+)[」』]/, + /输入[:：]\s*([^\n。；;]+)/, + /打字[:：]\s*([^\n。；;]+)/, + ]; + for (const pattern of patterns) { + const match = text.match(pattern); + const value = match?.[1]?.trim(); + if (value) return value; + } + return undefined; +} + +function shouldSubmitAfterTyping(objective) { + const text = String(objective || "").toLowerCase(); + return text.includes("发送") || text.includes("提交") || text.includes("回车") || text.includes("enter"); +} + +function escapeAppleScriptString(value) { + return String(value || "").replaceAll("\\", "\\\\").replaceAll('"', '\\"'); +} + +function buildAppleScript(targetApp, objective) { + const lines = [ + `tell application "${escapeAppleScriptString(targetApp)}"`, + "activate", + "end tell", + ]; + const typedText = extractQuotedText(objective); + if (typedText) { + lines.push("delay 0.2"); + lines.push('tell application "System Events"'); + lines.push(`keystroke "${escapeAppleScriptString(typedText)}"`); + if (shouldSubmitAfterTyping(objective)) { + lines.push("key code 36"); + } + lines.push("end tell"); + } + return lines.join("\n"); +} + +function runCommand(command, args, env = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + env: { + ...process.env, + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(stderr.trim() || `ssh computer use exited with ${code}`)); + return; + } + resolve({ stdout: stdout.trim(), stderr: stderr.trim() }); + }); + }); +} + +async function runRemoteAppleScript(script) { + const host = envString("BOSS_SSH_CONTROL_HOST"); + const user = envString("BOSS_SSH_CONTROL_USER"); + const port = envString("BOSS_SSH_CONTROL_PORT") || "22"; + const password = envString("BOSS_SSH_CONTROL_PASSWORD"); + if (!host) throw new Error("SSH_CONTROL_HOST_REQUIRED"); + if (!user) throw new Error("SSH_CONTROL_USER_REQUIRED"); + + const sshTarget = `${user}@${host}`; + const encodedScript = Buffer.from(script, "utf8").toString("base64"); + const remoteCommand = `printf '%s' '${encodedScript}' | base64 -D | osascript`; + const sshArgs = [ + "-o", + "StrictHostKeyChecking=no", + "-o", + "ConnectTimeout=8", + "-o", + "PreferredAuthentications=password", + "-o", + "PubkeyAuthentication=no", + "-o", + "NumberOfPasswordPrompts=1", + "-p", + port, + sshTarget, + remoteCommand, + ]; + if (password) { + await runCommand("sshpass", ["-e", "ssh", ...sshArgs], { SSHPASS: password }); + return; + } + await runCommand("ssh", sshArgs); +} + +const payload = parsePayload(await readStdin()); +if (!payload) { + writeJson({ status: "failed", error: "INVALID_SSH_COMPUTER_USE_PAYLOAD" }); + process.exit(0); +} + +const requestId = typeof payload.requestId === "string" ? payload.requestId : undefined; +const objective = typeof payload.objective === "string" && payload.objective.trim() + ? payload.objective.trim() + : "远程桌面控制 smoke 链路测试"; +const targetApp = detectTargetApp(objective); +const typedText = extractQuotedText(objective); + +try { + if (!envString("BOSS_SSH_CONTROL_HOST")) { + throw new Error("SSH_CONTROL_HOST_REQUIRED"); + } + if (!envString("BOSS_SSH_CONTROL_USER")) { + throw new Error("SSH_CONTROL_USER_REQUIRED"); + } + const appleScript = buildAppleScript(targetApp, objective); + if (!envBoolean("BOSS_SSH_CONTROL_DRY_RUN")) { + await runRemoteAppleScript(appleScript); + } + writeJson({ + status: "completed", + requestId, + replyBody: `SSH 桌面控制已完成：${objective}`, + executionSummary: `ssh osascript ${envBoolean("BOSS_SSH_CONTROL_DRY_RUN") ? "dry-run" : "executed"} (${targetApp})`, + targetApp, + typedText, + }); +} catch (error) { + writeJson({ + status: "failed", + requestId, + error: error instanceof Error ? error.message : "SSH_COMPUTER_USE_FAILED", + }); +} diff --git a/src/lib/boss-master-agent.ts b/src/lib/boss-master-agent.ts index b1c6bfd..0387da0 100644 --- a/src/lib/boss-master-agent.ts +++ b/src/lib/boss-master-agent.ts @@ -308,6 +308,62 @@ export function classifyMasterAgentControlIntent( export const classifyMasterAgentControlIntentForTesting = classifyMasterAgentControlIntent; +type ControlTargetDeviceInput = { + replyProjectId: string; + intentCategory: "browser_control" | "desktop_control"; + preferredDeviceId?: string; + authorizedDeviceIds: string[]; + devices: Array<{ + id: string; + status?: string; + capabilities?: { + browserAutomation?: { connected?: boolean }; + computerUse?: { connected?: boolean }; + }; + }>; + projects: Array<{ + id: string; + deviceIds?: string[]; + }>; +}; + +function isControlDeviceCapable( + device: ControlTargetDeviceInput["devices"][number] | undefined, + intentCategory: ControlTargetDeviceInput["intentCategory"], +) { + if (!device || device.status !== "online") return false; + if (intentCategory === "browser_control") { + return device.capabilities?.browserAutomation?.connected === true; + } + return device.capabilities?.computerUse?.connected === true; +} + +function resolveMasterAgentControlTargetDeviceId(input: ControlTargetDeviceInput) { + const authorized = new Set(input.authorizedDeviceIds); + const deviceById = new Map(input.devices.map((device) => [device.id, device])); + const project = input.projects.find((item) => item.id === input.replyProjectId); + const projectDevice = project?.deviceIds + ?.find((deviceId) => authorized.has(deviceId) && isControlDeviceCapable(deviceById.get(deviceId), input.intentCategory)); + if (projectDevice) return projectDevice; + + const authorizedCapableDevices = input.authorizedDeviceIds.filter((deviceId) => + isControlDeviceCapable(deviceById.get(deviceId), input.intentCategory), + ); + if (authorizedCapableDevices.length === 1) return authorizedCapableDevices[0]; + + if ( + input.preferredDeviceId && + authorized.has(input.preferredDeviceId) && + isControlDeviceCapable(deviceById.get(input.preferredDeviceId), input.intentCategory) + ) { + return input.preferredDeviceId; + } + + return authorizedCapableDevices[0] ?? input.preferredDeviceId ?? input.authorizedDeviceIds[0] ?? "mac-studio"; +} + +export const resolveMasterAgentControlTargetDeviceIdForTesting = resolveMasterAgentControlTargetDeviceId; + const GENERIC_COMPATIBLE_MODEL_OPTIONS = ["gpt-5.4-mini", "gpt-5.4", "gpt-5.1", "gpt-4.1"]; type QueuedMasterAgentReplyEnvelope = { @@ -3709,7 +3765,14 @@ export async function replyToMasterAgentUserMessage(params: { if ( controlIntent.intentCategory === "browser_control" || controlIntent.intentCategory === "desktop_control" ) { - const deviceId = runtime.account.nodeId || state.user.boundDeviceId || "mac-studio"; + const deviceId = resolveMasterAgentControlTargetDeviceId({ + replyProjectId, + intentCategory: controlIntent.intentCategory, + preferredDeviceId: runtime.account.nodeId || state.user.boundDeviceId || "mac-studio", + authorizedDeviceIds: authorizedScope.authorizedDeviceIds, + devices: state.devices, + projects: state.projects, + }); const taskType = controlIntent.intentCategory; const task = await queueMasterAgentTask({ projectId: replyProjectId, @@ -3803,6 +3866,14 @@ export async function replyToMasterAgentUserMessage(params: { ? "browser-automation-runtime" : "computer-use-runtime"; const taskType = controlIntent.intentCategory; + const controlDeviceId = resolveMasterAgentControlTargetDeviceId({ + replyProjectId, + intentCategory: controlIntent.intentCategory, + preferredDeviceId: deviceId, + authorizedDeviceIds: authorizedScope.authorizedDeviceIds, + devices: state.devices, + projects: state.projects, + }); const task = await queueMasterAgentTask({ projectId: replyProjectId, taskType, @@ -3811,7 +3882,7 @@ export async function replyToMasterAgentUserMessage(params: { executionPrompt: masterExecutionPrompt, requestedBy: params.requestedBy, requestedByAccount: params.requestedByAccount, - deviceId, + deviceId: controlDeviceId, accountId: selectedMasterAccount.accountId, accountLabel: selectedMasterAccount.label || runtime.summary.roleLabel, ...masterTaskAuthorization(["master_agent.ask", "computer.control"]), @@ -3820,7 +3891,7 @@ export async function replyToMasterAgentUserMessage(params: { riskLevel: controlIntent.riskLevel, confirmationPolicy: controlIntent.riskLevel === "high" ? "strong_confirm" : "light_confirm", requiresUserConfirmation: false, - confirmationScopeKey: `${deviceId}:${replyProjectId}`, + confirmationScopeKey: `${controlDeviceId}:${replyProjectId}`, externalReplyTarget: params.externalReplyTarget, }); diff --git a/tests/master-agent-control-intent-routing.test.ts b/tests/master-agent-control-intent-routing.test.ts index 21f0f8a..d5cc5a5 100644 --- a/tests/master-agent-control-intent-routing.test.ts +++ b/tests/master-agent-control-intent-routing.test.ts @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { classifyMasterAgentControlIntentForTesting } from "@/lib/boss-master-agent"; +import { + classifyMasterAgentControlIntentForTesting, + resolveMasterAgentControlTargetDeviceIdForTesting, +} from "@/lib/boss-master-agent"; test("routes ordinary product discussion to discussion_only", () => { const result = classifyMasterAgentControlIntentForTesting("帮我总结一下这个项目当前目标"); @@ -27,3 +30,56 @@ test("routes desktop gui asks to desktop_control", () => { assert.equal(result.executionMode, "desktop"); assert.equal(result.riskLevel, "medium"); }); + +test("routes desktop control to the current project device before global master node", () => { + const deviceId = resolveMasterAgentControlTargetDeviceIdForTesting({ + replyProjectId: "macbook-project", + intentCategory: "desktop_control", + preferredDeviceId: "mac-studio", + authorizedDeviceIds: ["macbook-air"], + devices: [ + { + id: "mac-studio", + status: "online", + capabilities: { computerUse: { connected: true } }, + }, + { + id: "macbook-air", + status: "online", + capabilities: { computerUse: { connected: true } }, + }, + ], + projects: [ + { + id: "macbook-project", + deviceIds: ["macbook-air"], + }, + ], + }); + + assert.equal(deviceId, "macbook-air"); +}); + +test("routes desktop control to the only authorized capable device for a subaccount", () => { + const deviceId = resolveMasterAgentControlTargetDeviceIdForTesting({ + replyProjectId: "master-agent", + intentCategory: "desktop_control", + preferredDeviceId: "mac-studio", + authorizedDeviceIds: ["macbook-air"], + devices: [ + { + id: "mac-studio", + status: "online", + capabilities: { computerUse: { connected: true } }, + }, + { + id: "macbook-air", + status: "online", + capabilities: { computerUse: { connected: true } }, + }, + ], + projects: [{ id: "master-agent", deviceIds: [] }], + }); + + assert.equal(deviceId, "macbook-air"); +}); diff --git a/tests/ssh-computer-use-smoke.test.mjs b/tests/ssh-computer-use-smoke.test.mjs new file mode 100644 index 0000000..45f180c --- /dev/null +++ b/tests/ssh-computer-use-smoke.test.mjs @@ -0,0 +1,70 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import path from "node:path"; + +const repoRoot = path.resolve(import.meta.dirname, ".."); + +function runRuntime(payload, env = {}) { + return new Promise((resolve) => { + const child = spawn(process.execPath, ["scripts/ssh-computer-use-smoke.mjs"], { + cwd: repoRoot, + env: { + ...process.env, + ...env, + }, + stdio: ["pipe", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("close", (status) => { + resolve({ status, stdout, stderr }); + }); + child.stdin.end(JSON.stringify(payload)); + }); +} + +test("ssh computer use runtime returns a dry-run desktop control summary", async () => { + const result = await runRuntime( + { + requestKind: "desktop_control", + requestId: "ssh-dry-run-1", + objective: "打开 Chrome 输入“Boss 远程控制测试”", + }, + { + BOSS_SSH_CONTROL_HOST: "192.168.31.114", + BOSS_SSH_CONTROL_USER: "jas", + BOSS_SSH_CONTROL_DRY_RUN: "true", + }, + ); + + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout); + assert.equal(payload.status, "completed"); + assert.equal(payload.requestId, "ssh-dry-run-1"); + assert.equal(payload.targetApp, "Chrome"); + assert.equal(payload.typedText, "Boss 远程控制测试"); + assert.match(payload.replyBody, /SSH 桌面控制已完成/); +}); + +test("ssh computer use runtime fails closed when host is missing", async () => { + const result = await runRuntime({ + requestKind: "desktop_control", + requestId: "ssh-missing-host", + objective: "打开 Finder", + }); + + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout); + assert.equal(payload.status, "failed"); + assert.equal(payload.requestId, "ssh-missing-host"); + assert.equal(payload.error, "SSH_CONTROL_HOST_REQUIRED"); +}); diff --git a/tsconfig.json b/tsconfig.json index 89e7813..004cbda 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,5 +30,5 @@ ".next/dev/types/**/*.ts", "**/*.mts" ], - "exclude": ["node_modules", "apps/boss-admin-web/**"] + "exclude": ["node_modules", "apps/boss-admin-web/**", "outputs/**"] }