feat: add fnos collector lan deploy

This commit is contained in:
kris
2026-03-26 22:12:42 +08:00
parent d0673d08a5
commit 82560d1415
6 changed files with 325 additions and 2 deletions

View File

@@ -0,0 +1,3 @@
.venv311
app/__pycache__
*.pyc

View File

@@ -1,4 +1,5 @@
FROM python:3.11-slim
ARG BASE_IMAGE=python:3.11-slim
FROM ${BASE_IMAGE}
RUN apt-get update \
&& apt-get install -y --no-install-recommends ffmpeg \

View File

@@ -0,0 +1,51 @@
services:
storyforge-collector-dev:
image: ${STORYFORGE_COLLECTOR_IMAGE:-storyforge-collector-dev:fnos}
build:
context: ../../storyforge/collector-service
args:
BASE_IMAGE: ${STORYFORGE_COLLECTOR_BASE_IMAGE:-docker.m.daocloud.io/library/python:3.11-slim}
container_name: storyforge-collector-dev
restart: unless-stopped
ports:
- "${STORYFORGE_COLLECTOR_DEV_PORT:-19193}:8081"
environment:
DATA_DIR: /data/collector
DATABASE_PATH: /data/collector/storyforge.db
DEFAULT_EXTERNAL_BASE_URL: ${DEFAULT_EXTERNAL_BASE_URL:-http://192.168.31.188:19193}
LOCAL_OPENAI_BASE_URL: ${LOCAL_OPENAI_BASE_URL:-}
LOCAL_OPENAI_MODEL: ${LOCAL_OPENAI_MODEL:-GLM-5}
LOCAL_OPENAI_API_KEY: ${LOCAL_OPENAI_API_KEY:-}
N8N_BASE_URL: ${N8N_BASE_URL:-}
N8N_ANALYSIS_WEBHOOK_PATH: ${N8N_ANALYSIS_WEBHOOK_PATH:-/webhook/storyforge-analysis}
N8N_REAL_CUT_WEBHOOK_PATH: ${N8N_REAL_CUT_WEBHOOK_PATH:-/webhook/storyforge-real-cut}
N8N_AI_VIDEO_WEBHOOK_PATH: ${N8N_AI_VIDEO_WEBHOOK_PATH:-/webhook/storyforge-ai-video}
N8N_CONTENT_SOURCE_SYNC_WEBHOOK_PATH: ${N8N_CONTENT_SOURCE_SYNC_WEBHOOK_PATH:-/webhook/storyforge-content-source-sync}
BOOTSTRAP_SUPERADMIN_USERNAME: ${BOOTSTRAP_SUPERADMIN_USERNAME:-}
BOOTSTRAP_SUPERADMIN_PASSWORD: ${BOOTSTRAP_SUPERADMIN_PASSWORD:-}
BOOTSTRAP_SUPERADMIN_DISPLAY_NAME: ${BOOTSTRAP_SUPERADMIN_DISPLAY_NAME:-StoryForge Admin}
WEB_AUTOLOGIN_ENABLED: ${WEB_AUTOLOGIN_ENABLED:-1}
WEB_AUTOLOGIN_ACCOUNT_USERNAME: ${WEB_AUTOLOGIN_ACCOUNT_USERNAME:-kris}
WEB_AUTOLOGIN_USERNAME: ${WEB_AUTOLOGIN_USERNAME:-}
WEB_AUTOLOGIN_PASSWORD: ${WEB_AUTOLOGIN_PASSWORD:-}
ORCHESTRATOR_SHARED_SECRET: ${ORCHESTRATOR_SHARED_SECRET:-storyforge-fnos-dev-secret}
CUTVIDEO_BASE_URL: ${CUTVIDEO_BASE_URL:-http://192.168.31.18:7860}
CUTVIDEO_API_KEY: ${CUTVIDEO_API_KEY:-}
CUTVIDEO_BASE_CONFIG: ${CUTVIDEO_BASE_CONFIG:-example.job.yaml}
CUTVIDEO_POLL_INTERVAL_SEC: ${CUTVIDEO_POLL_INTERVAL_SEC:-10}
CUTVIDEO_MAX_WAIT_SEC: ${CUTVIDEO_MAX_WAIT_SEC:-1800}
CUTVIDEO_UPLOAD_TIMEOUT_SEC: ${CUTVIDEO_UPLOAD_TIMEOUT_SEC:-1800}
HUOBAO_BASE_URL: ${HUOBAO_BASE_URL:-}
YTDLP_BIN: ${YTDLP_BIN:-yt-dlp}
FFMPEG_BIN: ${FFMPEG_BIN:-ffmpeg}
WHISPER_BIN: ${WHISPER_BIN:-}
WHISPER_MODEL: ${WHISPER_MODEL:-/data/collector/models/ggml-base.en.bin}
ASR_HTTP_BASE_URL: ${ASR_HTTP_BASE_URL:-}
ASR_HTTP_TRANSCRIBE_PATH: ${ASR_HTTP_TRANSCRIBE_PATH:-/transcribe}
ASR_HTTP_FIELD_NAME: ${ASR_HTTP_FIELD_NAME:-wav}
ASR_HTTP_TIMEOUT_SEC: ${ASR_HTTP_TIMEOUT_SEC:-120}
HUOBAO_POLL_INTERVAL_SEC: ${HUOBAO_POLL_INTERVAL_SEC:-10}
HUOBAO_MAX_WAIT_SEC: ${HUOBAO_MAX_WAIT_SEC:-900}
LIVE_RECORDER_BASE_URL: ${LIVE_RECORDER_BASE_URL:-http://192.168.31.188:19106}
volumes:
- ../../storyforge/data/collector:/data/collector

View File

@@ -0,0 +1,256 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
export CODEX_HOME="${CODEX_HOME:-$HOME/.codex}"
export FNOS_SKILL="${FNOS_SKILL:-$CODEX_HOME/skills/fnos-hyzq-deploy}"
export FNOS_SSH="${FNOS_SSH:-$FNOS_SKILL/scripts/fnos_ssh.sh}"
export FNOS_SCP="${FNOS_SCP:-$FNOS_SKILL/scripts/fnos_scp.sh}"
export AG_SERVER_SKILL="${AG_SERVER_SKILL:-$CODEX_HOME/skills/ai-glasses-server-debug}"
export AG_SERVER="${AG_SERVER:-$AG_SERVER_SKILL/scripts/server_ssh.sh}"
FNOS_HOST="${FNOS_HOST:-192.168.31.188}"
FNOS_USER="${FNOS_USER:-krisolo}"
AG_HOST="${AG_HOST:-111.231.132.51}"
AG_USER="${AG_USER:-ubuntu}"
REMOTE_ROOT="${STORYFORGE_FNOS_REMOTE_ROOT:-/vol1/docker/hyzq-stack/current/storyforge}"
REMOTE_COLLECTOR_PARENT="$REMOTE_ROOT"
REMOTE_COLLECTOR_DIR="$REMOTE_ROOT/collector-service"
REMOTE_DATA_DIR="$REMOTE_ROOT/data/collector"
REMOTE_COMPOSE_DIR="${STORYFORGE_FNOS_COMPOSE_DIR:-/vol1/docker/hyzq-stack/current/deploy/fnos}"
REMOTE_COMPOSE_FILE="$REMOTE_COMPOSE_DIR/storyforge-fnos-collector.compose.yaml"
REMOTE_WEB_ASSETS_DIR="$REMOTE_ROOT/web/storyforge-web-v4/assets"
REMOTE_IMAGE_DIR="$REMOTE_ROOT/images"
REMOTE_IMAGE_ARCHIVE="$REMOTE_IMAGE_DIR/storyforge-collector-dev-fnos.tar.gz"
COLLECTOR_PORT="${STORYFORGE_COLLECTOR_DEV_PORT:-19193}"
FRONTEND_PORT="${STORYFORGE_WEB_V4_DEV_PORT:-19192}"
BACKEND_URL="${STORYFORGE_FNOS_COLLECTOR_URL:-http://$FNOS_HOST:$COLLECTOR_PORT}"
DEPLOY_MODE="${STORYFORGE_FNOS_COLLECTOR_DEPLOY_MODE:-prebuilt_local}"
COLLECTOR_IMAGE="${STORYFORGE_COLLECTOR_IMAGE:-storyforge-collector-dev:fnos}"
COLLECTOR_LOCAL_BASE_IMAGE="${STORYFORGE_COLLECTOR_LOCAL_BASE_IMAGE:-python:3.11-slim}"
COLLECTOR_BASE_IMAGE="${STORYFORGE_COLLECTOR_BASE_IMAGE:-docker.m.daocloud.io/library/python:3.11-slim}"
WEB_AUTOLOGIN_ACCOUNT_USERNAME="${WEB_AUTOLOGIN_ACCOUNT_USERNAME:-kris}"
ORCHESTRATOR_SHARED_SECRET="${ORCHESTRATOR_SHARED_SECRET:-storyforge-fnos-dev-secret}"
LOCAL_OPENAI_BASE_URL="${LOCAL_OPENAI_BASE_URL:-}"
LOCAL_OPENAI_MODEL="${LOCAL_OPENAI_MODEL:-GLM-5}"
LOCAL_OPENAI_API_KEY="${LOCAL_OPENAI_API_KEY:-}"
N8N_BASE_URL="${N8N_BASE_URL:-}"
ASR_HTTP_BASE_URL="${ASR_HTTP_BASE_URL:-}"
HUOBAO_BASE_URL="${HUOBAO_BASE_URL:-}"
CUTVIDEO_BASE_URL="${CUTVIDEO_BASE_URL:-http://192.168.31.18:7860}"
LIVE_RECORDER_BASE_URL="${LIVE_RECORDER_BASE_URL:-http://192.168.31.188:19106}"
CLOUD_DB_PATH="${CLOUD_DB_PATH:-/home/ubuntu/storyforge/data/collector/storyforge.db}"
CLOUD_DB_SNAPSHOT_PATH="${CLOUD_DB_SNAPSHOT_PATH:-/home/ubuntu/storyforge/data/collector/storyforge-fnos-sync.db}"
need_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "missing required command: $1" >&2
exit 1
fi
}
need_cmd python3
need_cmd security
need_cmd rsync
need_cmd docker
shell_quote() {
python3 - "$1" <<'PY'
import shlex
import sys
print(shlex.quote(sys.argv[1]))
PY
}
json_quote() {
python3 - "$1" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1], ensure_ascii=False))
PY
}
resolve_fnos_password() {
if [ -n "${FNOS_PASSWORD:-}" ]; then
printf '%s' "$FNOS_PASSWORD"
return 0
fi
security find-internet-password -s "$FNOS_HOST" -a "$FNOS_USER" -w
}
resolve_ag_password() {
if [ -n "${AG_SERVER_PASSWORD:-}" ]; then
printf '%s' "$AG_SERVER_PASSWORD"
return 0
fi
security find-generic-password -a "$AG_USER" -s ai-glasses-debug-ssh -w
}
copy_cloud_file_to_local() {
local remote_path="$1"
local local_path="$2"
local password="$3"
AG_HOST="$AG_HOST" AG_USER="$AG_USER" AG_PASSWORD="$password" CLOUD_REMOTE_PATH="$remote_path" CLOUD_LOCAL_PATH="$local_path" /usr/bin/expect <<'EOF'
set timeout -1
set host $env(AG_HOST)
set user $env(AG_USER)
set pw $env(AG_PASSWORD)
set remote_path $env(CLOUD_REMOTE_PATH)
set local_path $env(CLOUD_LOCAL_PATH)
set argv [list scp -P 22 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$user@$host:$remote_path" $local_path]
spawn {*}$argv
expect {
-re {Are you sure you want to continue connecting \(yes/no(/\[fingerprint\])?\)\?} {
send "yes\r"
exp_continue
}
-re {[Pp]assword:} {
send "$pw\r"
exp_continue
}
eof
}
catch wait result
set exit_code [lindex $result 3]
exit $exit_code
EOF
}
run_fnos_sudo() {
local password="$1"
shift
local remote_cmd="$*"
local password_quoted
password_quoted="$(shell_quote "$password")"
"$FNOS_SSH" "$(shell_quote "printf '%s\\n' $password_quoted | sudo -S -p '' sh -lc $(shell_quote "$remote_cmd")")"
}
FNOS_PASSWORD_VALUE="$(resolve_fnos_password)"
AG_PASSWORD_VALUE="$(resolve_ag_password)"
TMPDIR_DEPLOY="$(mktemp -d)"
COLLECTOR_SYNC_DIR="$TMPDIR_DEPLOY/collector-service"
LOCAL_IMAGE_ARCHIVE="$TMPDIR_DEPLOY/storyforge-collector-dev-fnos.tar.gz"
DB_SNAPSHOT_LOCAL="$TMPDIR_DEPLOY/storyforge.db"
RUNTIME_CONFIG_FILE="$TMPDIR_DEPLOY/storyforge-runtime-config.js"
BACKEND_URL_JSON="$(json_quote "$BACKEND_URL")"
CLOUD_DB_PATH_JSON="$(json_quote "$CLOUD_DB_PATH")"
CLOUD_DB_SNAPSHOT_PATH_JSON="$(json_quote "$CLOUD_DB_SNAPSHOT_PATH")"
trap 'rm -rf "$TMPDIR_DEPLOY"' EXIT
mkdir -p "$COLLECTOR_SYNC_DIR/app"
cp "$ROOT/collector-service/Dockerfile" "$COLLECTOR_SYNC_DIR/Dockerfile"
cp "$ROOT/collector-service/requirements.txt" "$COLLECTOR_SYNC_DIR/requirements.txt"
rsync -a --delete \
--exclude '__pycache__' \
--exclude '*.pyc' \
"$ROOT/collector-service/app/" "$COLLECTOR_SYNC_DIR/app/"
cat >"$RUNTIME_CONFIG_FILE" <<EOF
(function () {
window.__STORYFORGE_RUNTIME_CONFIG__ = Object.assign(
{},
window.__STORYFORGE_RUNTIME_CONFIG__ || {},
{
backendUrl: $BACKEND_URL_JSON
}
);
})();
EOF
echo "[1/7] snapshot cloud sqlite"
"$AG_SERVER" exec "python3 - <<'PY'
import sqlite3
src = $CLOUD_DB_PATH_JSON
dst = $CLOUD_DB_SNAPSHOT_PATH_JSON
src_conn = sqlite3.connect(src)
dst_conn = sqlite3.connect(dst)
with dst_conn:
src_conn.backup(dst_conn)
dst_conn.close()
src_conn.close()
print(dst)
PY"
echo "[2/7] download sqlite snapshot"
copy_cloud_file_to_local "$CLOUD_DB_SNAPSHOT_PATH" "$DB_SNAPSHOT_LOCAL" "$AG_PASSWORD_VALUE"
echo "[3/7] prepare fnOS directories"
"$FNOS_SSH" "$(shell_quote "rm -rf $(shell_quote "$REMOTE_COLLECTOR_DIR") && mkdir -p $(shell_quote "$REMOTE_COLLECTOR_PARENT") $(shell_quote "$REMOTE_DATA_DIR") $(shell_quote "$REMOTE_COMPOSE_DIR") $(shell_quote "$REMOTE_WEB_ASSETS_DIR") $(shell_quote "$REMOTE_IMAGE_DIR")")"
echo "[4/7] sync collector source"
"$FNOS_SCP" "$REMOTE_COLLECTOR_PARENT" "$COLLECTOR_SYNC_DIR"
echo "[5/7] sync compose and sqlite"
"$FNOS_SCP" "$REMOTE_COMPOSE_DIR" "$ROOT/deploy/storyforge-fnos-collector.compose.yaml"
"$FNOS_SCP" "$REMOTE_DATA_DIR" "$DB_SNAPSHOT_LOCAL"
"$FNOS_SCP" "$REMOTE_WEB_ASSETS_DIR" "$RUNTIME_CONFIG_FILE"
echo "[6/7] build and restart collector container"
if [ "$DEPLOY_MODE" = "prebuilt_local" ]; then
echo " -> building local amd64 image"
DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build \
--platform linux/amd64 \
--build-arg BASE_IMAGE="$COLLECTOR_LOCAL_BASE_IMAGE" \
-t "$COLLECTOR_IMAGE" \
"$ROOT/collector-service"
echo " -> exporting image archive"
docker save "$COLLECTOR_IMAGE" | gzip >"$LOCAL_IMAGE_ARCHIVE"
echo " -> uploading image archive to fnOS"
"$FNOS_SCP" "$REMOTE_IMAGE_DIR" "$LOCAL_IMAGE_ARCHIVE"
run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \
gunzip -c $(shell_quote "$REMOTE_IMAGE_ARCHIVE") | docker load && \
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \
STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \
DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \
WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \
ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \
LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \
LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \
LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \
N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \
ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \
HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \
CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \
LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \
docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --no-build --force-recreate storyforge-collector-dev && \
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps"
else
run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \
STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \
STORYFORGE_COLLECTOR_BASE_IMAGE=$(shell_quote "$COLLECTOR_BASE_IMAGE") \
DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \
WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \
ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \
LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \
LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \
LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \
N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \
ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \
HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \
CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \
LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \
docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --build --force-recreate storyforge-collector-dev && \
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps"
fi
echo "[7/7] verify lan collector and web binding"
for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do
if curl -fsS "$BACKEND_URL/healthz" >/dev/null; then
break
fi
sleep 3
done
curl -fsS "$BACKEND_URL/healthz" >/dev/null
curl -fsS -X POST "$BACKEND_URL/v2/auth/auto-session" -H 'content-type: application/json' -d '{}' >/dev/null
curl -fsS "http://$FNOS_HOST:$FRONTEND_PORT/assets/storyforge-runtime-config.js" | grep -q "$BACKEND_URL"
echo "fnOS StoryForge collector ready: $BACKEND_URL"

View File

@@ -134,6 +134,18 @@ cd /Users/kris/code/StoryForge-gitea
./scripts/deploy_fnos_storyforge_web.sh
```
如果希望前后端都在局域网内联调,也可以把 collector 一起部署到飞牛 NAS
```bash
cd /Users/kris/code/StoryForge-gitea
./scripts/deploy_fnos_storyforge_collector.sh
```
当前默认局域网端口:
- 前端:`http://192.168.31.188:19192/`
- 后端:`http://192.168.31.188:19193/`
## 后续建议
- 继续补多平台各自更深的专属采集与解析能力,而不只是一套统一抽象层

View File

@@ -3253,7 +3253,7 @@ function renderDouyinInsightPanel() {
</div>
<div class="task-item compact">
<h4>分析报告</h4>
<p>分析报告来自 `/analysis-reports`,可直接对照结论和建议。</p>
<p>分析报告来自 <code>/analysis-reports</code>,可直接对照结论和建议。</p>
<div class="list" style="margin-top:10px;">
${analysisReports.map((report) => {
const suggestion = safeArray(report.suggestions)[0] || null;