diff --git a/collector-service/.dockerignore b/collector-service/.dockerignore new file mode 100644 index 0000000..e1ecd53 --- /dev/null +++ b/collector-service/.dockerignore @@ -0,0 +1,3 @@ +.venv311 +app/__pycache__ +*.pyc diff --git a/collector-service/Dockerfile b/collector-service/Dockerfile index 55a0d14..62deabe 100644 --- a/collector-service/Dockerfile +++ b/collector-service/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.11-slim +ARG BASE_IMAGE=python:3.11-slim +FROM ${BASE_IMAGE} RUN apt-get update \ && apt-get install -y --no-install-recommends ffmpeg \ diff --git a/deploy/storyforge-fnos-collector.compose.yaml b/deploy/storyforge-fnos-collector.compose.yaml new file mode 100644 index 0000000..2f70b6c --- /dev/null +++ b/deploy/storyforge-fnos-collector.compose.yaml @@ -0,0 +1,51 @@ +services: + storyforge-collector-dev: + image: ${STORYFORGE_COLLECTOR_IMAGE:-storyforge-collector-dev:fnos} + build: + context: ../../storyforge/collector-service + args: + BASE_IMAGE: ${STORYFORGE_COLLECTOR_BASE_IMAGE:-docker.m.daocloud.io/library/python:3.11-slim} + container_name: storyforge-collector-dev + restart: unless-stopped + ports: + - "${STORYFORGE_COLLECTOR_DEV_PORT:-19193}:8081" + environment: + DATA_DIR: /data/collector + DATABASE_PATH: /data/collector/storyforge.db + DEFAULT_EXTERNAL_BASE_URL: ${DEFAULT_EXTERNAL_BASE_URL:-http://192.168.31.188:19193} + LOCAL_OPENAI_BASE_URL: ${LOCAL_OPENAI_BASE_URL:-} + LOCAL_OPENAI_MODEL: ${LOCAL_OPENAI_MODEL:-GLM-5} + LOCAL_OPENAI_API_KEY: ${LOCAL_OPENAI_API_KEY:-} + N8N_BASE_URL: ${N8N_BASE_URL:-} + N8N_ANALYSIS_WEBHOOK_PATH: ${N8N_ANALYSIS_WEBHOOK_PATH:-/webhook/storyforge-analysis} + N8N_REAL_CUT_WEBHOOK_PATH: ${N8N_REAL_CUT_WEBHOOK_PATH:-/webhook/storyforge-real-cut} + N8N_AI_VIDEO_WEBHOOK_PATH: ${N8N_AI_VIDEO_WEBHOOK_PATH:-/webhook/storyforge-ai-video} + N8N_CONTENT_SOURCE_SYNC_WEBHOOK_PATH: ${N8N_CONTENT_SOURCE_SYNC_WEBHOOK_PATH:-/webhook/storyforge-content-source-sync} + BOOTSTRAP_SUPERADMIN_USERNAME: ${BOOTSTRAP_SUPERADMIN_USERNAME:-} + BOOTSTRAP_SUPERADMIN_PASSWORD: ${BOOTSTRAP_SUPERADMIN_PASSWORD:-} + BOOTSTRAP_SUPERADMIN_DISPLAY_NAME: ${BOOTSTRAP_SUPERADMIN_DISPLAY_NAME:-StoryForge Admin} + WEB_AUTOLOGIN_ENABLED: ${WEB_AUTOLOGIN_ENABLED:-1} + WEB_AUTOLOGIN_ACCOUNT_USERNAME: ${WEB_AUTOLOGIN_ACCOUNT_USERNAME:-kris} + WEB_AUTOLOGIN_USERNAME: ${WEB_AUTOLOGIN_USERNAME:-} + WEB_AUTOLOGIN_PASSWORD: ${WEB_AUTOLOGIN_PASSWORD:-} + ORCHESTRATOR_SHARED_SECRET: ${ORCHESTRATOR_SHARED_SECRET:-storyforge-fnos-dev-secret} + CUTVIDEO_BASE_URL: ${CUTVIDEO_BASE_URL:-http://192.168.31.18:7860} + CUTVIDEO_API_KEY: ${CUTVIDEO_API_KEY:-} + CUTVIDEO_BASE_CONFIG: ${CUTVIDEO_BASE_CONFIG:-example.job.yaml} + CUTVIDEO_POLL_INTERVAL_SEC: ${CUTVIDEO_POLL_INTERVAL_SEC:-10} + CUTVIDEO_MAX_WAIT_SEC: ${CUTVIDEO_MAX_WAIT_SEC:-1800} + CUTVIDEO_UPLOAD_TIMEOUT_SEC: ${CUTVIDEO_UPLOAD_TIMEOUT_SEC:-1800} + HUOBAO_BASE_URL: ${HUOBAO_BASE_URL:-} + YTDLP_BIN: ${YTDLP_BIN:-yt-dlp} + FFMPEG_BIN: ${FFMPEG_BIN:-ffmpeg} + WHISPER_BIN: ${WHISPER_BIN:-} + WHISPER_MODEL: ${WHISPER_MODEL:-/data/collector/models/ggml-base.en.bin} + ASR_HTTP_BASE_URL: ${ASR_HTTP_BASE_URL:-} + ASR_HTTP_TRANSCRIBE_PATH: ${ASR_HTTP_TRANSCRIBE_PATH:-/transcribe} + ASR_HTTP_FIELD_NAME: ${ASR_HTTP_FIELD_NAME:-wav} + ASR_HTTP_TIMEOUT_SEC: ${ASR_HTTP_TIMEOUT_SEC:-120} + HUOBAO_POLL_INTERVAL_SEC: ${HUOBAO_POLL_INTERVAL_SEC:-10} + HUOBAO_MAX_WAIT_SEC: ${HUOBAO_MAX_WAIT_SEC:-900} + LIVE_RECORDER_BASE_URL: ${LIVE_RECORDER_BASE_URL:-http://192.168.31.188:19106} + volumes: + - ../../storyforge/data/collector:/data/collector diff --git a/scripts/deploy_fnos_storyforge_collector.sh b/scripts/deploy_fnos_storyforge_collector.sh new file mode 100755 index 0000000..669372b --- /dev/null +++ b/scripts/deploy_fnos_storyforge_collector.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" + +export CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" +export FNOS_SKILL="${FNOS_SKILL:-$CODEX_HOME/skills/fnos-hyzq-deploy}" +export FNOS_SSH="${FNOS_SSH:-$FNOS_SKILL/scripts/fnos_ssh.sh}" +export FNOS_SCP="${FNOS_SCP:-$FNOS_SKILL/scripts/fnos_scp.sh}" + +export AG_SERVER_SKILL="${AG_SERVER_SKILL:-$CODEX_HOME/skills/ai-glasses-server-debug}" +export AG_SERVER="${AG_SERVER:-$AG_SERVER_SKILL/scripts/server_ssh.sh}" + +FNOS_HOST="${FNOS_HOST:-192.168.31.188}" +FNOS_USER="${FNOS_USER:-krisolo}" +AG_HOST="${AG_HOST:-111.231.132.51}" +AG_USER="${AG_USER:-ubuntu}" + +REMOTE_ROOT="${STORYFORGE_FNOS_REMOTE_ROOT:-/vol1/docker/hyzq-stack/current/storyforge}" +REMOTE_COLLECTOR_PARENT="$REMOTE_ROOT" +REMOTE_COLLECTOR_DIR="$REMOTE_ROOT/collector-service" +REMOTE_DATA_DIR="$REMOTE_ROOT/data/collector" +REMOTE_COMPOSE_DIR="${STORYFORGE_FNOS_COMPOSE_DIR:-/vol1/docker/hyzq-stack/current/deploy/fnos}" +REMOTE_COMPOSE_FILE="$REMOTE_COMPOSE_DIR/storyforge-fnos-collector.compose.yaml" +REMOTE_WEB_ASSETS_DIR="$REMOTE_ROOT/web/storyforge-web-v4/assets" +REMOTE_IMAGE_DIR="$REMOTE_ROOT/images" +REMOTE_IMAGE_ARCHIVE="$REMOTE_IMAGE_DIR/storyforge-collector-dev-fnos.tar.gz" + +COLLECTOR_PORT="${STORYFORGE_COLLECTOR_DEV_PORT:-19193}" +FRONTEND_PORT="${STORYFORGE_WEB_V4_DEV_PORT:-19192}" +BACKEND_URL="${STORYFORGE_FNOS_COLLECTOR_URL:-http://$FNOS_HOST:$COLLECTOR_PORT}" +DEPLOY_MODE="${STORYFORGE_FNOS_COLLECTOR_DEPLOY_MODE:-prebuilt_local}" +COLLECTOR_IMAGE="${STORYFORGE_COLLECTOR_IMAGE:-storyforge-collector-dev:fnos}" +COLLECTOR_LOCAL_BASE_IMAGE="${STORYFORGE_COLLECTOR_LOCAL_BASE_IMAGE:-python:3.11-slim}" +COLLECTOR_BASE_IMAGE="${STORYFORGE_COLLECTOR_BASE_IMAGE:-docker.m.daocloud.io/library/python:3.11-slim}" +WEB_AUTOLOGIN_ACCOUNT_USERNAME="${WEB_AUTOLOGIN_ACCOUNT_USERNAME:-kris}" +ORCHESTRATOR_SHARED_SECRET="${ORCHESTRATOR_SHARED_SECRET:-storyforge-fnos-dev-secret}" +LOCAL_OPENAI_BASE_URL="${LOCAL_OPENAI_BASE_URL:-}" +LOCAL_OPENAI_MODEL="${LOCAL_OPENAI_MODEL:-GLM-5}" +LOCAL_OPENAI_API_KEY="${LOCAL_OPENAI_API_KEY:-}" +N8N_BASE_URL="${N8N_BASE_URL:-}" +ASR_HTTP_BASE_URL="${ASR_HTTP_BASE_URL:-}" +HUOBAO_BASE_URL="${HUOBAO_BASE_URL:-}" +CUTVIDEO_BASE_URL="${CUTVIDEO_BASE_URL:-http://192.168.31.18:7860}" +LIVE_RECORDER_BASE_URL="${LIVE_RECORDER_BASE_URL:-http://192.168.31.188:19106}" +CLOUD_DB_PATH="${CLOUD_DB_PATH:-/home/ubuntu/storyforge/data/collector/storyforge.db}" +CLOUD_DB_SNAPSHOT_PATH="${CLOUD_DB_SNAPSHOT_PATH:-/home/ubuntu/storyforge/data/collector/storyforge-fnos-sync.db}" + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required command: $1" >&2 + exit 1 + fi +} + +need_cmd python3 +need_cmd security +need_cmd rsync +need_cmd docker + +shell_quote() { + python3 - "$1" <<'PY' +import shlex +import sys +print(shlex.quote(sys.argv[1])) +PY +} + +json_quote() { + python3 - "$1" <<'PY' +import json +import sys +print(json.dumps(sys.argv[1], ensure_ascii=False)) +PY +} + +resolve_fnos_password() { + if [ -n "${FNOS_PASSWORD:-}" ]; then + printf '%s' "$FNOS_PASSWORD" + return 0 + fi + security find-internet-password -s "$FNOS_HOST" -a "$FNOS_USER" -w +} + +resolve_ag_password() { + if [ -n "${AG_SERVER_PASSWORD:-}" ]; then + printf '%s' "$AG_SERVER_PASSWORD" + return 0 + fi + security find-generic-password -a "$AG_USER" -s ai-glasses-debug-ssh -w +} + +copy_cloud_file_to_local() { + local remote_path="$1" + local local_path="$2" + local password="$3" + AG_HOST="$AG_HOST" AG_USER="$AG_USER" AG_PASSWORD="$password" CLOUD_REMOTE_PATH="$remote_path" CLOUD_LOCAL_PATH="$local_path" /usr/bin/expect <<'EOF' +set timeout -1 + +set host $env(AG_HOST) +set user $env(AG_USER) +set pw $env(AG_PASSWORD) +set remote_path $env(CLOUD_REMOTE_PATH) +set local_path $env(CLOUD_LOCAL_PATH) + +set argv [list scp -P 22 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$user@$host:$remote_path" $local_path] +spawn {*}$argv + +expect { + -re {Are you sure you want to continue connecting \(yes/no(/\[fingerprint\])?\)\?} { + send "yes\r" + exp_continue + } + -re {[Pp]assword:} { + send "$pw\r" + exp_continue + } + eof +} + +catch wait result +set exit_code [lindex $result 3] +exit $exit_code +EOF +} + +run_fnos_sudo() { + local password="$1" + shift + local remote_cmd="$*" + local password_quoted + password_quoted="$(shell_quote "$password")" + "$FNOS_SSH" "$(shell_quote "printf '%s\\n' $password_quoted | sudo -S -p '' sh -lc $(shell_quote "$remote_cmd")")" +} + +FNOS_PASSWORD_VALUE="$(resolve_fnos_password)" +AG_PASSWORD_VALUE="$(resolve_ag_password)" +TMPDIR_DEPLOY="$(mktemp -d)" +COLLECTOR_SYNC_DIR="$TMPDIR_DEPLOY/collector-service" +LOCAL_IMAGE_ARCHIVE="$TMPDIR_DEPLOY/storyforge-collector-dev-fnos.tar.gz" +DB_SNAPSHOT_LOCAL="$TMPDIR_DEPLOY/storyforge.db" +RUNTIME_CONFIG_FILE="$TMPDIR_DEPLOY/storyforge-runtime-config.js" +BACKEND_URL_JSON="$(json_quote "$BACKEND_URL")" +CLOUD_DB_PATH_JSON="$(json_quote "$CLOUD_DB_PATH")" +CLOUD_DB_SNAPSHOT_PATH_JSON="$(json_quote "$CLOUD_DB_SNAPSHOT_PATH")" +trap 'rm -rf "$TMPDIR_DEPLOY"' EXIT + +mkdir -p "$COLLECTOR_SYNC_DIR/app" +cp "$ROOT/collector-service/Dockerfile" "$COLLECTOR_SYNC_DIR/Dockerfile" +cp "$ROOT/collector-service/requirements.txt" "$COLLECTOR_SYNC_DIR/requirements.txt" +rsync -a --delete \ + --exclude '__pycache__' \ + --exclude '*.pyc' \ + "$ROOT/collector-service/app/" "$COLLECTOR_SYNC_DIR/app/" + +cat >"$RUNTIME_CONFIG_FILE" < building local amd64 image" + DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build \ + --platform linux/amd64 \ + --build-arg BASE_IMAGE="$COLLECTOR_LOCAL_BASE_IMAGE" \ + -t "$COLLECTOR_IMAGE" \ + "$ROOT/collector-service" + echo " -> exporting image archive" + docker save "$COLLECTOR_IMAGE" | gzip >"$LOCAL_IMAGE_ARCHIVE" + echo " -> uploading image archive to fnOS" + "$FNOS_SCP" "$REMOTE_IMAGE_DIR" "$LOCAL_IMAGE_ARCHIVE" + run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \ + gunzip -c $(shell_quote "$REMOTE_IMAGE_ARCHIVE") | docker load && \ + STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \ + STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \ + DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \ + WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \ + ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \ + LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \ + LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \ + LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \ + N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \ + ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \ + HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \ + CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \ + LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \ + docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --no-build --force-recreate storyforge-collector-dev && \ + STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps" +else + run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \ + STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \ + STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \ + STORYFORGE_COLLECTOR_BASE_IMAGE=$(shell_quote "$COLLECTOR_BASE_IMAGE") \ + DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \ + WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \ + ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \ + LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \ + LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \ + LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \ + N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \ + ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \ + HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \ + CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \ + LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \ + docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --build --force-recreate storyforge-collector-dev && \ + STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps" +fi + +echo "[7/7] verify lan collector and web binding" +for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do + if curl -fsS "$BACKEND_URL/healthz" >/dev/null; then + break + fi + sleep 3 +done +curl -fsS "$BACKEND_URL/healthz" >/dev/null +curl -fsS -X POST "$BACKEND_URL/v2/auth/auto-session" -H 'content-type: application/json' -d '{}' >/dev/null +curl -fsS "http://$FNOS_HOST:$FRONTEND_PORT/assets/storyforge-runtime-config.js" | grep -q "$BACKEND_URL" + +echo "fnOS StoryForge collector ready: $BACKEND_URL" diff --git a/web/storyforge-web-v4/README.md b/web/storyforge-web-v4/README.md index 1516b55..bb47533 100644 --- a/web/storyforge-web-v4/README.md +++ b/web/storyforge-web-v4/README.md @@ -134,6 +134,18 @@ cd /Users/kris/code/StoryForge-gitea ./scripts/deploy_fnos_storyforge_web.sh ``` +如果希望前后端都在局域网内联调,也可以把 collector 一起部署到飞牛 NAS: + +```bash +cd /Users/kris/code/StoryForge-gitea +./scripts/deploy_fnos_storyforge_collector.sh +``` + +当前默认局域网端口: + +- 前端:`http://192.168.31.188:19192/` +- 后端:`http://192.168.31.188:19193/` + ## 后续建议 - 继续补多平台各自更深的专属采集与解析能力,而不只是一套统一抽象层 diff --git a/web/storyforge-web-v4/assets/app.js b/web/storyforge-web-v4/assets/app.js index 97a2d26..5e4ed8a 100644 --- a/web/storyforge-web-v4/assets/app.js +++ b/web/storyforge-web-v4/assets/app.js @@ -3253,7 +3253,7 @@ function renderDouyinInsightPanel() {

分析报告

-

分析报告来自 `/analysis-reports`,可直接对照结论和建议。

+

分析报告来自 /analysis-reports,可直接对照结论和建议。

${analysisReports.map((report) => { const suggestion = safeArray(report.suggestions)[0] || null;