feat: add fnos collector lan deploy
This commit is contained in:
kris
256
scripts/deploy_fnos_storyforge_collector.sh
Executable file
256
scripts/deploy_fnos_storyforge_collector.sh
Executable file
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
export CODEX_HOME="${CODEX_HOME:-$HOME/.codex}"
|
||||
export FNOS_SKILL="${FNOS_SKILL:-$CODEX_HOME/skills/fnos-hyzq-deploy}"
|
||||
export FNOS_SSH="${FNOS_SSH:-$FNOS_SKILL/scripts/fnos_ssh.sh}"
|
||||
export FNOS_SCP="${FNOS_SCP:-$FNOS_SKILL/scripts/fnos_scp.sh}"
|
||||
|
||||
export AG_SERVER_SKILL="${AG_SERVER_SKILL:-$CODEX_HOME/skills/ai-glasses-server-debug}"
|
||||
export AG_SERVER="${AG_SERVER:-$AG_SERVER_SKILL/scripts/server_ssh.sh}"
|
||||
|
||||
FNOS_HOST="${FNOS_HOST:-192.168.31.188}"
|
||||
FNOS_USER="${FNOS_USER:-krisolo}"
|
||||
AG_HOST="${AG_HOST:-111.231.132.51}"
|
||||
AG_USER="${AG_USER:-ubuntu}"
|
||||
|
||||
REMOTE_ROOT="${STORYFORGE_FNOS_REMOTE_ROOT:-/vol1/docker/hyzq-stack/current/storyforge}"
|
||||
REMOTE_COLLECTOR_PARENT="$REMOTE_ROOT"
|
||||
REMOTE_COLLECTOR_DIR="$REMOTE_ROOT/collector-service"
|
||||
REMOTE_DATA_DIR="$REMOTE_ROOT/data/collector"
|
||||
REMOTE_COMPOSE_DIR="${STORYFORGE_FNOS_COMPOSE_DIR:-/vol1/docker/hyzq-stack/current/deploy/fnos}"
|
||||
REMOTE_COMPOSE_FILE="$REMOTE_COMPOSE_DIR/storyforge-fnos-collector.compose.yaml"
|
||||
REMOTE_WEB_ASSETS_DIR="$REMOTE_ROOT/web/storyforge-web-v4/assets"
|
||||
REMOTE_IMAGE_DIR="$REMOTE_ROOT/images"
|
||||
REMOTE_IMAGE_ARCHIVE="$REMOTE_IMAGE_DIR/storyforge-collector-dev-fnos.tar.gz"
|
||||
|
||||
COLLECTOR_PORT="${STORYFORGE_COLLECTOR_DEV_PORT:-19193}"
|
||||
FRONTEND_PORT="${STORYFORGE_WEB_V4_DEV_PORT:-19192}"
|
||||
BACKEND_URL="${STORYFORGE_FNOS_COLLECTOR_URL:-http://$FNOS_HOST:$COLLECTOR_PORT}"
|
||||
DEPLOY_MODE="${STORYFORGE_FNOS_COLLECTOR_DEPLOY_MODE:-prebuilt_local}"
|
||||
COLLECTOR_IMAGE="${STORYFORGE_COLLECTOR_IMAGE:-storyforge-collector-dev:fnos}"
|
||||
COLLECTOR_LOCAL_BASE_IMAGE="${STORYFORGE_COLLECTOR_LOCAL_BASE_IMAGE:-python:3.11-slim}"
|
||||
COLLECTOR_BASE_IMAGE="${STORYFORGE_COLLECTOR_BASE_IMAGE:-docker.m.daocloud.io/library/python:3.11-slim}"
|
||||
WEB_AUTOLOGIN_ACCOUNT_USERNAME="${WEB_AUTOLOGIN_ACCOUNT_USERNAME:-kris}"
|
||||
ORCHESTRATOR_SHARED_SECRET="${ORCHESTRATOR_SHARED_SECRET:-storyforge-fnos-dev-secret}"
|
||||
LOCAL_OPENAI_BASE_URL="${LOCAL_OPENAI_BASE_URL:-}"
|
||||
LOCAL_OPENAI_MODEL="${LOCAL_OPENAI_MODEL:-GLM-5}"
|
||||
LOCAL_OPENAI_API_KEY="${LOCAL_OPENAI_API_KEY:-}"
|
||||
N8N_BASE_URL="${N8N_BASE_URL:-}"
|
||||
ASR_HTTP_BASE_URL="${ASR_HTTP_BASE_URL:-}"
|
||||
HUOBAO_BASE_URL="${HUOBAO_BASE_URL:-}"
|
||||
CUTVIDEO_BASE_URL="${CUTVIDEO_BASE_URL:-http://192.168.31.18:7860}"
|
||||
LIVE_RECORDER_BASE_URL="${LIVE_RECORDER_BASE_URL:-http://192.168.31.188:19106}"
|
||||
CLOUD_DB_PATH="${CLOUD_DB_PATH:-/home/ubuntu/storyforge/data/collector/storyforge.db}"
|
||||
CLOUD_DB_SNAPSHOT_PATH="${CLOUD_DB_SNAPSHOT_PATH:-/home/ubuntu/storyforge/data/collector/storyforge-fnos-sync.db}"
|
||||
|
||||
need_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "missing required command: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
need_cmd python3
|
||||
need_cmd security
|
||||
need_cmd rsync
|
||||
need_cmd docker
|
||||
|
||||
shell_quote() {
|
||||
python3 - "$1" <<'PY'
|
||||
import shlex
|
||||
import sys
|
||||
print(shlex.quote(sys.argv[1]))
|
||||
PY
|
||||
}
|
||||
|
||||
json_quote() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
print(json.dumps(sys.argv[1], ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
resolve_fnos_password() {
|
||||
if [ -n "${FNOS_PASSWORD:-}" ]; then
|
||||
printf '%s' "$FNOS_PASSWORD"
|
||||
return 0
|
||||
fi
|
||||
security find-internet-password -s "$FNOS_HOST" -a "$FNOS_USER" -w
|
||||
}
|
||||
|
||||
resolve_ag_password() {
|
||||
if [ -n "${AG_SERVER_PASSWORD:-}" ]; then
|
||||
printf '%s' "$AG_SERVER_PASSWORD"
|
||||
return 0
|
||||
fi
|
||||
security find-generic-password -a "$AG_USER" -s ai-glasses-debug-ssh -w
|
||||
}
|
||||
|
||||
copy_cloud_file_to_local() {
|
||||
local remote_path="$1"
|
||||
local local_path="$2"
|
||||
local password="$3"
|
||||
AG_HOST="$AG_HOST" AG_USER="$AG_USER" AG_PASSWORD="$password" CLOUD_REMOTE_PATH="$remote_path" CLOUD_LOCAL_PATH="$local_path" /usr/bin/expect <<'EOF'
|
||||
set timeout -1
|
||||
|
||||
set host $env(AG_HOST)
|
||||
set user $env(AG_USER)
|
||||
set pw $env(AG_PASSWORD)
|
||||
set remote_path $env(CLOUD_REMOTE_PATH)
|
||||
set local_path $env(CLOUD_LOCAL_PATH)
|
||||
|
||||
set argv [list scp -P 22 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$user@$host:$remote_path" $local_path]
|
||||
spawn {*}$argv
|
||||
|
||||
expect {
|
||||
-re {Are you sure you want to continue connecting \(yes/no(/\[fingerprint\])?\)\?} {
|
||||
send "yes\r"
|
||||
exp_continue
|
||||
}
|
||||
-re {[Pp]assword:} {
|
||||
send "$pw\r"
|
||||
exp_continue
|
||||
}
|
||||
eof
|
||||
}
|
||||
|
||||
catch wait result
|
||||
set exit_code [lindex $result 3]
|
||||
exit $exit_code
|
||||
EOF
|
||||
}
|
||||
|
||||
run_fnos_sudo() {
|
||||
local password="$1"
|
||||
shift
|
||||
local remote_cmd="$*"
|
||||
local password_quoted
|
||||
password_quoted="$(shell_quote "$password")"
|
||||
"$FNOS_SSH" "$(shell_quote "printf '%s\\n' $password_quoted | sudo -S -p '' sh -lc $(shell_quote "$remote_cmd")")"
|
||||
}
|
||||
|
||||
FNOS_PASSWORD_VALUE="$(resolve_fnos_password)"
|
||||
AG_PASSWORD_VALUE="$(resolve_ag_password)"
|
||||
TMPDIR_DEPLOY="$(mktemp -d)"
|
||||
COLLECTOR_SYNC_DIR="$TMPDIR_DEPLOY/collector-service"
|
||||
LOCAL_IMAGE_ARCHIVE="$TMPDIR_DEPLOY/storyforge-collector-dev-fnos.tar.gz"
|
||||
DB_SNAPSHOT_LOCAL="$TMPDIR_DEPLOY/storyforge.db"
|
||||
RUNTIME_CONFIG_FILE="$TMPDIR_DEPLOY/storyforge-runtime-config.js"
|
||||
BACKEND_URL_JSON="$(json_quote "$BACKEND_URL")"
|
||||
CLOUD_DB_PATH_JSON="$(json_quote "$CLOUD_DB_PATH")"
|
||||
CLOUD_DB_SNAPSHOT_PATH_JSON="$(json_quote "$CLOUD_DB_SNAPSHOT_PATH")"
|
||||
trap 'rm -rf "$TMPDIR_DEPLOY"' EXIT
|
||||
|
||||
mkdir -p "$COLLECTOR_SYNC_DIR/app"
|
||||
cp "$ROOT/collector-service/Dockerfile" "$COLLECTOR_SYNC_DIR/Dockerfile"
|
||||
cp "$ROOT/collector-service/requirements.txt" "$COLLECTOR_SYNC_DIR/requirements.txt"
|
||||
rsync -a --delete \
|
||||
--exclude '__pycache__' \
|
||||
--exclude '*.pyc' \
|
||||
"$ROOT/collector-service/app/" "$COLLECTOR_SYNC_DIR/app/"
|
||||
|
||||
cat >"$RUNTIME_CONFIG_FILE" <<EOF
|
||||
(function () {
|
||||
window.__STORYFORGE_RUNTIME_CONFIG__ = Object.assign(
|
||||
{},
|
||||
window.__STORYFORGE_RUNTIME_CONFIG__ || {},
|
||||
{
|
||||
backendUrl: $BACKEND_URL_JSON
|
||||
}
|
||||
);
|
||||
})();
|
||||
EOF
|
||||
|
||||
echo "[1/7] snapshot cloud sqlite"
|
||||
"$AG_SERVER" exec "python3 - <<'PY'
|
||||
import sqlite3
|
||||
src = $CLOUD_DB_PATH_JSON
|
||||
dst = $CLOUD_DB_SNAPSHOT_PATH_JSON
|
||||
src_conn = sqlite3.connect(src)
|
||||
dst_conn = sqlite3.connect(dst)
|
||||
with dst_conn:
|
||||
src_conn.backup(dst_conn)
|
||||
dst_conn.close()
|
||||
src_conn.close()
|
||||
print(dst)
|
||||
PY"
|
||||
|
||||
echo "[2/7] download sqlite snapshot"
|
||||
copy_cloud_file_to_local "$CLOUD_DB_SNAPSHOT_PATH" "$DB_SNAPSHOT_LOCAL" "$AG_PASSWORD_VALUE"
|
||||
|
||||
echo "[3/7] prepare fnOS directories"
|
||||
"$FNOS_SSH" "$(shell_quote "rm -rf $(shell_quote "$REMOTE_COLLECTOR_DIR") && mkdir -p $(shell_quote "$REMOTE_COLLECTOR_PARENT") $(shell_quote "$REMOTE_DATA_DIR") $(shell_quote "$REMOTE_COMPOSE_DIR") $(shell_quote "$REMOTE_WEB_ASSETS_DIR") $(shell_quote "$REMOTE_IMAGE_DIR")")"
|
||||
|
||||
echo "[4/7] sync collector source"
|
||||
"$FNOS_SCP" "$REMOTE_COLLECTOR_PARENT" "$COLLECTOR_SYNC_DIR"
|
||||
|
||||
echo "[5/7] sync compose and sqlite"
|
||||
"$FNOS_SCP" "$REMOTE_COMPOSE_DIR" "$ROOT/deploy/storyforge-fnos-collector.compose.yaml"
|
||||
"$FNOS_SCP" "$REMOTE_DATA_DIR" "$DB_SNAPSHOT_LOCAL"
|
||||
"$FNOS_SCP" "$REMOTE_WEB_ASSETS_DIR" "$RUNTIME_CONFIG_FILE"
|
||||
|
||||
echo "[6/7] build and restart collector container"
|
||||
if [ "$DEPLOY_MODE" = "prebuilt_local" ]; then
|
||||
echo " -> building local amd64 image"
|
||||
DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build \
|
||||
--platform linux/amd64 \
|
||||
--build-arg BASE_IMAGE="$COLLECTOR_LOCAL_BASE_IMAGE" \
|
||||
-t "$COLLECTOR_IMAGE" \
|
||||
"$ROOT/collector-service"
|
||||
echo " -> exporting image archive"
|
||||
docker save "$COLLECTOR_IMAGE" | gzip >"$LOCAL_IMAGE_ARCHIVE"
|
||||
echo " -> uploading image archive to fnOS"
|
||||
"$FNOS_SCP" "$REMOTE_IMAGE_DIR" "$LOCAL_IMAGE_ARCHIVE"
|
||||
run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \
|
||||
gunzip -c $(shell_quote "$REMOTE_IMAGE_ARCHIVE") | docker load && \
|
||||
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \
|
||||
STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \
|
||||
DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \
|
||||
WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \
|
||||
ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \
|
||||
LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \
|
||||
LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \
|
||||
LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \
|
||||
N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \
|
||||
ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \
|
||||
HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \
|
||||
CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \
|
||||
LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \
|
||||
docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --no-build --force-recreate storyforge-collector-dev && \
|
||||
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps"
|
||||
else
|
||||
run_fnos_sudo "$FNOS_PASSWORD_VALUE" "cd $(shell_quote "$REMOTE_COMPOSE_DIR") && \
|
||||
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") \
|
||||
STORYFORGE_COLLECTOR_IMAGE=$(shell_quote "$COLLECTOR_IMAGE") \
|
||||
STORYFORGE_COLLECTOR_BASE_IMAGE=$(shell_quote "$COLLECTOR_BASE_IMAGE") \
|
||||
DEFAULT_EXTERNAL_BASE_URL=$(shell_quote "$BACKEND_URL") \
|
||||
WEB_AUTOLOGIN_ACCOUNT_USERNAME=$(shell_quote "$WEB_AUTOLOGIN_ACCOUNT_USERNAME") \
|
||||
ORCHESTRATOR_SHARED_SECRET=$(shell_quote "$ORCHESTRATOR_SHARED_SECRET") \
|
||||
LOCAL_OPENAI_BASE_URL=$(shell_quote "$LOCAL_OPENAI_BASE_URL") \
|
||||
LOCAL_OPENAI_MODEL=$(shell_quote "$LOCAL_OPENAI_MODEL") \
|
||||
LOCAL_OPENAI_API_KEY=$(shell_quote "$LOCAL_OPENAI_API_KEY") \
|
||||
N8N_BASE_URL=$(shell_quote "$N8N_BASE_URL") \
|
||||
ASR_HTTP_BASE_URL=$(shell_quote "$ASR_HTTP_BASE_URL") \
|
||||
HUOBAO_BASE_URL=$(shell_quote "$HUOBAO_BASE_URL") \
|
||||
CUTVIDEO_BASE_URL=$(shell_quote "$CUTVIDEO_BASE_URL") \
|
||||
LIVE_RECORDER_BASE_URL=$(shell_quote "$LIVE_RECORDER_BASE_URL") \
|
||||
docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") up -d --build --force-recreate storyforge-collector-dev && \
|
||||
STORYFORGE_COLLECTOR_DEV_PORT=$(shell_quote "$COLLECTOR_PORT") docker compose -f $(shell_quote "$REMOTE_COMPOSE_FILE") ps"
|
||||
fi
|
||||
|
||||
echo "[7/7] verify lan collector and web binding"
|
||||
for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do
|
||||
if curl -fsS "$BACKEND_URL/healthz" >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
curl -fsS "$BACKEND_URL/healthz" >/dev/null
|
||||
curl -fsS -X POST "$BACKEND_URL/v2/auth/auto-session" -H 'content-type: application/json' -d '{}' >/dev/null
|
||||
curl -fsS "http://$FNOS_HOST:$FRONTEND_PORT/assets/storyforge-runtime-config.js" | grep -q "$BACKEND_URL"
|
||||
|
||||
echo "fnOS StoryForge collector ready: $BACKEND_URL"
|
||||
Reference in New Issue
Block a user