feat: finish storyforge workbench and runtime closure

2026-03-26 13:55:06 +08:00
parent 160cece196
commit 38b02a9799
16 changed files with 1530 additions and 2360 deletions
--- a/scripts/deploy_public_storyforge.sh
+++ b/scripts/deploy_public_storyforge.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
+
+HOST="${STORYFORGE_PUBLIC_HOST:-111.231.132.51}"
+USER_NAME="${STORYFORGE_PUBLIC_USER:-ubuntu}"
+PORT="${STORYFORGE_PUBLIC_PORT:-22}"
+BASE_URL="${STORYFORGE_PUBLIC_BASE_URL:-https://storyforge.hyzq.net}"
+REMOTE_BASE="${STORYFORGE_PUBLIC_REMOTE_BASE:-/home/ubuntu/storyforge}"
+KEYCHAIN_SERVICE="${STORYFORGE_PUBLIC_KEYCHAIN_SERVICE:-ai-glasses-debug-ssh}"
+SYNC_COLLECTOR="${STORYFORGE_PUBLIC_SYNC_COLLECTOR:-1}"
+CURL_MAX_TIME="${STORYFORGE_PUBLIC_CURL_MAX_TIME:-60}"
+
+need_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "missing required command: $1" >&2
+    exit 1
+  fi
+}
+
+need_cmd rsync
+need_cmd ssh
+need_cmd curl
+
+resolve_password() {
+  if [ -n "${STORYFORGE_PUBLIC_PASSWORD:-}" ]; then
+    printf '%s' "${STORYFORGE_PUBLIC_PASSWORD}"
+    return 0
+  fi
+  if [ -n "$KEYCHAIN_SERVICE" ] && command -v security >/dev/null 2>&1; then
+    security find-generic-password -a "$USER_NAME" -s "$KEYCHAIN_SERVICE" -w 2>/dev/null || true
+    return 0
+  fi
+  return 0
+}
+
+PASSWORD="$(resolve_password)"
+SSH_OPTS=(-p "$PORT" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null)
+RSYNC_RSH=(ssh "${SSH_OPTS[@]}")
+
+run_ssh() {
+  if [ -n "$PASSWORD" ]; then
+    need_cmd sshpass
+    SSHPASS="$PASSWORD" sshpass -e ssh "${SSH_OPTS[@]}" "$USER_NAME@$HOST" "$@"
+  else
+    ssh "${SSH_OPTS[@]}" "$USER_NAME@$HOST" "$@"
+  fi
+}
+
+run_rsync() {
+  if [ -n "$PASSWORD" ]; then
+    need_cmd sshpass
+    SSHPASS="$PASSWORD" sshpass -e rsync -az --delete -e "$(printf '%q ' "${RSYNC_RSH[@]}")" "$@"
+  else
+    rsync -az --delete -e "$(printf '%q ' "${RSYNC_RSH[@]}")" "$@"
+  fi
+}
+
+echo "[1/6] backup remote web"
+run_ssh "mkdir -p '$REMOTE_BASE/backups'; ts=\$(date +%Y%m%d-%H%M%S); tar -czf '$REMOTE_BASE/backups/storyforge-web-v4-'\$ts'.tgz' -C '$REMOTE_BASE/web' storyforge-web-v4 && echo web-backup:'$REMOTE_BASE/backups/storyforge-web-v4-'\$ts'.tgz'"
+
+if [ "$SYNC_COLLECTOR" = "1" ]; then
+  echo "[2/6] backup remote collector app"
+  run_ssh "mkdir -p '$REMOTE_BASE/backups'; ts=\$(date +%Y%m%d-%H%M%S); tar -czf '$REMOTE_BASE/backups/storyforge-collector-app-'\$ts'.tgz' -C '$REMOTE_BASE/collector-service' app && echo collector-backup:'$REMOTE_BASE/backups/storyforge-collector-app-'\$ts'.tgz'"
+else
+  echo "[2/6] skip collector backup"
+fi
+
+echo "[3/6] sync web/storyforge-web-v4"
+run_rsync "$ROOT/web/storyforge-web-v4/" "$USER_NAME@$HOST:$REMOTE_BASE/web/storyforge-web-v4/"
+
+if [ "$SYNC_COLLECTOR" = "1" ]; then
+  echo "[4/6] sync collector-service/app"
+  if [ -n "$PASSWORD" ]; then
+    need_cmd sshpass
+    SSHPASS="$PASSWORD" sshpass -e rsync -az --delete \
+      --exclude '__pycache__/' \
+      --exclude '*.pyc' \
+      -e "$(printf '%q ' "${RSYNC_RSH[@]}")" \
+      "$ROOT/collector-service/app/" \
+      "$USER_NAME@$HOST:$REMOTE_BASE/collector-service/app/"
+  else
+    rsync -az --delete \
+      --exclude '__pycache__/' \
+      --exclude '*.pyc' \
+      -e "$(printf '%q ' "${RSYNC_RSH[@]}")" \
+      "$ROOT/collector-service/app/" \
+      "$USER_NAME@$HOST:$REMOTE_BASE/collector-service/app/"
+  fi
+else
+  echo "[4/6] skip collector sync"
+fi
+
+echo "[5/6] restart remote services"
+if [ "$SYNC_COLLECTOR" = "1" ]; then
+  run_ssh "sudo systemctl restart storyforge-collector storyforge-web-v4 && sleep 2 && systemctl is-active storyforge-collector storyforge-web-v4"
+else
+  run_ssh "sudo systemctl restart storyforge-web-v4 && sleep 2 && systemctl is-active storyforge-web-v4"
+fi
+
+echo "[6/6] verify public health"
+curl -fsS --max-time "$CURL_MAX_TIME" "$BASE_URL/healthz" >/dev/null
+"$ROOT/scripts/smoke_public_storyforge.sh"
+
+echo "public deploy finished: $BASE_URL"
--- a/scripts/smoke_business.sh
+++ b/scripts/smoke_business.sh
@@ -4,7 +4,6 @@ set -eu
 BASE_URL="${STORYFORGE_BASE_URL:-http://127.0.0.1:8081}"
 USERNAME="${STORYFORGE_USERNAME:-storyforge-admin}"
 PASSWORD="${STORYFORGE_PASSWORD:-}"
-ACCOUNT_ID="${STORYFORGE_SMOKE_ACCOUNT_ID:-dyacct_c2b62842b228406cb48f05fac16fdfdf}"

 if [ -z "$PASSWORD" ]; then
  echo "STORYFORGE_PASSWORD is required. Export the bootstrap super-admin password before running smoke_business.sh." >&2
@@ -19,13 +18,23 @@ import urllib.request
 base = os.environ.get("BASE_URL", "http://127.0.0.1:8081").rstrip("/")
 username = os.environ.get("USERNAME", "storyforge-admin")
 password = os.environ.get("PASSWORD", "")
-account_id = os.environ.get("ACCOUNT_ID", "dyacct_c2b62842b228406cb48f05fac16fdfdf")
+platforms = ["douyin", "xiaohongshu", "bilibili", "kuaishou", "wechat_video"]

 if not password:
    raise SystemExit("STORYFORGE_PASSWORD is required")

-with urllib.request.urlopen(base + "/readyz", timeout=20) as resp:
-    ready = json.load(resp)
+def request_json(path: str, *, method: str = "GET", payload: dict | None = None, headers: dict | None = None, timeout: int = 30):
+    body = None
+    req_headers = {"content-type": "application/json"}
+    if headers:
+        req_headers.update(headers)
+    if payload is not None:
+        body = json.dumps(payload).encode()
+    req = urllib.request.Request(base + path, data=body, headers=req_headers, method=method)
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return json.load(resp)
+
+ready = request_json("/readyz", timeout=20)
 if not ready.get("ready"):
    raise SystemExit("collector readyz is not healthy")

@@ -40,32 +49,60 @@ with urllib.request.urlopen(login_req, timeout=20) as resp:
 token = login["token"]
 headers = {"authorization": "Bearer " + token}

-checks = [
-    ("/v2/douyin/accounts", "accounts"),
-    (f"/v2/douyin/accounts/{account_id}/workspace", "workspace"),
-    (f"/v2/douyin/accounts/{account_id}/videos?limit=5&sort_by=score", "videos"),
-]
-
 print("smoke login: ok")
-for path, label in checks:
-    req = urllib.request.Request(base + path, headers=headers)
-    with urllib.request.urlopen(req, timeout=30) as resp:
-        payload = json.load(resp)
-    if label == "accounts":
-        summary = {"accounts": len(payload)}
-    elif label == "workspace":
-        summary = {
-            "account": payload.get("account", {}).get("nickname"),
-            "reports": len(payload.get("recent_reports") or []),
-            "linked_accounts": len(payload.get("linked_accounts") or []),
-            "high_score_threshold": (payload.get("video_workspace") or {}).get("high_score_threshold"),
-        }
-    else:
-        items = payload.get("items") or []
-        summary = {
-            "videos": len(items),
-            "first_title": items[0].get("title") if items else None,
-            "first_has_analysis": bool(items and items[0].get("latest_analysis")),
-        }
-    print(f"{label}: " + json.dumps(summary, ensure_ascii=False))
+
+platform_agents = request_json("/v2/platform-agents", headers=headers)
+tenant_quota = request_json("/v2/tenant/quota", headers=headers)
+if not isinstance(platform_agents, dict):
+    raise SystemExit("/v2/platform-agents did not return an object")
+if not isinstance(tenant_quota, dict):
+    raise SystemExit("/v2/tenant/quota did not return an object")
+print("platform-agents: " + json.dumps({"items": len(platform_agents.get("items") or [])}, ensure_ascii=False))
+print("tenant-quota: " + json.dumps({"keys": sorted(tenant_quota.keys())[:6]}, ensure_ascii=False))
+
+for platform in platforms:
+    accounts = request_json(f"/v2/{platform}/accounts", headers=headers)
+    if not isinstance(accounts, list):
+        raise SystemExit(f"/v2/{platform}/accounts did not return a list")
+    digest = request_json(f"/v2/{platform}/tracking/digest", headers=headers)
+    if not isinstance(digest, dict):
+        raise SystemExit(f"/v2/{platform}/tracking/digest did not return an object")
+    digest_keys = {"generated_at", "since", "items", "tracked_accounts", "cursor_last_seen_at"}
+    if not digest_keys.issubset(digest.keys()):
+        raise SystemExit(f"/v2/{platform}/tracking/digest missing keys: {sorted(digest_keys - set(digest.keys()))}")
+
+    summary = {
+        "accounts": len(accounts),
+        "tracked_accounts": len(digest.get("tracked_accounts") or []),
+        "digest_items": len(digest.get("items") or []),
+    }
+
+    if accounts:
+        account_id = accounts[0]["id"]
+        workspace = request_json(f"/v2/{platform}/accounts/{account_id}/workspace", headers=headers)
+        analysis_reports = request_json(f"/v2/{platform}/accounts/{account_id}/analysis-reports", headers=headers)
+        if not isinstance(workspace, dict):
+            raise SystemExit(f"/v2/{platform}/accounts/{{id}}/workspace did not return an object")
+        if not isinstance(analysis_reports, list):
+            raise SystemExit(f"/v2/{platform}/accounts/{{id}}/analysis-reports did not return a list")
+        if (workspace.get("account") or {}).get("platform") != platform:
+            raise SystemExit(f"/v2/{platform}/accounts/{{id}}/workspace returned wrong platform")
+        summary.update({
+            "workspace_reports": len(workspace.get("recent_reports") or []),
+            "analysis_reports": len(analysis_reports),
+        })
+
+        if platform == "douyin":
+            snapshots = request_json(f"/v2/{platform}/accounts/{account_id}/snapshots", headers=headers)
+            if not isinstance(snapshots, list):
+                raise SystemExit("/v2/douyin/accounts/{id}/snapshots did not return a list")
+            summary["snapshots"] = len(snapshots)
+            creator_snapshots = [item for item in snapshots if item.get("snapshot_type") == "creator_center"]
+            if creator_snapshots:
+                creator_fields = request_json(f"/v2/{platform}/accounts/{account_id}/creator-fields", headers=headers)
+                if creator_fields.get("snapshot_type") != "creator_center":
+                    raise SystemExit("/v2/douyin/accounts/{id}/creator-fields returned an unexpected snapshot type")
+                summary["creator_fields"] = creator_fields.get("field_count", 0)
+
+    print(f"{platform}: " + json.dumps(summary, ensure_ascii=False))
 PY
--- a/scripts/smoke_public_storyforge.sh
+++ b/scripts/smoke_public_storyforge.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE_URL="${STORYFORGE_PUBLIC_BASE_URL:-https://storyforge.hyzq.net}"
+CURL_MAX_TIME="${STORYFORGE_PUBLIC_CURL_MAX_TIME:-60}"
+
+need_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "missing required command: $1" >&2
+    exit 1
+  fi
+}
+
+need_cmd curl
+need_cmd python3
+need_cmd rg
+
+curl_fetch() {
+  curl -fsS --max-time "$CURL_MAX_TIME" "$@"
+}
+
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf "$tmp_dir"' EXIT
+
+health_file="$tmp_dir/health.json"
+html_file="$tmp_dir/index.html"
+js_file="$tmp_dir/app.js"
+openapi_file="$tmp_dir/openapi.json"
+
+echo "[1/4] check public healthz"
+curl_fetch "$BASE_URL/healthz" >"$health_file"
+python3 - "$health_file" <<'PY'
+import json
+import pathlib
+import sys
+
+payload = json.loads(pathlib.Path(sys.argv[1]).read_text())
+status = str(payload.get("status") or "").lower()
+if status != "ok":
+    raise SystemExit(f"unexpected health status: {status!r}")
+print("healthz ok")
+PY
+
+echo "[2/4] check public index"
+curl_fetch "$BASE_URL/" >"$html_file"
+rg -q "StoryForge" "$html_file"
+echo "index ok"
+
+echo "[3/4] check deployed web bundle"
+curl_fetch "$BASE_URL/assets/app.js" >"$js_file"
+rg -q "select-platform" "$js_file"
+rg -q "trackingCursorMap" "$js_file"
+rg -q "renderPlatformSwitchChips" "$js_file"
+echo "bundle ok"
+
+echo "[4/4] check public openapi routes"
+curl_fetch "$BASE_URL/openapi.json" >"$openapi_file"
+for route in \
+  '"/v2/xiaohongshu/accounts"' \
+  '"/v2/bilibili/accounts"' \
+  '"/v2/kuaishou/accounts"' \
+  '"/v2/wechat_video/accounts"' \
+  '"/v2/platform-agents"' \
+  '"/v2/tenant/quota"'
+do
+  rg -q "$route" "$openapi_file"
+done
+echo "openapi ok"
+
+echo "public smoke passed: $BASE_URL"