commit acb1103b71ecc3cb1c131d11d998bc4e4ebe0c86 Author: kris Date: Sat Mar 14 21:32:55 2026 +0800 chore: import storyforge baseline clean diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..193ac1b --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +DEFAULT_EXTERNAL_BASE_URL=http://test.hyzq.net:8081 +LOCAL_OPENAI_BASE_URL=http://127.0.0.1:8317/v1 +LOCAL_OPENAI_MODEL=GLM-5 +LOCAL_OPENAI_API_KEY= +FASTGPT_BASE_URL=http://127.0.0.1:3000 +FASTGPT_DATASET_API_KEY= +YTDLP_BIN=yt-dlp +FFMPEG_BIN=ffmpeg +WHISPER_BIN= +WHISPER_MODEL=./data/collector/models/ggml-base.en.bin +POSTGRES_DB=fastgpt +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin +CLIPROXY_IMAGE=storyforge/cli-proxy-api:patched diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9c15fa1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +.DS_Store +.env +.env.local + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.pytest_cache/ +.mypy_cache/ +.venv/ +.venv*/ + +# Android / Gradle +.gradle/ +local.properties +build/ +**/build/ +.kotlin/ +**/.gradle/ +**/.kotlin/ + +# Runtime data and artifacts +data/ +output/ +*.log + +# macOS / editors +.idea/ +.vscode/ diff --git a/CONTENT_LEARNING_WORKFLOW.md b/CONTENT_LEARNING_WORKFLOW.md new file mode 100644 index 0000000..8db0ab9 --- /dev/null +++ b/CONTENT_LEARNING_WORKFLOW.md @@ -0,0 +1,18 @@ +# Content Learning Workflow + +1. 用户登录 StoryForge +2. 用户选择知识库和文案助手 +3. 用户通过三种方式导入素材 + - 输入短视频链接 + - 上传视频文件 + - 直接输入文字 +4. collector-service 创建学习任务 +5. 如果素材是视频 + - 使用 `yt-dlp` 下载或接收上传文件 + - 使用 `ffmpeg` 提取音频 + - 使用 `whisper.cpp` 转写，若环境未就绪则保留原始素材并进入降级流程 +6. collector-service 调用本机 OpenAI 兼容模型提炼文案风格 +7. 结果写入用户自己的知识库文档 +8. 如果配置了 `FASTGPT_DATASET_API_KEY` + - 同步到 FastGPT 数据集 +9. 文案助手生成时按知识库关联关系取素材，结合提示词输出文案 diff --git a/Common/DEPLOYMENT_AUDIT_PROMPT.md b/Common/DEPLOYMENT_AUDIT_PROMPT.md new file mode 100644 index 0000000..cd928b1 --- /dev/null +++ b/Common/DEPLOYMENT_AUDIT_PROMPT.md @@ -0,0 +1,19 @@ +# StoryForge AI 系统部署评估与自动补齐提示词 + +用于检查以下系统组件是否完整： + +1. Cloud Server +2. Mac AI Node +3. FastGPT +4. Backend API +5. Web Console +6. Android Client +7. 网络连接 +8. AI 生成流程 + +输出要求： + +- 系统部署状态 +- 缺失组件列表 +- 修复方案 +- 生成代码或部署脚本 diff --git a/Common/MAC_NODE_CONNECTIVITY_SPEC.md b/Common/MAC_NODE_CONNECTIVITY_SPEC.md new file mode 100644 index 0000000..2667548 --- /dev/null +++ b/Common/MAC_NODE_CONNECTIVITY_SPEC.md @@ -0,0 +1,18 @@ +# StoryForge Mac AI Node — Server Connectivity Specification + +The Mac node should only do the following: + +1. Deploy FastGPT locally +2. Ensure the cloud backend can reach FastGPT +3. Maintain a private network connection to the server +4. Provide the FastGPT endpoint to the backend + +Recommended ports: + +- FastGPT: 3000 +- MongoDB: 27017 +- PostgreSQL: 5432 +- Redis: 6379 +- MinIO: 9000 + +FastGPT must not be exposed to the public internet directly. diff --git a/Common/STORYFORGE_MAC_AI_NODE_TASKS.md b/Common/STORYFORGE_MAC_AI_NODE_TASKS.md new file mode 100644 index 0000000..d3b0e87 --- /dev/null +++ b/Common/STORYFORGE_MAC_AI_NODE_TASKS.md @@ -0,0 +1,27 @@ +You are responsible for the StoryForge Mac AI node. + +Tasks: + +- Deploy FastGPT using Docker. +- Services: + - FastGPT + - MongoDB + - PostgreSQL + pgvector + - Redis + - MinIO +- Build collector-service in Python. +- Collector features: + - yt-dlp video download + - ffmpeg audio extraction + - whisper.cpp transcription + - text cleaning + - knowledge upload +- Collector APIs: + - POST /collect/video + - POST /collect/audio + - POST /collect/text +- Output: + - docker-compose + - collector service + - setup scripts + - README diff --git a/MAC_NODE_CONNECTIVITY.md b/MAC_NODE_CONNECTIVITY.md new file mode 100644 index 0000000..5800a52 --- /dev/null +++ b/MAC_NODE_CONNECTIVITY.md @@ -0,0 +1,6 @@ +# Mac Node Connectivity + +- FastGPT 默认本机端口：`3000` +- Collector Service 默认本机端口：`8081` +- Local OpenAI Compatible API：`127.0.0.1:8317/v1` +- 如需通过云端访问，优先使用内网或隧道，不直接暴露 Mac 上的 FastGPT 管理接口 diff --git a/PROJECT_STATUS_2026-03-14.md b/PROJECT_STATUS_2026-03-14.md new file mode 100644 index 0000000..46bb386 --- /dev/null +++ b/PROJECT_STATUS_2026-03-14.md @@ -0,0 +1,13 @@ +# Project Status 2026-03-14 + +## 已完成 + +- `AI-glasses/android-app` 已恢复为独立 AI Glasses 应用 +- `StoryForge/android-app` 已作为独立 Android 工程保留 +- StoryForge 后端工程骨架已按当前 Android API 契约重建 +- 最高管理员账号 `kris / Asd123456.` 已在 collector-service 启动时自动补齐 + +## 注意事项 + +- 当前 macOS 文件系统对 `storyforge` / `StoryForge` 大小写不敏感，后续统一使用大写路径作为项目名 +- `yt-dlp` / `ffmpeg` / `whisper.cpp` 是否可用取决于本机环境，collector-service 已做降级处理 diff --git a/README.md b/README.md new file mode 100644 index 0000000..3eabcd8 --- /dev/null +++ b/README.md @@ -0,0 +1,40 @@ +# StoryForge + +StoryForge 现在拆成独立项目目录，和 `AI-glasses` 分开维护。 + +## 目录 + +- `android-app/`：StoryForge Android 客户端 +- `collector-service/`：FastAPI 后端，提供登录、审批、素材导入、知识库、智能体和 OTA +- `docker-compose.yml`：本地 FastGPT / collector / 基础依赖编排 +- `Common/`：项目约束和架构说明 +- `data/collector/`：SQLite、任务文件、下载产物 + +## Android + +```bash +cd /Users/kris/code/StoryForge/android-app +./gradlew assembleDebug +``` + +## Collector Service + +```bash +cd /Users/kris/code/StoryForge/collector-service +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +uvicorn app.main:app --host 0.0.0.0 --port 8081 --reload +``` + +默认会创建最高权限账号： + +- `kris` +- `Asd123456.` + +## 说明 + +- 新注册账号默认 `pending` +- 主管理员审批后才可使用核心业务接口 +- 素材入口支持文字、视频链接、视频上传 +- 可选对接本机 OpenAI 兼容模型服务和 FastGPT 数据集 API diff --git a/TECH_ARCHITECTURE.md b/TECH_ARCHITECTURE.md new file mode 100644 index 0000000..e6f37d5 --- /dev/null +++ b/TECH_ARCHITECTURE.md @@ -0,0 +1,26 @@ +# StoryForge Technical Architecture + +## Core Components + +- Android App: 素材探索、文案生产、个人配置、管理员审批、OTA +- Collector Service: FastAPI + SQLite，负责业务流程编排 +- Local Model API: 默认指向本机 `cli-proxy-api` +- FastGPT: 负责数据集和后续工作流扩展 +- MongoDB / PostgreSQL + pgvector / Redis / MinIO: FastGPT 运行依赖 + +## Main Flow + +User -> Android App -> Collector Service -> Local Model / FastGPT + +## Data Isolation + +- `accounts` +- `knowledge_bases` +- `assistants` +- `assistant_knowledge_bases` +- `knowledge_documents` +- `jobs` +- `model_profiles` +- `app_updates` + +每个用户的数据通过 `user_id` 进行隔离。 diff --git a/android-app/README.md b/android-app/README.md new file mode 100644 index 0000000..955c958 --- /dev/null +++ b/android-app/README.md @@ -0,0 +1,44 @@ +# AI Glasses Android App + +Demo Android client for backend API validation and BLE integration scaffold. + +## What is implemented + +- Backend API calls: + - `bind-confirm` + - `create session` + - `stop session` + - `device status` +- Compose UI for debug flow +- Hichips BLE protocol manager: + - service/char: `3D20(3D21/3D22/3D23)`, `5DC0(5DC1/5DC2/5DC3)` + - packet codec: `HICH + Command + Index + Length + CRC16 + Data + IPSE` + - handshake flow (`AG_CMD_HS_DEV_UUID` -> `AG_CMD_HS_APP_UUID` -> `AG_CMD_HS_DEV_INFO`) + - wake-up audio uplink (`ASR_*` commands, audio from `5DC2`) + - camera trigger (`AG_CMD_P_TAKE_START`) and thumbnail events +- New "开始对话(硬件)" button: + - BLE scan/connect -> handshake -> backend bind/create session + - start wake-up audio stream + periodic camera capture + - app reports aggregated audio/camera relay stats to backend events + +## Default backend + +The app is hardcoded to: + +`http://test.hyzq.net` + +## Build APK + +Open this folder in Android Studio: + +`/Users/kris/code/AI-glasses/android-app` + +Then run: + +```bash +./gradlew assembleDebug +``` + +APK output: + +`app/build/outputs/apk/debug/app-debug.apk` diff --git a/android-app/app/build.gradle.kts b/android-app/app/build.gradle.kts new file mode 100644 index 0000000..2bb5545 --- /dev/null +++ b/android-app/app/build.gradle.kts @@ -0,0 +1,86 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("org.jetbrains.kotlin.plugin.serialization") +} + +android { + namespace = "com.aiglasses.app" + compileSdk = 35 + + defaultConfig { + applicationId = "com.storyforge.app" + minSdk = 26 + targetSdk = 35 + versionCode = 37 + versionName = "0.6.4" + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + buildConfigField("String", "DEFAULT_STORYFORGE_BASE_URL", "\"https://test.hyzq.net/storyforge\"") + buildConfigField("String", "DEFAULT_STORYFORGE_FALLBACK_IP", "\"111.231.132.51\"") + buildConfigField("String", "DEFAULT_LOCAL_MODEL_BASE_URL", "\"http://127.0.0.1:8317/v1\"") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + kotlinOptions { + jvmTarget = "17" + } + + buildFeatures { + compose = true + buildConfig = true + } + composeOptions { + kotlinCompilerExtensionVersion = "1.5.14" + } +} + +dependencies { + val composeBom = platform("androidx.compose:compose-bom:2025.02.00") + implementation(composeBom) + androidTestImplementation(composeBom) + + implementation("androidx.core:core-ktx:1.15.0") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.7") + implementation("androidx.activity:activity-compose:1.10.0") + implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.7") + implementation("com.google.android.material:material:1.12.0") + + implementation("androidx.compose.ui:ui") + implementation("androidx.compose.ui:ui-tooling-preview") + implementation("androidx.compose.material3:material3") + debugImplementation("androidx.compose.ui:ui-tooling") + debugImplementation("androidx.compose.ui:ui-test-manifest") + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3") + + implementation("androidx.camera:camera-core:1.4.2") + implementation("androidx.camera:camera-camera2:1.4.2") + implementation("androidx.camera:camera-lifecycle:1.4.2") + implementation(files("libs/brtc-3.5.0.1a.aar")) + implementation(files("libs/lib_agent-1.0.1.4.aar")) + + implementation("com.squareup.okhttp3:okhttp:4.12.0") + implementation("com.squareup.okhttp3:logging-interceptor:4.12.0") + implementation("com.squareup.retrofit2:retrofit:2.11.0") + implementation("com.jakewharton.retrofit:retrofit2-kotlinx-serialization-converter:1.0.0") + + testImplementation("junit:junit:4.13.2") + androidTestImplementation("androidx.test.ext:junit:1.2.1") + androidTestImplementation("androidx.test.espresso:espresso-core:3.6.1") + androidTestImplementation("androidx.compose.ui:ui-test-junit4") +} diff --git a/android-app/app/libs/brtc-3.5.0.1a.aar b/android-app/app/libs/brtc-3.5.0.1a.aar new file mode 100644 index 0000000..8e870c3 Binary files /dev/null and b/android-app/app/libs/brtc-3.5.0.1a.aar differ diff --git a/android-app/app/libs/lib_agent-1.0.1.4.aar b/android-app/app/libs/lib_agent-1.0.1.4.aar new file mode 100644 index 0000000..2602820 Binary files /dev/null and b/android-app/app/libs/lib_agent-1.0.1.4.aar differ diff --git a/android-app/app/proguard-rules.pro b/android-app/app/proguard-rules.pro new file mode 100644 index 0000000..99142f9 --- /dev/null +++ b/android-app/app/proguard-rules.pro @@ -0,0 +1,2 @@ +# Keep default for demo stage. + diff --git a/android-app/app/src/main/AndroidManifest.xml b/android-app/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..3b38ddf --- /dev/null +++ b/android-app/app/src/main/AndroidManifest.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android-app/app/src/main/java/com/aiglasses/app/MainActivity.kt b/android-app/app/src/main/java/com/aiglasses/app/MainActivity.kt new file mode 100644 index 0000000..84f57ad --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/MainActivity.kt @@ -0,0 +1,51 @@ +package com.aiglasses.app + +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.compose.setContent +import androidx.activity.enableEdgeToEdge +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.lifecycle.viewmodel.compose.viewModel +import com.aiglasses.app.storyforge.StoryForgeScreen +import com.aiglasses.app.storyforge.StoryForgeViewModel +import com.aiglasses.app.ui.theme.AIGlassesTheme +import com.aiglasses.app.update.AppOtaUpdater + +class MainActivity : ComponentActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + enableEdgeToEdge() + setContent { + AIGlassesTheme { + val vm: StoryForgeViewModel = viewModel() + val state by vm.state.collectAsState() + val otaUpdater = AppOtaUpdater(this) { vm.onOtaLog(it) } + DisposableEffect(Unit) { + otaUpdater.register() + onDispose { otaUpdater.release() } + } + val videoPicker = rememberLauncherForActivityResult( + contract = ActivityResultContracts.OpenDocument() + ) { uri -> + if (uri != null) { + val fileName = contentResolver.query(uri, null, null, null, null)?.use { cursor -> + val nameIndex = cursor.getColumnIndex(android.provider.OpenableColumns.DISPLAY_NAME) + if (nameIndex >= 0 && cursor.moveToFirst()) cursor.getString(nameIndex) else null + } ?: (uri.lastPathSegment ?: "selected-video.mp4") + vm.setPickedVideo(uri, fileName) + } + } + StoryForgeScreen( + state = state, + vm = vm, + onPickVideo = { videoPicker.launch(arrayOf("video/*")) }, + onInstallLatestUpdate = { vm.installLatestUpdate(otaUpdater) } + ) + } + } + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/ble/BleManager.kt b/android-app/app/src/main/java/com/aiglasses/app/ble/BleManager.kt new file mode 100644 index 0000000..5e0bc88 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/ble/BleManager.kt @@ -0,0 +1,638 @@ +package com.aiglasses.app.ble + +import android.annotation.SuppressLint +import android.bluetooth.BluetoothAdapter +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.bluetooth.BluetoothGattCallback +import android.bluetooth.BluetoothGattCharacteristic +import android.bluetooth.BluetoothGattDescriptor +import android.bluetooth.BluetoothGattService +import android.bluetooth.BluetoothManager +import android.bluetooth.BluetoothProfile +import android.bluetooth.BluetoothStatusCodes +import android.bluetooth.le.ScanCallback +import android.bluetooth.le.ScanFilter +import android.bluetooth.le.ScanResult +import android.bluetooth.le.ScanSettings +import android.content.Context +import android.os.Build +import android.os.ParcelUuid +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.ArrayDeque +import java.util.UUID +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow +import org.json.JSONObject + +private const val MAX_FRAME_DATA = 8 * 1024 + +data class BleLinkState( + val scanning: Boolean = false, + val connected: Boolean = false, + val notificationsReady: Boolean = false, + val handshaked: Boolean = false, + val deviceName: String = "", + val deviceAddress: String = "", + val devUuid: String = "", + val lastError: String = "" +) + +sealed interface GlassesBleEvent { + data class Log(val message: String) : GlassesBleEvent + data class HandshakeOk( + val devUuid: String, + val devName: String, + val devFwVer: String + ) : GlassesBleEvent + data class StatusUpdate(val payloadJson: String) : GlassesBleEvent + data class AudioFrame(val bytes: ByteArray, val index: Int) : GlassesBleEvent + data class CameraThumbInfo(val sourceFileName: String, val isVideo: Boolean) : GlassesBleEvent + data class CameraThumbData(val bytes: ByteArray, val index: Int, val isVideo: Boolean) : GlassesBleEvent +} + +private data class HichipsFrame( + val command: Int, + val index: Int, + val payload: ByteArray +) + +private object HichipsUuid { + val service3D20: UUID = shortUuid("3d20") + val char3D21Notify: UUID = shortUuid("3d21") + val char3D22NotifyData: UUID = shortUuid("3d22") + val char3D23Write: UUID = shortUuid("3d23") + + val service5DC0: UUID = shortUuid("5dc0") + val char5DC1Notify: UUID = shortUuid("5dc1") + val char5DC2NotifyData: UUID = shortUuid("5dc2") + val char5DC3Write: UUID = shortUuid("5dc3") + + val cccd: UUID = shortUuid("2902") + + private fun shortUuid(hex: String): UUID { + return UUID.fromString("0000${hex.lowercase()}-0000-1000-8000-00805f9b34fb") + } +} + +private object HichipsCmd { + // 5DC0 wake-up stream commands + const val ASR_DEV_WAKE_UP = 0x0000 + const val ASR_APP_WAKE_UP = 0x0001 + const val ASR_TRANS_SETTING = 0x0002 + const val ASR_TRANS_START = 0x0003 + const val ASR_TRANS_FLOW_CTRL = 0x0004 + const val ASR_TRANS_AUDIO = 0x0005 + const val ASR_TRANS_APP_SET_STOP = 0x0006 + const val ASR_TRANS_STOP = 0x0007 + + // 3D20 common commands + const val AG_HS_DEV_UUID = 0x0000 + const val AG_HS_APP_UUID = 0x0001 + const val AG_HS_DEV_INFO = 0x0002 + const val AG_GET_ALL_STATUS = 0x0013 + const val AG_P_TAKE_START = 0x00A0 + const val AG_P_TAKE_STOP = 0x00A1 + const val AG_P_THUMB_INFO = 0x00A2 + const val AG_P_THUMB_DATA = 0x00A3 + const val AG_V_THUMB_INFO = 0x0094 + const val AG_V_THUMB_DATA = 0x0095 +} + +private class FrameAssembler { + private var buffer = byteArrayOf() + private val head = byteArrayOf(0x48, 0x49, 0x43, 0x48) // HICH + private val end = byteArrayOf(0x49, 0x50, 0x53, 0x45) // IPSE + + fun append(chunk: ByteArray): List { + if (chunk.isEmpty()) return emptyList() + buffer += chunk + val out = mutableListOf() + while (true) { + val start = indexOf(buffer, head) + if (start < 0) { + buffer = if (buffer.size > 3) buffer.copyOfRange(buffer.size - 3, buffer.size) else buffer + break + } + if (start > 0) { + buffer = buffer.copyOfRange(start, buffer.size) + } + if (buffer.size < 18) break + + val dataLength = leUInt32(buffer, 8) + if (dataLength < 0 || dataLength > MAX_FRAME_DATA) { + buffer = buffer.copyOfRange(1, buffer.size) + continue + } + val total = 18 + dataLength + if (buffer.size < total) break + val tail = buffer.copyOfRange(total - 4, total) + if (!tail.contentEquals(end)) { + buffer = buffer.copyOfRange(1, buffer.size) + continue + } + + val command = leUInt16(buffer, 4) + val index = leUInt16(buffer, 6) + val payload = if (dataLength > 0) { + buffer.copyOfRange(14, 14 + dataLength) + } else { + byteArrayOf() + } + val crcExpected = leUInt16(buffer, 12) + val crcActual = crc16(payload) + if (crcExpected == crcActual) { + out += HichipsFrame(command = command, index = index, payload = payload) + } + buffer = if (buffer.size == total) byteArrayOf() else buffer.copyOfRange(total, buffer.size) + } + return out + } + + fun hasPendingFrame(): Boolean { + return buffer.isNotEmpty() + } + + private fun leUInt16(bytes: ByteArray, offset: Int): Int { + return ((bytes[offset].toInt() and 0xFF) or ((bytes[offset + 1].toInt() and 0xFF) shl 8)) + } + + private fun leUInt32(bytes: ByteArray, offset: Int): Int { + val b0 = bytes[offset].toInt() and 0xFF + val b1 = bytes[offset + 1].toInt() and 0xFF + val b2 = bytes[offset + 2].toInt() and 0xFF + val b3 = bytes[offset + 3].toInt() and 0xFF + return b0 or (b1 shl 8) or (b2 shl 16) or (b3 shl 24) + } + + private fun indexOf(source: ByteArray, target: ByteArray): Int { + if (target.isEmpty()) return 0 + if (source.size < target.size) return -1 + for (i in 0..(source.size - target.size)) { + var matched = true + for (j in target.indices) { + if (source[i + j] != target[j]) { + matched = false + break + } + } + if (matched) return i + } + return -1 + } + + private fun crc16(data: ByteArray): Int { + var crc = 0xFFFF + for (b in data) { + crc = ((crc ushr 8) or ((crc and 0xFF) shl 8)) and 0xFFFF + crc = crc xor (b.toInt() and 0xFF) + crc = crc xor ((crc and 0xFF) ushr 4) + crc = crc xor ((crc shl 8) shl 4) + crc = crc xor (((crc and 0xFF) shl 4) shl 1) + crc = crc and 0xFFFF + } + return crc and 0xFFFF + } +} + +class BleManager(private val context: Context) { + private val btManager = context.getSystemService(Context.BLUETOOTH_SERVICE) as BluetoothManager + private val adapter: BluetoothAdapter? = btManager.adapter + + private val _state = MutableStateFlow(BleLinkState()) + val state: StateFlow = _state.asStateFlow() + + private val _events = MutableSharedFlow(extraBufferCapacity = 256) + val events: SharedFlow = _events.asSharedFlow() + + private var gatt: BluetoothGatt? = null + private var scannerCallback: ScanCallback? = null + private var pendingAppUuid: String = "" + private var waitingAsrStart = false + + private var write3D23: BluetoothGattCharacteristic? = null + private var write5DC3: BluetoothGattCharacteristic? = null + + private val notifyQueue = ArrayDeque() + private val assembler3D21 = FrameAssembler() + private val assembler3D22 = FrameAssembler() + private val assembler5DC1 = FrameAssembler() + private val assembler5DC2 = FrameAssembler() + + @SuppressLint("MissingPermission") + fun connectAndHandshake(appUuid: String, nameHint: String? = null) { + val bt = adapter + if (bt == null || !bt.isEnabled) { + updateError("Bluetooth not enabled") + return + } + pendingAppUuid = appUuid.take(32) + if (_state.value.connected) { + emitLog("BLE already connected, waiting for handshake packets") + return + } + stopScan() + _state.value = _state.value.copy(scanning = true, lastError = "") + val filters = listOf( + ScanFilter.Builder() + .setServiceUuid(ParcelUuid(HichipsUuid.service3D20)) + .build() + ) + val settings = ScanSettings.Builder() + .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) + .build() + scannerCallback = object : ScanCallback() { + override fun onScanResult(callbackType: Int, result: ScanResult) { + val device = result.device ?: return + val deviceName = runCatching { device.name.orEmpty() }.getOrDefault("") + if (nameHint.isNullOrBlank().not() && !deviceName.contains(nameHint!!, ignoreCase = true)) { + return + } + stopScan() + emitLog("BLE found ${device.address} ${deviceName.ifBlank { "(no-name)" }}") + connectDevice(device) + } + + override fun onScanFailed(errorCode: Int) { + updateError("BLE scan failed: $errorCode") + } + } + bt.bluetoothLeScanner?.startScan(filters, settings, scannerCallback) + emitLog("BLE scanning...") + } + + @SuppressLint("MissingPermission") + fun disconnect() { + stopScan() + runCatching { gatt?.disconnect() } + runCatching { gatt?.close() } + gatt = null + _state.value = BleLinkState() + } + + fun startWakeUpAudio() { + waitingAsrStart = true + val ok = sendAsrCommand(HichipsCmd.ASR_APP_WAKE_UP, null) + emitLog(if (ok) "ASR wake-up command sent" else "ASR wake-up send failed") + } + + fun stopWakeUpAudio() { + waitingAsrStart = false + val ok = sendAsrCommand(HichipsCmd.ASR_TRANS_APP_SET_STOP, null) + emitLog(if (ok) "ASR stop command sent" else "ASR stop send failed") + } + + fun triggerPhotoCapture() { + val ok = sendAgCommand(HichipsCmd.AG_P_TAKE_START, null) + emitLog(if (ok) "Photo capture command sent" else "Photo capture send failed") + } + + fun requestAllStatus() { + sendAgCommand(HichipsCmd.AG_GET_ALL_STATUS, null) + } + + @SuppressLint("MissingPermission") + private fun connectDevice(device: BluetoothDevice) { + runCatching { gatt?.close() } + gatt = device.connectGatt(context, false, callback, BluetoothDevice.TRANSPORT_LE) + _state.value = _state.value.copy( + scanning = false, + connected = false, + notificationsReady = false, + handshaked = false, + deviceAddress = device.address, + deviceName = runCatching { device.name.orEmpty() }.getOrDefault("") + ) + } + + @SuppressLint("MissingPermission") + private fun stopScan() { + scannerCallback?.let { cb -> + adapter?.bluetoothLeScanner?.stopScan(cb) + } + scannerCallback = null + _state.value = _state.value.copy(scanning = false) + } + + private val callback = object : BluetoothGattCallback() { + @SuppressLint("MissingPermission") + override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) { + if (status != BluetoothGatt.GATT_SUCCESS) { + updateError("BLE connect error status=$status") + return + } + if (newState == BluetoothProfile.STATE_CONNECTED) { + _state.value = _state.value.copy(connected = true, lastError = "") + emitLog("BLE connected, discovering services") + gatt.requestMtu(247) + gatt.discoverServices() + } else if (newState == BluetoothProfile.STATE_DISCONNECTED) { + _state.value = _state.value.copy( + connected = false, + notificationsReady = false, + handshaked = false + ) + emitLog("BLE disconnected") + } + } + + override fun onMtuChanged(gatt: BluetoothGatt, mtu: Int, status: Int) { + emitLog("BLE mtu=$mtu status=$status") + } + + override fun onServicesDiscovered(gatt: BluetoothGatt, status: Int) { + if (status != BluetoothGatt.GATT_SUCCESS) { + updateError("Service discovery failed: $status") + return + } + bindCharacteristics(gatt) + startEnableNotifications() + } + + override fun onDescriptorWrite( + gatt: BluetoothGatt, + descriptor: BluetoothGattDescriptor, + status: Int + ) { + if (status != BluetoothGatt.GATT_SUCCESS) { + updateError("Descriptor write failed: $status") + return + } + writeNextNotificationDescriptor() + } + + @Deprecated("Deprecated in API 33") + override fun onCharacteristicChanged( + gatt: BluetoothGatt, + characteristic: BluetoothGattCharacteristic + ) { + handleCharacteristicChanged(characteristic.uuid, characteristic.value ?: byteArrayOf()) + } + + override fun onCharacteristicChanged( + gatt: BluetoothGatt, + characteristic: BluetoothGattCharacteristic, + value: ByteArray + ) { + handleCharacteristicChanged(characteristic.uuid, value) + } + } + + @SuppressLint("MissingPermission") + private fun bindCharacteristics(gatt: BluetoothGatt) { + val s3 = gatt.getService(HichipsUuid.service3D20) + val s5 = gatt.getService(HichipsUuid.service5DC0) + write3D23 = s3?.getCharacteristic(HichipsUuid.char3D23Write) + write5DC3 = s5?.getCharacteristic(HichipsUuid.char5DC3Write) + } + + private fun startEnableNotifications() { + val g = gatt ?: return + notifyQueue.clear() + enqueueNotify(g, HichipsUuid.service3D20, HichipsUuid.char3D21Notify) + enqueueNotify(g, HichipsUuid.service3D20, HichipsUuid.char3D22NotifyData) + enqueueNotify(g, HichipsUuid.service5DC0, HichipsUuid.char5DC1Notify) + enqueueNotify(g, HichipsUuid.service5DC0, HichipsUuid.char5DC2NotifyData) + writeNextNotificationDescriptor() + } + + private fun enqueueNotify(gatt: BluetoothGatt, serviceUuid: UUID, charUuid: UUID) { + val characteristic = gatt.getService(serviceUuid)?.getCharacteristic(charUuid) ?: return + notifyQueue.add(characteristic) + } + + @SuppressLint("MissingPermission") + private fun writeNextNotificationDescriptor() { + val g = gatt ?: return + if (notifyQueue.isEmpty()) { + _state.value = _state.value.copy(notificationsReady = true) + emitLog("BLE notifications enabled") + return + } + val c = notifyQueue.removeFirst() + g.setCharacteristicNotification(c, true) + val descriptor = c.getDescriptor(HichipsUuid.cccd) ?: run { + writeNextNotificationDescriptor() + return + } + val value = BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val result = g.writeDescriptor(descriptor, value) + if (result != BluetoothStatusCodes.SUCCESS) { + updateError("writeDescriptor failed: $result") + } + } else { + @Suppress("DEPRECATION") + run { + descriptor.value = value + val ok = g.writeDescriptor(descriptor) + if (!ok) updateError("writeDescriptor returned false") + } + } + } + + private fun handleCharacteristicChanged(uuid: UUID, value: ByteArray) { + if (value.isEmpty()) return + when (uuid) { + HichipsUuid.char3D21Notify -> decodeAndDispatchFrames(value, assembler3D21, isWakeChannel = false, isDataChannel = false) + HichipsUuid.char5DC1Notify -> decodeAndDispatchFrames(value, assembler5DC1, isWakeChannel = true, isDataChannel = false) + HichipsUuid.char3D22NotifyData -> decodeAndDispatchFrames(value, assembler3D22, isWakeChannel = false, isDataChannel = true) + HichipsUuid.char5DC2NotifyData -> decodeAndDispatchFrames(value, assembler5DC2, isWakeChannel = true, isDataChannel = true) + } + } + + private fun decodeAndDispatchFrames( + value: ByteArray, + assembler: FrameAssembler, + isWakeChannel: Boolean, + isDataChannel: Boolean + ) { + val isPacketized = value.size >= 4 && + value[0] == 0x48.toByte() && + value[1] == 0x49.toByte() && + value[2] == 0x43.toByte() && + value[3] == 0x48.toByte() + + if (isDataChannel && !isPacketized && !assembler.hasPendingFrame()) { + if (isWakeChannel) { + _events.tryEmit(GlassesBleEvent.AudioFrame(bytes = value, index = 0)) + } + return + } + + val frames = assembler.append(value) + for (frame in frames) { + onFrame(frame, isWakeChannel, isDataChannel) + } + } + + private fun onFrame(frame: HichipsFrame, isWakeChannel: Boolean, isDataChannel: Boolean) { + if (isWakeChannel && isDataChannel && frame.command == HichipsCmd.ASR_TRANS_AUDIO) { + _events.tryEmit(GlassesBleEvent.AudioFrame(bytes = frame.payload, index = frame.index)) + return + } + + if (!isWakeChannel && isDataChannel) { + when (frame.command) { + HichipsCmd.AG_P_THUMB_DATA -> _events.tryEmit( + GlassesBleEvent.CameraThumbData( + bytes = frame.payload, + index = frame.index, + isVideo = false + ) + ) + HichipsCmd.AG_V_THUMB_DATA -> _events.tryEmit( + GlassesBleEvent.CameraThumbData( + bytes = frame.payload, + index = frame.index, + isVideo = true + ) + ) + } + return + } + + if (isWakeChannel) { + when (frame.command) { + HichipsCmd.ASR_DEV_WAKE_UP -> { + emitLog("Device wake-up received") + if (waitingAsrStart) { + val setting = JSONObject() + .put("FlowCtrl", 0) + .put("LengthByte", 80) + .put("IntervalMs", 20) + .put("Packag", 1) + sendAsrCommand(HichipsCmd.ASR_TRANS_SETTING, setting.toString()) + } + } + HichipsCmd.ASR_TRANS_START -> emitLog("ASR trans start") + HichipsCmd.ASR_TRANS_STOP -> emitLog("ASR trans stop") + } + return + } + + when (frame.command) { + HichipsCmd.AG_HS_DEV_UUID -> { + val json = parseJson(frame.payload) + val devUuid = json?.optString("DevUuid", "").orEmpty() + if (devUuid.isNotBlank()) { + _state.value = _state.value.copy(devUuid = devUuid) + val appUuidPayload = JSONObject() + .put("Time", System.currentTimeMillis() / 1000L) + .put("AppUuid", pendingAppUuid.take(32)) + .toString() + sendAgCommand(HichipsCmd.AG_HS_APP_UUID, appUuidPayload) + emitLog("Handshake step2 done, app uuid sent") + } + } + HichipsCmd.AG_HS_DEV_INFO -> { + val json = parseJson(frame.payload) + val fail = json?.optString("Status") == "Fail" + if (fail) { + updateError("Handshake rejected: ${json?.optInt("ErrorCode", -1)}") + return + } + val devUuid = json?.optString("DevUuid", _state.value.devUuid).orEmpty() + val devName = json?.optString("DevName", "").orEmpty() + val fw = json?.optString("DevFwVer", "").orEmpty() + _state.value = _state.value.copy( + handshaked = true, + devUuid = devUuid.ifBlank { _state.value.devUuid }, + deviceName = devName.ifBlank { _state.value.deviceName } + ) + _events.tryEmit(GlassesBleEvent.HandshakeOk(devUuid = _state.value.devUuid, devName = devName, devFwVer = fw)) + emitLog("Handshake completed") + } + HichipsCmd.AG_GET_ALL_STATUS -> { + val jsonText = frame.payload.decodeToString() + _events.tryEmit(GlassesBleEvent.StatusUpdate(jsonText)) + } + HichipsCmd.AG_P_THUMB_INFO -> { + val source = parseJson(frame.payload)?.optString("SourceFileName", "").orEmpty() + _events.tryEmit(GlassesBleEvent.CameraThumbInfo(sourceFileName = source, isVideo = false)) + } + HichipsCmd.AG_V_THUMB_INFO -> { + val source = parseJson(frame.payload)?.optString("SourceFileName", "").orEmpty() + _events.tryEmit(GlassesBleEvent.CameraThumbInfo(sourceFileName = source, isVideo = true)) + } + } + } + + private fun parseJson(bytes: ByteArray): JSONObject? { + if (bytes.isEmpty()) return null + return runCatching { + JSONObject(bytes.decodeToString()) + }.getOrNull() + } + + private fun sendAgCommand(command: Int, jsonPayload: String?): Boolean { + val payload = jsonPayload?.toByteArray(Charsets.UTF_8) ?: byteArrayOf() + return writeFrame(write3D23, command, payload) + } + + private fun sendAsrCommand(command: Int, jsonPayload: String?): Boolean { + val payload = jsonPayload?.toByteArray(Charsets.UTF_8) ?: byteArrayOf() + return writeFrame(write5DC3, command, payload) + } + + @SuppressLint("MissingPermission") + private fun writeFrame( + characteristic: BluetoothGattCharacteristic?, + command: Int, + payload: ByteArray + ): Boolean { + val g = gatt ?: return false + val c = characteristic ?: return false + val frame = buildFrame(command = command, index = 0, payload = payload) + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + g.writeCharacteristic(c, frame, BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE) == + BluetoothStatusCodes.SUCCESS + } else { + @Suppress("DEPRECATION") + run { + c.writeType = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE + c.value = frame + g.writeCharacteristic(c) + } + } + } + + private fun buildFrame(command: Int, index: Int, payload: ByteArray): ByteArray { + val buffer = ByteBuffer.allocate(18 + payload.size).order(ByteOrder.LITTLE_ENDIAN) + buffer.put(byteArrayOf(0x48, 0x49, 0x43, 0x48)) // HICH + buffer.putShort(command.toShort()) + buffer.putShort(index.toShort()) + buffer.putInt(payload.size) + buffer.putShort(crc16(payload).toShort()) + if (payload.isNotEmpty()) buffer.put(payload) + buffer.put(byteArrayOf(0x49, 0x50, 0x53, 0x45)) // IPSE + return buffer.array() + } + + private fun crc16(data: ByteArray): Int { + var crc = 0xFFFF + for (b in data) { + crc = ((crc ushr 8) or ((crc and 0xFF) shl 8)) and 0xFFFF + crc = crc xor (b.toInt() and 0xFF) + crc = crc xor ((crc and 0xFF) ushr 4) + crc = crc xor ((crc shl 8) shl 4) + crc = crc xor (((crc and 0xFF) shl 4) shl 1) + crc = crc and 0xFFFF + } + return crc and 0xFFFF + } + + private fun emitLog(message: String) { + _events.tryEmit(GlassesBleEvent.Log(message)) + } + + private fun updateError(message: String) { + _state.value = _state.value.copy(lastError = message) + _events.tryEmit(GlassesBleEvent.Log("ERROR: $message")) + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/software/BaiduConversationAgent.kt b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduConversationAgent.kt new file mode 100644 index 0000000..c08aa0c --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduConversationAgent.kt @@ -0,0 +1,325 @@ +package com.aiglasses.app.software + +import android.content.Context +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import com.baidu.rtc.agent.AIAgentEngine +import com.baidu.rtc.agent.AIAgentEngineCallback +import com.baidu.rtc.agent.Constants +import java.io.File + +private const val BAIDU_AGENT_RECONNECT_DELAY_MS = 900L +private const val BAIDU_IMAGE_UPLOAD_EXPIRE_SECONDS = 0 + +class BaiduConversationAgent( + context: Context, + private val onLog: (String) -> Unit, + private val onCallReady: () -> Unit, + private val onCallEnded: (String) -> Unit, + private val onFinalAsr: (String) -> Unit, + private val onAgentText: (String) -> Unit, + private val onTtsStart: () -> Unit, + private val onTtsEnd: () -> Unit, + private val onPlaybackAudio: (pcm: ByteArray, sampleRate: Int, channelCount: Int) -> Unit, + private val onImageUploadRequest: () -> Unit, +) { + private val appContext = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + + private var engine: AIAgentEngine? = null + private var session: SessionConfig? = null + private var running = false + private var callBegun = false + private var reconnectScheduled = false + private var stopRequested = false + private var pendingUploadFile: File? = null + + private val callback = object : AIAgentEngineCallback() { + override fun onConnectionStateChange(state: Int) { + onLog("Baidu agent connection state=$state") + } + + override fun onCallStateChange(state: Int) { + when (state) { + Constants.CallState.ON_CALL_BEGIN -> { + callBegun = true + onLog("Baidu agent call begin") + onCallReady() + flushPendingUpload() + } + + Constants.CallState.ON_CALL_END -> { + callBegun = false + onLog("Baidu agent call ended") + onCallEnded("call_end") + if (running && !stopRequested) { + scheduleReconnect("call_end") + } + } + } + } + + override fun onError(error: Int, msg: String?, bundle: Bundle?) { + onLog("Baidu agent error: code=$error, msg=${msg?.take(80) ?: "-"}") + onCallEnded("error:$error") + if (running && !stopRequested) { + restart("error:$error") + } + } + + override fun onLicenseStatus(code: Int) { + onLog("Baidu agent license status=$code") + } + + override fun onUserAsrSubtitle(text: String?, isFinal: Boolean) { + if (!isFinal) return + val normalized = sanitizeText(text.orEmpty()) + if (normalized.isNotBlank()) { + onFinalAsr(normalized) + } + } + + override fun onAIAgentSubtitle(text: String?, isFinal: Boolean) { + if (!isFinal) return + val normalized = sanitizeText(text.orEmpty()) + if (normalized.isNotBlank()) { + onAgentText(normalized) + } + } + + override fun onAIAgentAudioStateChange(newState: Int) { + when (newState) { + Constants.AIAgentAudioStateType.SPEAKING -> onTtsStart() + Constants.AIAgentAudioStateType.STOPPED -> onTtsEnd() + } + } + + override fun onPlaybackAudioFrame(data: ByteArray?, sampleRate: Int, channelCount: Int) { + val frame = data ?: return + if (frame.isEmpty()) return + onPlaybackAudio(frame, sampleRate, channelCount) + } + + override fun onAgentIntent(type: String?, bundle: Bundle?) { + if (type == Constants.AgentIntentType.IMAGE_UPLOAD) { + onImageUploadRequest() + } + } + + override fun onUploadFileStatus(code: Int, msg: String?) { + onLog("Baidu visual upload status: code=$code, msg=${msg?.take(80) ?: "-"}") + } + + override fun onMessage(message: String?) { + val text = sanitizeText(message.orEmpty()) + if (text.isNotBlank()) { + onLog("Baidu agent message: ${text.take(120)}") + } + } + } + + fun updateSession( + appId: String, + cid: String, + token: String, + contextJson: String, + deviceId: String, + appUserId: String, + licenseKey: String, + ) { + val next = SessionConfig( + appId = appId.trim(), + cid = cid.trim(), + token = token.trim(), + contextJson = contextJson.trim(), + deviceId = deviceId.trim(), + appUserId = appUserId.trim(), + licenseKey = licenseKey.trim(), + ) + val changed = next != session + session = next + if (running && changed) { + onLog("Baidu session updated, restarting agent") + restart("session_updated") + } + } + + fun start() { + running = true + stopRequested = false + startIfReady() + } + + fun stop() { + running = false + stopRequested = true + reconnectScheduled = false + mainHandler.removeCallbacksAndMessages(RECONNECT_TOKEN) + pendingUploadFile?.let { safeDelete(it) } + pendingUploadFile = null + destroyEngine() + } + + fun isCallActive(): Boolean = callBegun + + fun pushAudioFrame(pcm: ByteArray, sampleRate: Int, channelCount: Int) { + if (!callBegun || pcm.isEmpty()) return + runCatching { + engine?.pushAudioFrame(pcm, System.nanoTime(), sampleRate, channelCount) + }.onFailure { + onLog("Baidu audio push failed: ${it.message}") + } + } + + fun interrupt() { + if (!callBegun) return + runCatching { engine?.interrupt() } + .onFailure { onLog("Baidu interrupt failed: ${it.message}") } + } + + fun uploadJpeg(jpegBytes: ByteArray): Boolean { + val file = prepareUploadFile(jpegBytes) ?: return false + if (!callBegun) { + pendingUploadFile?.let { safeDelete(it) } + pendingUploadFile = file + onLog("Baidu visual upload queued: waiting call begin") + return true + } + return sendUploadFile(file) + } + + private fun startIfReady() { + if (!running || engine != null) return + val cfg = session ?: run { + onLog("Baidu agent start pending: session missing") + return + } + if (cfg.appId.isBlank() || cfg.cid.isBlank() || cfg.token.isBlank()) { + onLog("Baidu agent start pending: missing appId/cid/token") + return + } + val cidLong = cfg.cid.toLongOrNull() + if (cidLong == null) { + onLog("Baidu agent start failed: cid not numeric") + return + } + val params = AIAgentEngine.AIAgentEngineParams().apply { + appId = cfg.appId + workflow = "voiceChat" + aiAgentInstanceId = cidLong + context = cfg.contextJson + verbose = true + enableExternalAudioInput = true + enableExternalAudioOutput = true + enableVoiceInterrupt = false + licenseKey = cfg.licenseKey + // SDK internal license activation sends devId=userId, so this must be the device identity. + userId = cfg.deviceId + } + val nextEngine = runCatching { AIAgentEngine.init(appContext, params) } + .onFailure { onLog("Baidu agent init failed: ${it.message}") } + .getOrNull() ?: return + engine = nextEngine + nextEngine.setCallback(callback) + onLog( + "Baidu agent calling: cid=${cfg.cid}, deviceId=${cfg.deviceId}, " + + "appUserId=${cfg.appUserId}, contextLen=${cfg.contextJson.length}" + ) + runCatching { + nextEngine.call(cfg.token, cidLong) + nextEngine.switchToSpeaker(true) + }.onFailure { + onLog("Baidu agent call failed: ${it.message}") + destroyEngine() + scheduleReconnect("call_failed") + } + } + + private fun restart(reason: String) { + destroyEngine() + scheduleReconnect(reason) + } + + private fun scheduleReconnect(reason: String) { + if (!running || reconnectScheduled) return + reconnectScheduled = true + onLog("Baidu agent reconnect scheduled: $reason") + mainHandler.postAtTime( + { + reconnectScheduled = false + if (!running) return@postAtTime + startIfReady() + }, + RECONNECT_TOKEN, + SystemClock.uptimeMillis() + BAIDU_AGENT_RECONNECT_DELAY_MS + ) + } + + private fun destroyEngine() { + val current = engine ?: run { + callBegun = false + return + } + engine = null + callBegun = false + runCatching { current.hangup() } + runCatching { current.destroy() } + } + + private fun flushPendingUpload() { + val pending = pendingUploadFile ?: return + pendingUploadFile = null + sendUploadFile(pending) + } + + private fun sendUploadFile(file: File): Boolean { + val current = engine ?: run { + safeDelete(file) + return false + } + val ok = runCatching { current.uploadFile(file.absolutePath, BAIDU_IMAGE_UPLOAD_EXPIRE_SECONDS) } + .onFailure { onLog("Baidu visual upload call failed: ${it.message}") } + .getOrDefault(false) + if (ok) { + onLog("Baidu visual upload sent: ${file.name}, bytes=${file.length()}") + mainHandler.postDelayed({ safeDelete(file) }, 60_000L) + } else { + safeDelete(file) + onLog("Baidu visual upload send failed") + } + return ok + } + + private fun prepareUploadFile(jpegBytes: ByteArray): File? { + return runCatching { + val dir = File(appContext.cacheDir, "baidu_uploads").apply { mkdirs() } + File.createTempFile("vision_", ".jpg", dir).apply { writeBytes(jpegBytes) } + }.onFailure { + onLog("Baidu visual file prepare failed: ${it.message}") + }.getOrNull() + } + + private fun sanitizeText(raw: String): String { + return raw.substringBefore("|||").trim() + } + + private fun safeDelete(file: File) { + runCatching { file.delete() } + } + + private data class SessionConfig( + val appId: String, + val cid: String, + val token: String, + val contextJson: String, + val deviceId: String, + val appUserId: String, + val licenseKey: String, + ) + + private companion object { + val RECONNECT_TOKEN = Any() + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/software/BaiduRealtimeWsClient.kt b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduRealtimeWsClient.kt new file mode 100644 index 0000000..be9a60e --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduRealtimeWsClient.kt @@ -0,0 +1,98 @@ +package com.aiglasses.app.software + +import java.util.concurrent.TimeUnit +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener +import okio.ByteString +import okio.ByteString.Companion.toByteString + +class BaiduRealtimeWsClient( + private val onLog: (String) -> Unit, + private val onOpen: () -> Unit, + private val onText: (String) -> Unit, + private val onBinary: (ByteArray) -> Unit, + private val onClosed: (reason: String, byClient: Boolean) -> Unit, +) { + private val client = OkHttpClient.Builder() + .retryOnConnectionFailure(true) + .pingInterval(20, TimeUnit.SECONDS) + .build() + + @Volatile + private var webSocket: WebSocket? = null + + @Volatile + private var closedByClient = false + + fun connect(url: String) { + disconnect("reconnect") + closedByClient = false + val request = Request.Builder().url(url).build() + webSocket = client.newWebSocket(request, listener) + } + + fun disconnect(reason: String = "client_stop") { + closedByClient = true + val current = webSocket + webSocket = null + runCatching { current?.close(1000, reason) } + runCatching { current?.cancel() } + } + + fun sendText(text: String): Boolean { + return runCatching { webSocket?.send(text) == true } + .onFailure { onLog("Realtime WS send text failed: ${it.message}") } + .getOrDefault(false) + } + + fun sendBinary(bytes: ByteArray): Boolean { + if (bytes.isEmpty()) return false + return runCatching { webSocket?.send(bytes.toByteString()) == true } + .onFailure { onLog("Realtime WS send binary failed: ${it.message}") } + .getOrDefault(false) + } + + fun release() { + disconnect("release") + runCatching { client.dispatcher.executorService.shutdown() } + runCatching { client.connectionPool.evictAll() } + } + + private val listener = object : WebSocketListener() { + override fun onOpen(webSocket: WebSocket, response: Response) { + this@BaiduRealtimeWsClient.webSocket = webSocket + onOpen() + } + + override fun onMessage(webSocket: WebSocket, text: String) { + onText(text) + } + + override fun onMessage(webSocket: WebSocket, bytes: ByteString) { + onBinary(bytes.toByteArray()) + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + if (this@BaiduRealtimeWsClient.webSocket === webSocket) { + this@BaiduRealtimeWsClient.webSocket = null + } + onClosed("closed:$code:${reason.ifBlank { "-" }}", closedByClient) + } + + override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { + runCatching { webSocket.close(code, reason) } + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + if (this@BaiduRealtimeWsClient.webSocket === webSocket) { + this@BaiduRealtimeWsClient.webSocket = null + } + val code = response?.code ?: -1 + val message = t.message ?: response?.message ?: "unknown" + onClosed("failure:$code:$message", closedByClient) + } + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/software/BaiduVisualUploader.kt b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduVisualUploader.kt new file mode 100644 index 0000000..b741e2b --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/software/BaiduVisualUploader.kt @@ -0,0 +1,240 @@ +package com.aiglasses.app.software + +import android.content.Context +import android.os.Bundle +import com.baidu.rtc.agent.AIAgentEngine +import com.baidu.rtc.agent.AIAgentEngineCallback +import com.baidu.rtc.agent.Constants +import java.io.File + +private const val VISUAL_UPLOAD_EXPIRE_SECONDS = 0 +private const val VISUAL_UPLOAD_KEEP_MS = 10 * 60 * 1000L + +class BaiduVisualUploader( + context: Context, + private val onLog: (String) -> Unit +) { + private data class SessionConfig( + val appId: String, + val cid: String, + val token: String, + val userId: String, + val licenseKey: String + ) { + fun isValid(): Boolean = appId.isNotBlank() && cid.isNotBlank() && token.isNotBlank() + fun key(): String = listOf(appId, cid, token, userId, licenseKey).joinToString("|") + } + + private val appContext = context.applicationContext + private val uploadDir = File(appContext.cacheDir, "baidu_visual_uploads").apply { mkdirs() } + + private var sessionConfig: SessionConfig? = null + private var startedKey = "" + private var engine: AIAgentEngine? = null + private var ready = false + private var activeUploadFile: File? = null + private var pendingUploadFile: File? = null + + private val callback = object : AIAgentEngineCallback() { + override fun onCallStateChange(state: Int) { + when (state) { + Constants.CallState.ON_CALL_BEGIN -> { + ready = true + engine?.muteMic(true) + engine?.mutePlayback(true) + onLog("Baidu visual uploader ready") + flushPendingUpload() + } + + Constants.CallState.ON_CALL_END -> { + ready = false + onLog("Baidu visual uploader call ended") + } + } + } + + override fun onConnectionStateChange(state: Int) { + onLog("Baidu visual connection state=$state") + } + + override fun onUploadFileStatus(code: Int, msg: String) { + onLog("Baidu visual upload status: code=$code, msg=${msg.take(80)}") + deleteFile(activeUploadFile) + activeUploadFile = null + } + + override fun onLicenseStatus(code: Int) { + onLog("Baidu visual license status=$code") + } + + override fun onAgentIntent(type: String, bundle: Bundle?) { + if (type == Constants.AgentIntentType.IMAGE_UPLOAD) { + onLog("Baidu visual agent intent: IMAGE_UPLOAD") + } + } + + override fun onError(error: Int, msg: String?, bundle: Bundle?) { + onLog("Baidu visual uploader error: code=$error, msg=${msg ?: "-"}") + } + + override fun onMessage(message: String?) { + if (!message.isNullOrBlank()) { + onLog("Baidu visual message: ${message.take(80)}") + } + } + } + + fun updateSession(appId: String, cid: String, token: String, userId: String, licenseKey: String) { + val next = SessionConfig( + appId = appId.trim(), + cid = cid.trim(), + token = token.trim(), + userId = userId.trim(), + licenseKey = licenseKey.trim() + ) + if (next == sessionConfig) return + sessionConfig = next + val key = next.key() + if (engine != null && startedKey.isNotBlank() && key != startedKey) { + onLog("Baidu visual uploader session changed, restarting") + stop() + } + } + + fun start() { + ensureStarted() + } + + fun stop() { + ready = false + startedKey = "" + runCatching { engine?.hangup() } + runCatching { engine?.destroy() } + engine = null + deleteFile(activeUploadFile) + activeUploadFile = null + deleteFile(pendingUploadFile) + pendingUploadFile = null + } + + fun uploadJpeg(jpegBytes: ByteArray): Boolean { + if (jpegBytes.isEmpty()) return false + val cfg = sessionConfig + if (cfg == null || !cfg.isValid()) { + onLog("Baidu visual uploader skipped: missing appId/cid/token") + return false + } + cleanupStaleFiles() + val file = runCatching { + File(uploadDir, "visual_${System.currentTimeMillis()}.jpg").apply { + writeBytes(jpegBytes) + } + }.getOrElse { + onLog("Baidu visual file prepare failed: ${it.message}") + return false + } + if (!ensureStarted()) { + deleteFile(file) + return false + } + if (!ready) { + replacePendingUpload(file) + onLog("Baidu visual upload queued: waiting call begin") + return true + } + return sendUploadFile(file) + } + + private fun ensureStarted(): Boolean { + val cfg = sessionConfig + if (cfg == null || !cfg.isValid()) return false + val key = cfg.key() + if (engine != null && startedKey == key) return true + val cidLong = cfg.cid.toLongOrNull() + if (cidLong == null) { + onLog("Baidu visual uploader skipped: cid not numeric") + return false + } + stop() + val params = AIAgentEngine.AIAgentEngineParams().apply { + appId = cfg.appId + workflow = "voiceChat" + context = "" + verbose = true + enableExternalAudioInput = true + enableExternalAudioOutput = true + licenseKey = cfg.licenseKey + userId = cfg.userId + } + val nextEngine = runCatching { + AIAgentEngine.init(appContext, params) + }.getOrElse { + onLog("Baidu visual uploader init failed: ${it.message}") + return false + } + engine = nextEngine + engine?.setCallback(callback) + ready = false + startedKey = key + onLog("Baidu visual uploader calling: cid=${cfg.cid}") + runCatching { + nextEngine.call(cfg.token, cidLong) + }.onFailure { + onLog("Baidu visual uploader call failed: ${it.message}") + stop() + return false + } + return true + } + + private fun flushPendingUpload() { + val file = pendingUploadFile ?: return + pendingUploadFile = null + if (!sendUploadFile(file)) { + replacePendingUpload(file) + } + } + + private fun sendUploadFile(file: File): Boolean { + val nextEngine = engine ?: return false + deleteFile(activeUploadFile) + activeUploadFile = file + val ok = runCatching { + nextEngine.uploadFile(file.absolutePath, VISUAL_UPLOAD_EXPIRE_SECONDS) + }.getOrElse { + onLog("Baidu visual upload call failed: ${it.message}") + false + } + if (ok) { + onLog("Baidu visual upload sent: ${file.name}, bytes=${file.length()}") + } else { + onLog("Baidu visual upload send failed") + deleteFile(activeUploadFile) + activeUploadFile = null + } + return ok + } + + private fun replacePendingUpload(file: File) { + deleteFile(pendingUploadFile) + pendingUploadFile = file + } + + private fun cleanupStaleFiles() { + val cutoff = System.currentTimeMillis() - VISUAL_UPLOAD_KEEP_MS + uploadDir.listFiles()?.forEach { file -> + if (file.lastModified() < cutoff) { + deleteFile(file) + } + } + } + + private fun deleteFile(file: File?) { + if (file == null) return + runCatching { + if (file.exists()) { + file.delete() + } + } + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/software/SoftwareConversationController.kt b/android-app/app/src/main/java/com/aiglasses/app/software/SoftwareConversationController.kt new file mode 100644 index 0000000..e5a5982 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/software/SoftwareConversationController.kt @@ -0,0 +1,1482 @@ +package com.aiglasses.app.software + +import android.content.Context +import android.content.Intent +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.AudioTrack +import android.media.MediaRecorder +import android.media.MediaPlayer +import android.media.ToneGenerator +import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.AutomaticGainControl +import android.media.audiofx.NoiseSuppressor +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.speech.tts.TextToSpeech +import android.util.Size +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.core.content.ContextCompat +import androidx.lifecycle.LifecycleOwner +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.Locale +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import android.util.Base64 +import kotlin.math.PI +import kotlin.math.sin + +private const val CAMERA_INTERVAL_MS = 12_000L +private const val FIRST_CAPTURE_DELAY_MS = 1_200L +private const val MIC_SAMPLE_RATE = 16_000 +private const val MIC_LOG_INTERVAL_MS = 3_000L +private const val REALTIME_FRAME_MS = 20 +private const val REALTIME_PCM_FRAME_BYTES = MIC_SAMPLE_RATE * 2 * REALTIME_FRAME_MS / 1000 +private const val REALTIME_BINARY_LOG_INTERVAL_MS = 3_000L +private const val REALTIME_UPLINK_LOG_INTERVAL_MS = 3_000L +private const val REALTIME_ENABLE_AUDIO_GRACE_MS = 240L +private const val BEEP_SAMPLE_RATE = 16_000 +private const val CAMERA_TARGET_WIDTH = 960 +private const val CAMERA_TARGET_HEIGHT = 540 +private const val CAMERA_JPEG_QUALITY = 62 +private const val BARGE_IN_RMS_THRESHOLD = 2_200 +private const val BARGE_IN_REMOTE_RMS_THRESHOLD = 3_000 +private const val BARGE_IN_CONSECUTIVE_FRAMES = 4 +private const val BARGE_IN_MIN_AFTER_TTS_MS = 900L +private const val BARGE_IN_COOLDOWN_MS = 1800L +private const val REALTIME_IMAGE_STALE_MS = 20_000L +private const val REALTIME_IMAGE_FRESH_MS = 3_000L +private const val REMOTE_AUDIO_DROP_AFTER_BREAK_MS = 1_200L +private const val VISION_RESPONSE_TIMEOUT_MS = 8_000L +private const val TTS_BINARY_LOG_SAMPLE_BYTES = 640 +private const val REALTIME_IMAGE_UPLOAD_MAX_DIM = 960 +private const val REALTIME_IMAGE_UPLOAD_TARGET_BYTES = 72_000 +private const val REALTIME_IMAGE_UPLOAD_MIN_QUALITY = 38 +private const val REALTIME_ASR_RETRY_WINDOW_MS = 800L +private const val VISION_SYSTEM_PROMPT = + "你具备视觉理解能力。收到上传图片后，必须结合最近一次上传的图片回答用户关于当前画面的问题。" + + "优先描述主体、位置、颜色、动作和环境；如果图片模糊、过暗或被遮挡，再说明原因，不要直接说自己看不到。" + +class SoftwareConversationController( + private val context: Context, + private val lifecycleOwner: LifecycleOwner, + private val onLog: (String) -> Unit, + private val onSpeechText: (String) -> Unit, + private val onRealtimeAsrText: (String) -> Unit, + private val onBargeInDetected: () -> Unit, + private val onCameraCapture: (imageBase64: String, bytes: Int, width: Int, height: Int) -> Unit, + private val onRealtimeVisionFrameReady: ( + imageBase64: String, + bytes: Int, + width: Int, + height: Int, + source: String, + ) -> Unit, + private val onMicFallbackFrame: (pcmBytes: ByteArray, rms: Int, sampleRate: Int) -> Unit, +) { + private val mainHandler = Handler(Looper.getMainLooper()) + private val cameraExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + private val realtimeWsClient = BaiduRealtimeWsClient( + onLog = onLog, + onOpen = { + realtimeConnected = true + reconnectRealtimePending = false + onLog("Realtime WS connected") + }, + onText = { text -> + onLog("Realtime WS text: ${text.take(160)}") + handleRealtimeText(text) + }, + onBinary = { pcm -> + val now = SystemClock.elapsedRealtime() + remoteVoiceActive = true + onRealtimeBinary(now, pcm.size) + if (now - lastRealtimeBinaryLogAt >= REALTIME_BINARY_LOG_INTERVAL_MS) { + lastRealtimeBinaryLogAt = now + onLog("Realtime PCM packet: bytes=${pcm.size}, sampleRate=$MIC_SAMPLE_RATE, channels=1") + } + playRealtimePcm(pcm, MIC_SAMPLE_RATE, 1) + }, + onClosed = { reason, byClient -> + realtimeConnected = false + remoteVoiceActive = false + if (byClient) { + onLog("Realtime WS closed: $reason") + } else { + onLog("Realtime WS failed: $reason") + maybeReconnectRealtime(reason) + } + } + ) + private val baiduConversationAgent = BaiduConversationAgent( + context = context, + onLog = onLog, + onCallReady = { + realtimeConnected = true + realtimeAsrEnabled = true + realtimeMicSuppressed = false + realtimeDeviceInfoSent = true + realtimeVisionPromptSent = true + onLog("Realtime ASR pipeline enabled: official agent") + }, + onCallEnded = { + realtimeConnected = false + realtimeAsrEnabled = false + realtimeMicSuppressed = false + remoteVoiceActive = false + }, + onFinalAsr = { recognized -> + onLog("Realtime ASR: ${recognized.take(80)}") + maybeDispatchRealtimeAsr(recognized) + }, + onAgentText = { answer -> + onLog("Realtime AI: ${answer.take(80)}") + }, + onTtsStart = { + realtimeMicSuppressed = true + ttsBeginAt = SystemClock.elapsedRealtime() + bargeInConsecutiveFrames = 0 + remoteVoiceActive = true + resetRealtimeTtsStats() + onLog("Realtime MIC suppressed: TTS begin") + }, + onTtsEnd = { + realtimeMicSuppressed = false + bargeInConsecutiveFrames = 0 + remoteVoiceActive = false + logRealtimeTtsSummary() + onLog("Realtime MIC resumed: TTS end") + }, + onPlaybackAudio = { pcm, sampleRate, channelCount -> + val now = SystemClock.elapsedRealtime() + if (now >= dropRemoteAudioUntil) { + remoteVoiceActive = true + onRealtimeBinary(now, pcm.size) + if (now - lastRealtimeBinaryLogAt >= REALTIME_BINARY_LOG_INTERVAL_MS) { + lastRealtimeBinaryLogAt = now + onLog("Realtime PCM packet: bytes=${pcm.size}, sampleRate=$sampleRate, channels=$channelCount") + } + playRealtimePcm(pcm, sampleRate, channelCount) + } + }, + onImageUploadRequest = { + handleRealtimeImageRequest() + } + ) + + private var running = false + private var speechRecognizer: SpeechRecognizer? = null + private var textToSpeech: TextToSpeech? = null + private var ttsReady = false + private var toneGenerator: ToneGenerator? = null + private var cameraProvider: ProcessCameraProvider? = null + private var imageCapture: ImageCapture? = null + private var cameraTask: Runnable? = null + private var audioRecord: AudioRecord? = null + private var micThread: Thread? = null + private var mediaPlayer: MediaPlayer? = null + private var acousticEchoCanceler: AcousticEchoCanceler? = null + private var noiseSuppressor: NoiseSuppressor? = null + private var automaticGainControl: AutomaticGainControl? = null + private var realtimeUrl: String = "" + private var realtimeConnected = false + private var realtimeAudioTrack: AudioTrack? = null + private var realtimeAudioTrackSampleRate = MIC_SAMPLE_RATE + private var realtimeAudioTrackChannelCount = 1 + private var activationDevId: String = "" + private var activationUserId: String = "" + private var activationLicKey: String = "" + private var visualAppId: String = "" + private var visualCid: String = "" + private var visualToken: String = "" + private var visualContext = "" + private var realtimeLicensePassed = true + private var realtimeMediaReady = false + private var realtimeAsrEnabled = false + private var realtimeMicSuppressed = false + private var lastRealtimeBinaryLogAt = 0L + private var ttsBeginAt = 0L + private var bargeInConsecutiveFrames = 0 + private var lastBargeInAt = 0L + private var lastRealtimeAsrText = "" + private var lastRealtimeAsrEmitAt = 0L + private var remoteVoiceActive = false + private var dropRemoteAudioUntil = 0L + private var originalAudioMode = AudioManager.MODE_NORMAL + private var latestCameraImageBase64 = "" + private var latestCameraImageBytes = 0 + private var latestCameraImageWidth = 0 + private var latestCameraImageHeight = 0 + private var latestCameraImageAt = 0L + private var pendingRealtimeImageUpload = false + private var cameraCaptureInFlight = false + private var realtimeDeviceInfoSent = false + private var realtimeVisionPromptSent = false + private var lastVisionImageUploadAt = 0L + private var currentTtsPacketCount = 0 + private var currentTtsBytes = 0 + private var currentTtsFirstPacketAt = 0L + private var currentTtsLastPacketAt = 0L + private var reconnectRealtimePending = false + private var realtimeClosedByClient = false + private var lastRealtimeAsrEnableAt = 0L + private var realtimeAudioSendAfterAt = 0L + private var lastRealtimeAudioGateLogAt = 0L + private var lastRealtimeUplinkLogAt = 0L + private var realtimeUplinkFrames = 0 + private var realtimeUplinkBytes = 0 + private var realtimeUplinkFailures = 0 + private var realtimeUplinkFirstSendAt = 0L + private var realtimeUplinkLastSendAt = 0L + + fun start() { + if (running) return + running = true + configureCommunicationAudio() + onLog("Software flow starting") + initTextToSpeech() + bindCamera() + startCameraLoop() + speak("软件对话已开启") + } + + fun stop() { + if (!running) return + running = false + stopCameraLoop() + releaseSpeechRecognizer() + stopMicFallback() + releaseCamera() + disconnectRealtime() + releaseMediaPlayer() + releaseTextToSpeech() + restoreCommunicationAudio() + onLog("Software flow stopped") + } + + fun release() { + stop() + releaseRealtimeAudioTrack() + realtimeWsClient.release() + cameraExecutor.shutdown() + } + + fun connectRealtime(url: String) { + if (!running) return + realtimeUrl = url + if (speechRecognizer != null) { + releaseSpeechRecognizer() + } + startMicFallback() + realtimeAsrEnabled = false + realtimeMicSuppressed = false + ttsBeginAt = 0L + bargeInConsecutiveFrames = 0 + lastBargeInAt = 0L + lastRealtimeAsrText = "" + lastRealtimeAsrEmitAt = 0L + remoteVoiceActive = false + dropRemoteAudioUntil = 0L + realtimeDeviceInfoSent = false + realtimeVisionPromptSent = false + lastVisionImageUploadAt = 0L + resetRealtimeTtsStats() + reconnectRealtimePending = false + realtimeClosedByClient = false + lastRealtimeAsrEnableAt = 0L + realtimeAudioSendAfterAt = 0L + lastRealtimeAudioGateLogAt = 0L + resetRealtimeUplinkStats() + // Match the known-good 0.4.13/0.4.14 sequencing: + // always wait for LIC PASS before enabling realtime ASR. + realtimeLicensePassed = false + realtimeMediaReady = false + onLog("Realtime WS connecting...") + realtimeWsClient.connect(realtimeUrl) + } + + fun updateActivationInfo(devId: String, userId: String, licenseKey: String) { + activationDevId = devId.trim() + activationUserId = userId.trim() + activationLicKey = licenseKey.trim() + } + + fun updateBaiduVisualSessionInfo( + appId: String, + cid: String, + token: String, + contextJson: String, + deviceId: String, + appUserId: String, + licenseKey: String + ) { + visualAppId = appId.trim() + visualCid = cid.trim() + visualToken = token.trim() + visualContext = contextJson.trim() + baiduConversationAgent.updateSession( + appId = visualAppId, + cid = visualCid, + token = visualToken, + contextJson = visualContext, + deviceId = deviceId.trim(), + appUserId = appUserId.trim(), + licenseKey = licenseKey.trim() + ) + } + + fun disconnectRealtime() { + sendRealtimeAsrStop() + realtimeConnected = false + reconnectRealtimePending = false + realtimeClosedByClient = true + realtimeUrl = "" + realtimeLicensePassed = true + realtimeAsrEnabled = false + realtimeAudioSendAfterAt = 0L + lastRealtimeAudioGateLogAt = 0L + realtimeMicSuppressed = false + ttsBeginAt = 0L + bargeInConsecutiveFrames = 0 + lastRealtimeAsrText = "" + lastRealtimeAsrEmitAt = 0L + remoteVoiceActive = false + dropRemoteAudioUntil = 0L + realtimeDeviceInfoSent = false + realtimeVisionPromptSent = false + lastVisionImageUploadAt = 0L + resetRealtimeTtsStats() + resetRealtimeUplinkStats() + realtimeMediaReady = false + realtimeWsClient.disconnect("client_stop") + baiduConversationAgent.stop() + releaseRealtimeAudioTrack() + stopMicFallback() + } + + fun playServerAudio(audioBase64: String, audioUrl: String): Boolean { + if (!running) return false + if (audioBase64.isBlank() && audioUrl.isBlank()) return false + return if (audioBase64.isNotBlank()) { + playBase64Audio(audioBase64) + } else { + playUrlAudio(audioUrl) + } + } + + fun speak(text: String): Boolean { + if (!running || text.isBlank()) return false + onLog("Speak request: ${text.take(24)}") + if (ttsReady) { + val ok = runCatching { + textToSpeech?.speak( + text, + TextToSpeech.QUEUE_ADD, + null, + "soft-flow-${System.currentTimeMillis()}" + ) + }.isSuccess + if (ok) { + return true + } + onLog("TTS speak failed, fallback to tone") + return playFallbackTone() + } + return playFallbackTone() + } + + private fun initTextToSpeech() { + if (textToSpeech != null) return + textToSpeech = TextToSpeech(context) { status -> + if (status != TextToSpeech.SUCCESS) { + onLog("TTS init failed: status=$status") + initToneFallback() + return@TextToSpeech + } + val setLanguageResult = textToSpeech?.setLanguage(Locale.SIMPLIFIED_CHINESE) + ttsReady = setLanguageResult != TextToSpeech.LANG_NOT_SUPPORTED && + setLanguageResult != TextToSpeech.LANG_MISSING_DATA + if (!ttsReady) { + onLog("TTS language unsupported, fallback to default voice") + ttsReady = true + } + } + } + + private fun releaseTextToSpeech() { + runCatching { textToSpeech?.stop() } + runCatching { textToSpeech?.shutdown() } + textToSpeech = null + ttsReady = false + runCatching { toneGenerator?.release() } + toneGenerator = null + } + + private fun releaseMediaPlayer() { + runCatching { mediaPlayer?.stop() } + runCatching { mediaPlayer?.release() } + mediaPlayer = null + } + + private fun sendRealtimePcm(pcm: ByteArray) { + if (!running || !realtimeConnected || !realtimeAsrEnabled || realtimeMicSuppressed || pcm.isEmpty()) { + return + } + val now = SystemClock.elapsedRealtime() + if (now < realtimeAudioSendAfterAt) { + if (now - lastRealtimeAudioGateLogAt >= REALTIME_UPLINK_LOG_INTERVAL_MS) { + lastRealtimeAudioGateLogAt = now + onLog("Realtime PCM gated: wait=${realtimeAudioSendAfterAt - now}ms after ASR enable") + } + return + } + val ok = realtimeWsClient.sendBinary(pcm) + if (ok) { + realtimeUplinkFrames += 1 + realtimeUplinkBytes += pcm.size + realtimeUplinkLastSendAt = now + if (realtimeUplinkFirstSendAt == 0L) { + realtimeUplinkFirstSendAt = now + onLog("Realtime PCM first uplink: delay=${now - lastRealtimeAsrEnableAt}ms, bytes=${pcm.size}") + } + if (now - lastRealtimeUplinkLogAt >= REALTIME_UPLINK_LOG_INTERVAL_MS) { + lastRealtimeUplinkLogAt = now + onLog( + "Realtime PCM uplink: frames=$realtimeUplinkFrames, bytes=$realtimeUplinkBytes, failures=$realtimeUplinkFailures, lastOkAgo=${now - realtimeUplinkLastSendAt}ms" + ) + } + } else { + realtimeUplinkFailures += 1 + onLog("Realtime PCM send failed: bytes=${pcm.size}") + } + } + + private fun handleRealtimeText(text: String) { + if (text.contains("[E]:[VOICE_COMING]")) { + remoteVoiceActive = true + if (lastVisionImageUploadAt > 0L) { + val delay = SystemClock.elapsedRealtime() - lastVisionImageUploadAt + if (delay in 1..15_000L) { + onLog("Vision response voice started: delay=${delay}ms") + } + } + } + if (text.contains("[E]:[VOICE_DISAPPEAR]")) { + remoteVoiceActive = false + } + if (text.contains("[E]:[LIC]:[MUST]")) { + realtimeLicensePassed = false + realtimeAsrEnabled = false + sendRealtimeLicenseActive() + } + if (text.contains("[E]:[LIC]:[RES]:[PASS]")) { + realtimeLicensePassed = true + maybeEnableRealtimeAsr() + } + if (text.contains("[E]:[MEDIA]:[READY]")) { + realtimeMediaReady = true + maybeEnableRealtimeAsr() + } + if (text.contains("[E]:[TTS_BEGIN_SPEAKING]")) { + realtimeMicSuppressed = true + ttsBeginAt = SystemClock.elapsedRealtime() + bargeInConsecutiveFrames = 0 + remoteVoiceActive = true + resetRealtimeTtsStats() + onLog("Realtime MIC suppressed: TTS begin") + } + if (text.contains("[E]:[TTS_END_SPEAKING]")) { + realtimeMicSuppressed = false + bargeInConsecutiveFrames = 0 + remoteVoiceActive = false + logRealtimeTtsSummary() + onLog("Realtime MIC resumed: TTS end") + } + if (text.contains("[E]:[UPLOAD_IMAGE]")) { + handleRealtimeImageRequest() + } + if (text.startsWith("[E]:[PLAY_AUDIO]:")) { + val audioUrl = text.substringAfter("[E]:[PLAY_AUDIO]:").trim() + if (audioUrl.startsWith("http://") || audioUrl.startsWith("https://")) { + playUrlAudio(audioUrl) + } + } + if (text.startsWith("[Q]:")) { + val recognized = extractRealtimeTextPayload(text, "[Q]:") + if (recognized.isNotBlank() && !isRealtimeAsrPartial(text)) { + onLog("Realtime ASR: ${recognized.take(80)}") + maybeDispatchRealtimeAsr(recognized) + } + } + if (text.startsWith("[A]:")) { + val answer = extractRealtimeTextPayload(text, "[A]:") + if (answer.isNotBlank()) { + onLog("Realtime AI: ${answer.take(80)}") + } + } + } + + private fun isRealtimeAsrPartial(text: String): Boolean { + return text.startsWith("[Q]:[M]:") + } + + private fun maybeDispatchRealtimeAsr(text: String) { + val normalized = text.trim() + if (normalized.isBlank()) return + val now = SystemClock.elapsedRealtime() + if (normalized == lastRealtimeAsrText && now - lastRealtimeAsrEmitAt < 1500L) { + return + } + lastRealtimeAsrText = normalized + lastRealtimeAsrEmitAt = now + onRealtimeAsrText(normalized) + } + + private fun sendRealtimeLicenseActive() { + if (!running || !realtimeConnected) return + if (activationDevId.isBlank() || activationLicKey.isBlank()) { + onLog("Realtime license active skipped: activation info missing") + return + } + val payload = "[E]:[LIC]:[ACTIVE]:{\"devId\":\"${jsonEscape(activationDevId)}\",\"uId\":\"${jsonEscape(activationUserId.ifBlank { activationDevId })}\",\"licKey\":\"${jsonEscape(activationLicKey)}\"}" + val ok = realtimeWsClient.sendText(payload) + if (ok) { + onLog("Realtime license active sent") + } else { + onLog("Realtime license active send failed") + } + } + + private fun maybeEnableRealtimeAsr() { + if (!running || !realtimeConnected) return + if (!realtimeMediaReady) { + onLog("Realtime ASR enable pending: media not ready") + return + } + if (!realtimeLicensePassed) { + onLog("Realtime ASR enable pending: license not pass yet") + return + } + if (realtimeAsrEnabled) return + val now = SystemClock.elapsedRealtime() + if (now - lastRealtimeAsrEnableAt < REALTIME_ASR_RETRY_WINDOW_MS) return + lastRealtimeAsrEnableAt = now + val deviceInfoOk = sendRealtimeDeviceInfo() + val visionPromptOk = sendRealtimeVisionPrompt() + val autoInt = sendRealtimeCmd("[SET]:[AUTO_INT]:[TRUE]", "AUTO_INT") + val realtimeAsr = sendRealtimeCmd("[E]:[CMD]:[ASR_ENABLE_REALTIME]", "ASR_ENABLE_REALTIME") + val longText = sendRealtimeCmd("[E]:[CMD]:[ASR_START_LONGTEXT_REC]", "ASR_START_LONGTEXT_REC") + if (autoInt || realtimeAsr || longText) { + realtimeAsrEnabled = true + realtimeAudioSendAfterAt = SystemClock.elapsedRealtime() + REALTIME_ENABLE_AUDIO_GRACE_MS + lastRealtimeAudioGateLogAt = 0L + resetRealtimeUplinkStats() + onLog("Realtime ASR pipeline enabled: deviceInfo=$deviceInfoOk, visionPrompt=$visionPromptOk") + onLog("Realtime PCM uplink armed: grace=${REALTIME_ENABLE_AUDIO_GRACE_MS}ms") + } + } + + private fun sendRealtimeAsrStop() { + sendRealtimeCmd("[E]:[CMD]:[ASR_STOP_LONGTEXT_REC]", "ASR_STOP_LONGTEXT_REC") + realtimeAsrEnabled = false + realtimeMicSuppressed = false + remoteVoiceActive = false + } + + private fun sendRealtimeCmd(cmd: String, tag: String): Boolean { + if (!running || !realtimeConnected) return false + val ok = realtimeWsClient.sendText(cmd) + onLog( + if (ok) "Realtime CMD sent: $tag" else "Realtime CMD failed: $tag" + ) + return ok + } + + private fun extractRealtimeTextPayload(raw: String, prefix: String): String { + if (!raw.startsWith(prefix)) return "" + var text = raw.removePrefix(prefix).trim() + while (text.startsWith("[") && text.contains("]:")) { + text = text.substringAfter("]:").trimStart(':').trim() + } + text = text.substringBefore("|||").trim() + if (text.isBlank()) return "" + return text + } + + private fun jsonEscape(value: String): String { + return value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + } + + private fun sendRealtimeDeviceInfo(): Boolean { + if (realtimeDeviceInfoSent) return true + realtimeDeviceInfoSent = true + return true + } + + private fun sendRealtimeVisionPrompt(): Boolean { + if (realtimeVisionPromptSent) return true + if (!running || !realtimeConnected) return false + val payload = "[SET]:[UPDATE_SYSTEM_PROMPT]:{\"model_type\":\"2\",\"prompt\":\"${jsonEscape(VISION_SYSTEM_PROMPT)}\"}" + val ok = realtimeWsClient.sendText(payload) + if (ok) { + realtimeVisionPromptSent = true + onLog("Realtime vision prompt sent") + } else { + onLog("Realtime vision prompt failed") + } + return ok + } + + private fun handleRealtimeImageRequest() { + if (!running || !realtimeConnected) return + if (!realtimeVisionPromptSent) { + sendRealtimeVisionPrompt() + } + pendingRealtimeImageUpload = true + onLog("Realtime image request received") + val now = System.currentTimeMillis() + val hasFreshFrame = latestCameraImageBase64.isNotBlank() && + latestCameraImageBytes > 0 && + now - latestCameraImageAt <= REALTIME_IMAGE_FRESH_MS + if (hasFreshFrame) { + uploadLatestRealtimeImage(source = "fresh-cache") + return + } + onLog("Realtime image request: capturing fresh frame") + mainHandler.post { captureOnce() } + } + + private fun uploadLatestRealtimeImage(source: String) { + if (!running || !realtimeConnected) return + if (!realtimeVisionPromptSent) { + sendRealtimeVisionPrompt() + } + if (latestCameraImageBase64.isBlank() || latestCameraImageBytes <= 0) { + onLog("Realtime image upload skipped: no camera frame") + return + } + val now = System.currentTimeMillis() + if (now - latestCameraImageAt > REALTIME_IMAGE_STALE_MS) { + onLog("Realtime image upload skipped: stale frame") + return + } + val uploadImage = prepareRealtimeImagePayload() + if (uploadImage == null) { + onLog("Realtime image upload skipped: image prep failed") + return + } + pendingRealtimeImageUpload = false + lastVisionImageUploadAt = SystemClock.elapsedRealtime() + scheduleVisionResponseWatchdog() + val imageBase64 = Base64.encodeToString(uploadImage.bytes, Base64.NO_WRAP) + onLog( + "Realtime image prepared($source): ${uploadImage.width}x${uploadImage.height}, jpeg=${uploadImage.bytes.size}, b64=${imageBase64.length}" + ) + mainHandler.post { + if (!running) return@post + onRealtimeVisionFrameReady( + imageBase64, + uploadImage.bytes.size, + uploadImage.width, + uploadImage.height, + source, + ) + } + } + + private fun playRealtimePcm(pcm: ByteArray, sampleRate: Int, channelCount: Int) { + if (pcm.isEmpty()) return + val track = if ( + realtimeAudioTrack == null || + realtimeAudioTrackSampleRate != sampleRate || + realtimeAudioTrackChannelCount != channelCount + ) { + releaseRealtimeAudioTrack() + buildRealtimeAudioTrack(sampleRate, channelCount) + } else { + realtimeAudioTrack + } ?: return + val written = runCatching { + track.write(pcm, 0, pcm.size, AudioTrack.WRITE_NON_BLOCKING) + }.getOrElse { + onLog("Realtime audio write failed: ${it.message}") + -1 + } + if (written > 0 && track.playState != AudioTrack.PLAYSTATE_PLAYING) { + runCatching { track.play() } + } + } + + private fun buildRealtimeAudioTrack(sampleRate: Int, channelCount: Int): AudioTrack? { + val channelMask = if (channelCount <= 1) { + AudioFormat.CHANNEL_OUT_MONO + } else { + AudioFormat.CHANNEL_OUT_STEREO + } + val minBuffer = AudioTrack.getMinBufferSize( + sampleRate, + channelMask, + AudioFormat.ENCODING_PCM_16BIT + ) + if (minBuffer <= 0) { + onLog("Realtime audio track init failed: minBuffer=$minBuffer") + return null + } + val track = runCatching { + AudioTrack( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + AudioFormat.Builder() + .setSampleRate(sampleRate) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setChannelMask(channelMask) + .build(), + minBuffer * 4, + AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE + ) + }.getOrElse { + onLog("Realtime audio track init failed: ${it.message}") + return null + } + realtimeAudioTrack = track + realtimeAudioTrackSampleRate = sampleRate + realtimeAudioTrackChannelCount = channelCount + runCatching { track.play() } + onLog("Realtime audio track ready: sampleRate=$sampleRate, channels=$channelCount") + return track + } + + private fun releaseRealtimeAudioTrack() { + runCatching { realtimeAudioTrack?.stop() } + runCatching { realtimeAudioTrack?.release() } + realtimeAudioTrack = null + realtimeAudioTrackSampleRate = MIC_SAMPLE_RATE + realtimeAudioTrackChannelCount = 1 + remoteVoiceActive = false + } + + private fun initSpeechRecognizer() { + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + onLog("SpeechRecognizer unavailable on this device") + onLog("Speech fallback: raw microphone capture only (no STT)") + startMicFallback() + return + } + if (speechRecognizer == null) { + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context).apply { + setRecognitionListener(recognitionListener) + } + } + startListeningDelayed(350L) + } + + private fun releaseSpeechRecognizer() { + mainHandler.removeCallbacksAndMessages(SPEECH_TOKEN) + runCatching { speechRecognizer?.stopListening() } + runCatching { speechRecognizer?.cancel() } + runCatching { speechRecognizer?.destroy() } + speechRecognizer = null + } + + private fun initToneFallback() { + if (toneGenerator != null) return + val vol = audioManager.getStreamVolume(AudioManager.STREAM_MUSIC) + val max = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC) + onLog("Audio stream volume: music=$vol/$max") + toneGenerator = runCatching { ToneGenerator(AudioManager.STREAM_MUSIC, 100) } + .onFailure { onLog("Tone fallback init failed: ${it.message}") } + .getOrNull() + } + + private fun playFallbackTone(): Boolean { + if (!running) return false + if (toneGenerator == null) initToneFallback() + val toneOk = runCatching { + toneGenerator?.startTone(ToneGenerator.TONE_PROP_BEEP2, 260) ?: false + }.getOrDefault(false) + if (toneOk) { + onLog("Tone fallback played") + return true + } + onLog("Tone fallback failed, trying AudioTrack beep") + return playPcmBeep() + } + + private fun playPcmBeep(): Boolean { + return runCatching { + val durationMs = 280 + val totalSamples = BEEP_SAMPLE_RATE * durationMs / 1000 + val pcm = ShortArray(totalSamples) + val freqHz = 880.0 + for (i in 0 until totalSamples) { + val t = i.toDouble() / BEEP_SAMPLE_RATE + val amp = 0.32 * sin(2.0 * PI * freqHz * t) + pcm[i] = (amp * Short.MAX_VALUE).toInt().toShort() + } + val track = AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANCE_SONIFICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION) + .build() + ) + .setAudioFormat( + AudioFormat.Builder() + .setSampleRate(BEEP_SAMPLE_RATE) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build() + ) + .setTransferMode(AudioTrack.MODE_STATIC) + .setBufferSizeInBytes(pcm.size * 2) + .build() + track.write(pcm, 0, pcm.size) + track.play() + Thread.sleep((durationMs + 60).toLong()) + track.stop() + track.release() + onLog("AudioTrack beep played") + true + }.onFailure { + onLog("AudioTrack beep failed: ${it.message}") + }.getOrDefault(false) + } + + private fun playBase64Audio(audioBase64: String): Boolean { + return runCatching { + val bytes = Base64.decode(audioBase64, Base64.DEFAULT) + if (bytes.isEmpty()) return@runCatching false + val ext = when { + bytes.size >= 4 && + bytes[0] == 'R'.code.toByte() && + bytes[1] == 'I'.code.toByte() && + bytes[2] == 'F'.code.toByte() && + bytes[3] == 'F'.code.toByte() -> ".wav" + bytes.size >= 3 && + bytes[0] == 'I'.code.toByte() && + bytes[1] == 'D'.code.toByte() && + bytes[2] == '3'.code.toByte() -> ".mp3" + else -> ".bin" + } + val tmp = File.createTempFile("tts_", ext, context.cacheDir) + tmp.writeBytes(bytes) + playFileAudio(tmp) + }.onFailure { + onLog("Play base64 audio failed: ${it.message}") + }.getOrDefault(false) + } + + private fun playUrlAudio(url: String): Boolean { + return runCatching { + if (url.isBlank()) return@runCatching false + releaseMediaPlayer() + val p = MediaPlayer().apply { + setAudioStreamType(AudioManager.STREAM_MUSIC) + setDataSource(url) + setOnPreparedListener { it.start() } + setOnCompletionListener { + onLog("Server audio played(url)") + releaseMediaPlayer() + } + setOnErrorListener { _, what, extra -> + onLog("Server audio play error(url): what=$what extra=$extra") + releaseMediaPlayer() + true + } + prepareAsync() + } + mediaPlayer = p + onLog("Server audio loading(url)") + true + }.onFailure { + onLog("Play url audio failed: ${it.message}") + }.getOrDefault(false) + } + + private fun playFileAudio(file: File): Boolean { + return runCatching { + releaseMediaPlayer() + val p = MediaPlayer().apply { + setAudioStreamType(AudioManager.STREAM_MUSIC) + setDataSource(file.absolutePath) + setOnPreparedListener { it.start() } + setOnCompletionListener { + onLog("Server audio played(base64)") + runCatching { file.delete() } + releaseMediaPlayer() + } + setOnErrorListener { _, what, extra -> + onLog("Server audio play error(base64): what=$what extra=$extra") + runCatching { file.delete() } + releaseMediaPlayer() + true + } + prepareAsync() + } + mediaPlayer = p + onLog("Server audio loading(base64, ${file.length()} bytes)") + true + }.onFailure { + runCatching { file.delete() } + onLog("Play file audio failed: ${it.message}") + }.getOrDefault(false) + } + + private fun startMicFallback() { + if (audioRecord != null) return + val minBuffer = AudioRecord.getMinBufferSize( + MIC_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ) + if (minBuffer <= 0) { + onLog("Mic fallback init failed: bad buffer size=$minBuffer") + return + } + val bufferSize = maxOf(minBuffer * 2, MIC_SAMPLE_RATE) + val record = buildFallbackAudioRecord(bufferSize) ?: return + if (record.state != AudioRecord.STATE_INITIALIZED) { + onLog("Mic fallback init failed: state=${record.state}") + runCatching { record.release() } + return + } + audioRecord = record + enableAudioEffects(record.audioSessionId) + runCatching { record.startRecording() } + .onFailure { + onLog("Mic fallback start failed: ${it.message}") + releaseAudioEffects() + runCatching { record.release() } + audioRecord = null + return + } + onLog("Mic fallback recording started") + micThread = Thread { + val frame = ByteArray(REALTIME_PCM_FRAME_BYTES) + var lastLogAt = 0L + while (running && audioRecord === record) { + val read = record.read(frame, 0, frame.size) + if (read <= 0) { + if (read < 0) { + onLog("Mic fallback read error=$read") + } + continue + } + val rms = calculateRms(frame, read) + val pcmCopy = frame.copyOf(read) + mainHandler.post { + if (running) { + onMicFallbackFrame(pcmCopy, rms, MIC_SAMPLE_RATE) + } + } + maybeHandleBargeIn(rms) + sendRealtimePcm(pcmCopy) + val now = SystemClock.elapsedRealtime() + if (now - lastLogAt >= MIC_LOG_INTERVAL_MS) { + lastLogAt = now + onLog("Mic fallback active: bytes=$read, rms=$rms") + } + } + }.apply { + name = "software-mic-fallback" + isDaemon = true + start() + } + } + + private fun maybeHandleBargeIn(rms: Int) { + if (!running || !realtimeConnected || !realtimeAsrEnabled) return + if (!realtimeMicSuppressed && !remoteVoiceActive) return + val now = SystemClock.elapsedRealtime() + if (realtimeMicSuppressed && now - ttsBeginAt < BARGE_IN_MIN_AFTER_TTS_MS) return + if (now - lastBargeInAt < BARGE_IN_COOLDOWN_MS) return + val rmsThreshold = if (remoteVoiceActive) BARGE_IN_REMOTE_RMS_THRESHOLD else BARGE_IN_RMS_THRESHOLD + if (rms >= rmsThreshold) { + bargeInConsecutiveFrames += 1 + } else { + bargeInConsecutiveFrames = 0 + return + } + if (bargeInConsecutiveFrames < BARGE_IN_CONSECUTIVE_FRAMES) return + bargeInConsecutiveFrames = 0 + lastBargeInAt = now + realtimeMicSuppressed = false + remoteVoiceActive = false + dropRemoteAudioUntil = now + REMOTE_AUDIO_DROP_AFTER_BREAK_MS + sendRealtimeBreakWithWindow() + stopRealtimePlaybackForBargeIn() + onLog("Barge-in detected: rms=$rms") + mainHandler.post { + if (running) { + onBargeInDetected() + } + } + } + + private fun sendRealtimeBreakWithWindow() { + onLog("Realtime interrupt sent") + baiduConversationAgent.interrupt() + } + + private fun stopRealtimePlaybackForBargeIn() { + runCatching { mediaPlayer?.stop() } + runCatching { mediaPlayer?.release() } + mediaPlayer = null + runCatching { realtimeAudioTrack?.pause() } + runCatching { realtimeAudioTrack?.flush() } + runCatching { realtimeAudioTrack?.play() } + } + + private fun stopMicFallback() { + val record = audioRecord + audioRecord = null + micThread?.interrupt() + micThread = null + releaseAudioEffects() + if (record != null) { + runCatching { record.stop() } + runCatching { record.release() } + } + } + + private fun buildFallbackAudioRecord(bufferSize: Int): AudioRecord? { + val sources = listOf( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + MediaRecorder.AudioSource.MIC + ) + for (source in sources) { + val record = runCatching { + AudioRecord( + source, + MIC_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + bufferSize + ) + }.getOrNull() ?: continue + if (record.state == AudioRecord.STATE_INITIALIZED) { + onLog("Mic fallback source selected: $source") + return record + } + runCatching { record.release() } + } + onLog("Mic fallback init failed: no usable audio source") + return null + } + + private fun enableAudioEffects(audioSessionId: Int) { + releaseAudioEffects() + acousticEchoCanceler = runCatching { + if (AcousticEchoCanceler.isAvailable()) { + AcousticEchoCanceler.create(audioSessionId)?.apply { enabled = true } + } else { + null + } + }.getOrNull() + noiseSuppressor = runCatching { + if (NoiseSuppressor.isAvailable()) { + NoiseSuppressor.create(audioSessionId)?.apply { enabled = true } + } else { + null + } + }.getOrNull() + automaticGainControl = runCatching { + if (AutomaticGainControl.isAvailable()) { + AutomaticGainControl.create(audioSessionId)?.apply { enabled = true } + } else { + null + } + }.getOrNull() + onLog( + "Mic audio effects: aec=${acousticEchoCanceler != null}, ns=${noiseSuppressor != null}, agc=${automaticGainControl != null}" + ) + } + + private fun releaseAudioEffects() { + runCatching { acousticEchoCanceler?.release() } + runCatching { noiseSuppressor?.release() } + runCatching { automaticGainControl?.release() } + acousticEchoCanceler = null + noiseSuppressor = null + automaticGainControl = null + } + + private fun configureCommunicationAudio() { + originalAudioMode = audioManager.mode + if (audioManager.mode != AudioManager.MODE_IN_COMMUNICATION) { + audioManager.mode = AudioManager.MODE_IN_COMMUNICATION + onLog("Audio mode -> MODE_IN_COMMUNICATION") + } + } + + private fun restoreCommunicationAudio() { + if (audioManager.mode != originalAudioMode) { + audioManager.mode = originalAudioMode + onLog("Audio mode restored: $originalAudioMode") + } + } + + private fun onRealtimeBinary(now: Long, size: Int) { + if (ttsBeginAt <= 0L) return + currentTtsPacketCount += 1 + currentTtsBytes += size + currentTtsLastPacketAt = now + if (currentTtsFirstPacketAt == 0L) { + currentTtsFirstPacketAt = now + onLog("Realtime TTS first audio: delay=${now - ttsBeginAt}ms, bytes=$size") + } + if (size != TTS_BINARY_LOG_SAMPLE_BYTES && currentTtsPacketCount == 1) { + onLog("Realtime TTS first packet size=$size") + } + } + + private fun resetRealtimeTtsStats() { + currentTtsPacketCount = 0 + currentTtsBytes = 0 + currentTtsFirstPacketAt = 0L + currentTtsLastPacketAt = 0L + } + + private fun resetRealtimeUplinkStats() { + realtimeUplinkFrames = 0 + realtimeUplinkBytes = 0 + realtimeUplinkFailures = 0 + realtimeUplinkFirstSendAt = 0L + realtimeUplinkLastSendAt = 0L + lastRealtimeUplinkLogAt = 0L + } + + private fun logRealtimeTtsSummary() { + if (ttsBeginAt <= 0L) return + val now = SystemClock.elapsedRealtime() + val firstDelay = if (currentTtsFirstPacketAt > 0L) currentTtsFirstPacketAt - ttsBeginAt else -1L + val playDuration = if (currentTtsLastPacketAt > 0L) currentTtsLastPacketAt - ttsBeginAt else 0L + onLog( + "Realtime TTS summary: firstDelay=${firstDelay}ms, playDuration=${playDuration}ms, packets=$currentTtsPacketCount, bytes=$currentTtsBytes, endGap=${now - (currentTtsLastPacketAt.takeIf { it > 0L } ?: now)}ms" + ) + ttsBeginAt = 0L + resetRealtimeTtsStats() + } + + private fun maybeReconnectRealtime(reason: String) { + if (!running || realtimeClosedByClient || realtimeUrl.isBlank()) return + if (reconnectRealtimePending) return + reconnectRealtimePending = true + onLog("Realtime reconnect scheduled: $reason") + mainHandler.postDelayed( + { + reconnectRealtimePending = false + if (!running || realtimeUrl.isBlank()) return@postDelayed + onLog("Realtime reconnecting...") + connectRealtime(realtimeUrl) + }, + 900L + ) + } + + private fun scheduleVisionResponseWatchdog() { + val expectedAt = lastVisionImageUploadAt + mainHandler.postDelayed( + { + if (!running || !realtimeConnected) return@postDelayed + if (expectedAt != lastVisionImageUploadAt) return@postDelayed + val idleFor = SystemClock.elapsedRealtime() - expectedAt + if (idleFor >= VISION_RESPONSE_TIMEOUT_MS && !remoteVoiceActive) { + onLog("Vision response pending: no voice reply ${idleFor}ms after image upload") + } + }, + VISION_RESPONSE_TIMEOUT_MS + ) + } + + private data class PreparedRealtimeImage( + val bytes: ByteArray, + val width: Int, + val height: Int + ) + + private fun prepareRealtimeImagePayload(): PreparedRealtimeImage? { + val rawBase64 = latestCameraImageBase64 + if (rawBase64.isBlank()) return null + val rawBytes = runCatching { Base64.decode(rawBase64, Base64.DEFAULT) }.getOrNull() ?: return null + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(rawBytes, 0, rawBytes.size, bounds) + val srcWidth = bounds.outWidth.takeIf { it > 0 } ?: latestCameraImageWidth + val srcHeight = bounds.outHeight.takeIf { it > 0 } ?: latestCameraImageHeight + if (rawBytes.size <= REALTIME_IMAGE_UPLOAD_TARGET_BYTES && srcWidth <= REALTIME_IMAGE_UPLOAD_MAX_DIM) { + return PreparedRealtimeImage( + bytes = rawBytes, + width = srcWidth, + height = srcHeight + ) + } + val bitmap = BitmapFactory.decodeByteArray(rawBytes, 0, rawBytes.size) ?: return null + val scaled = scaleBitmapForRealtime(bitmap) + if (scaled !== bitmap) { + bitmap.recycle() + } + var quality = 58 + var encoded = encodeBitmapJpeg(scaled, quality) + while (encoded.size > REALTIME_IMAGE_UPLOAD_TARGET_BYTES && quality > REALTIME_IMAGE_UPLOAD_MIN_QUALITY) { + quality -= 6 + encoded = encodeBitmapJpeg(scaled, quality) + } + val width = scaled.width + val height = scaled.height + scaled.recycle() + return PreparedRealtimeImage( + bytes = encoded, + width = width, + height = height + ) + } + + private fun scaleBitmapForRealtime(bitmap: Bitmap): Bitmap { + val maxDim = maxOf(bitmap.width, bitmap.height) + if (maxDim <= REALTIME_IMAGE_UPLOAD_MAX_DIM) return bitmap + val ratio = REALTIME_IMAGE_UPLOAD_MAX_DIM.toFloat() / maxDim.toFloat() + val targetWidth = (bitmap.width * ratio).toInt().coerceAtLeast(1) + val targetHeight = (bitmap.height * ratio).toInt().coerceAtLeast(1) + return Bitmap.createScaledBitmap(bitmap, targetWidth, targetHeight, true) + } + + private fun encodeBitmapJpeg(bitmap: Bitmap, quality: Int): ByteArray { + val out = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, quality, out) + return out.toByteArray() + } + + private fun calculateRms(buffer: ByteArray, length: Int): Int { + if (length < 2) return 0 + var sum = 0L + var count = 0 + var i = 0 + while (i + 1 < length) { + val low = buffer[i].toInt() and 0xFF + val high = buffer[i + 1].toInt() + val sample = ((high shl 8) or low).toShort().toInt() + sum += kotlin.math.abs(sample) + count += 1 + i += 2 + } + if (count == 0) return 0 + return (sum / count).toInt() + } + + private fun buildSpeechIntent(): Intent { + return Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.SIMPLIFIED_CHINESE.toLanguageTag()) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, false) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1) + } + } + + private fun startListeningDelayed(delayMs: Long) { + if (!running) return + mainHandler.postAtTime( + { + if (!running) return@postAtTime + runCatching { + speechRecognizer?.cancel() + speechRecognizer?.startListening(buildSpeechIntent()) + }.onFailure { + onLog("Speech start failed: ${it.message}") + } + }, + SPEECH_TOKEN, + System.currentTimeMillis() + delayMs + ) + } + + private fun bindCamera() { + val future = ProcessCameraProvider.getInstance(context) + future.addListener( + { + if (!running) return@addListener + val provider = runCatching { future.get() }.getOrElse { + onLog("Camera provider failed: ${it.message}") + return@addListener + } + val capture = ImageCapture.Builder() + .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) + .setTargetResolution(Size(CAMERA_TARGET_WIDTH, CAMERA_TARGET_HEIGHT)) + .setJpegQuality(CAMERA_JPEG_QUALITY) + .build() + val selector = chooseCamera(provider) ?: run { + onLog("No usable camera found") + return@addListener + } + runCatching { + provider.unbindAll() + provider.bindToLifecycle(lifecycleOwner, selector, capture) + }.onSuccess { + cameraProvider = provider + imageCapture = capture + onLog("Camera bound for software flow") + }.onFailure { + onLog("Camera bind failed: ${it.message}") + } + }, + ContextCompat.getMainExecutor(context) + ) + } + + private fun chooseCamera(provider: ProcessCameraProvider): CameraSelector? { + return runCatching { + when { + provider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA) -> + CameraSelector.DEFAULT_BACK_CAMERA + provider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA) -> + CameraSelector.DEFAULT_FRONT_CAMERA + else -> null + } + }.getOrNull() + } + + private fun releaseCamera() { + runCatching { cameraProvider?.unbindAll() } + imageCapture = null + cameraProvider = null + latestCameraImageBase64 = "" + latestCameraImageBytes = 0 + latestCameraImageWidth = 0 + latestCameraImageHeight = 0 + latestCameraImageAt = 0L + pendingRealtimeImageUpload = false + cameraCaptureInFlight = false + } + + private fun startCameraLoop() { + stopCameraLoop() + val task = object : Runnable { + override fun run() { + if (!running) return + captureOnce() + mainHandler.postDelayed(this, CAMERA_INTERVAL_MS) + } + } + cameraTask = task + mainHandler.postDelayed(task, FIRST_CAPTURE_DELAY_MS) + } + + private fun stopCameraLoop() { + cameraTask?.let { mainHandler.removeCallbacks(it) } + cameraTask = null + } + + private fun captureOnce() { + val capture = imageCapture ?: return + if (cameraCaptureInFlight) return + cameraCaptureInFlight = true + val output = runCatching { + File.createTempFile("software_cam_", ".jpg", context.cacheDir) + }.getOrElse { + cameraCaptureInFlight = false + onLog("Camera capture skipped: ${it.message}") + return + } + val options = ImageCapture.OutputFileOptions.Builder(output).build() + capture.takePicture( + options, + cameraExecutor, + object : ImageCapture.OnImageSavedCallback { + override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { + cameraCaptureInFlight = false + val raw = runCatching { output.readBytes() } + .onFailure { onLog("Camera read failed: ${it.message}") } + .getOrNull() + if (raw == null || raw.isEmpty()) { + runCatching { output.delete() } + return + } + val opts = BitmapFactory.Options().apply { inJustDecodeBounds = true } + runCatching { BitmapFactory.decodeFile(output.absolutePath, opts) } + val width = opts.outWidth.coerceAtLeast(0) + val height = opts.outHeight.coerceAtLeast(0) + val base64 = Base64.encodeToString(raw, Base64.NO_WRAP) + latestCameraImageBase64 = base64 + latestCameraImageBytes = raw.size + latestCameraImageWidth = width + latestCameraImageHeight = height + latestCameraImageAt = System.currentTimeMillis() + runCatching { output.delete() } + mainHandler.post { + if (running) { + onCameraCapture(base64, raw.size, width, height) + if (pendingRealtimeImageUpload) { + uploadLatestRealtimeImage(source = "fresh-capture") + } + } + } + } + + override fun onError(exception: ImageCaptureException) { + cameraCaptureInFlight = false + runCatching { output.delete() } + onLog("Camera capture failed: ${exception.message}") + } + } + ) + } + + private val recognitionListener = object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) = Unit + + override fun onBeginningOfSpeech() = Unit + + override fun onRmsChanged(rmsdB: Float) = Unit + + override fun onBufferReceived(buffer: ByteArray?) = Unit + + override fun onEndOfSpeech() = Unit + + override fun onError(error: Int) { + if (!running) return + if (error != SpeechRecognizer.ERROR_NO_MATCH && error != SpeechRecognizer.ERROR_SPEECH_TIMEOUT) { + onLog("Speech recognition error=$error") + } + startListeningDelayed(600L) + } + + override fun onResults(results: Bundle?) { + if (!running) return + val text = results + ?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + ?.firstOrNull() + ?.trim() + .orEmpty() + if (text.isNotBlank()) { + onSpeechText(text) + } + startListeningDelayed(450L) + } + + override fun onPartialResults(partialResults: Bundle?) = Unit + + override fun onEvent(eventType: Int, params: Bundle?) = Unit + } + + private companion object { + val SPEECH_TOKEN = Any() + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeApiService.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeApiService.kt new file mode 100644 index 0000000..a57b060 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeApiService.kt @@ -0,0 +1,106 @@ +package com.aiglasses.app.storyforge + +import okhttp3.MultipartBody +import okhttp3.RequestBody +import retrofit2.http.Body +import retrofit2.http.GET +import retrofit2.http.Multipart +import retrofit2.http.PATCH +import retrofit2.http.POST +import retrofit2.http.Part +import retrofit2.http.Path +import retrofit2.http.Query + +interface StoryForgeApiService { + @POST("v2/auth/register") + suspend fun register(@Body request: RegisterAccountRequest): AccountDto + + @POST("v2/auth/login") + suspend fun login(@Body request: LoginRequest): AuthResponseDto + + @POST("v2/auth/logout") + suspend fun logout(): Map + + @GET("v2/me") + suspend fun me(): AccountDto + + @GET("v2/me/dashboard") + suspend fun dashboard(): DashboardDto + + @GET("v2/model-profiles") + suspend fun modelProfiles(): List + + @POST("v2/model-profiles") + suspend fun createModelProfile(@Body request: ModelProfileRequest): ModelProfileDto + + @POST("v2/me/preferences/analysis-model") + suspend fun setPreferredAnalysisModel(@Body request: PreferredModelRequest): AccountDto + + @GET("v2/knowledge-bases") + suspend fun knowledgeBases(): List + + @POST("v2/knowledge-bases") + suspend fun createKnowledgeBase(@Body request: KnowledgeBaseCreateRequest): KnowledgeBaseDto + + @GET("v2/knowledge-bases/{knowledgeBaseId}/documents") + suspend fun knowledgeDocuments(@Path("knowledgeBaseId") knowledgeBaseId: String): List + + @GET("v2/explore/jobs") + suspend fun jobs(): List + + @GET("v2/explore/jobs/{jobId}") + suspend fun job(@Path("jobId") jobId: String): JobDto + + @POST("v2/explore/video-link") + suspend fun createVideoLinkJob(@Body request: ExploreVideoLinkRequest): JobDto + + @POST("v2/explore/text") + suspend fun createTextJob(@Body request: ExploreTextRequest): JobDto + + @Multipart + @POST("v2/explore/upload-video") + suspend fun uploadVideo( + @Part file: MultipartBody.Part, + @Part("title") title: RequestBody, + @Part("knowledge_base_id") knowledgeBaseId: RequestBody, + @Part("assistant_id") assistantId: RequestBody, + @Part("analysis_model_profile_id") analysisModelProfileId: RequestBody + ): JobDto + + @GET("v2/assistants") + suspend fun assistants(): List + + @POST("v2/assistants") + suspend fun createAssistant(@Body request: AssistantCreateRequest): AssistantDto + + @PATCH("v2/assistants/{assistantId}") + suspend fun updateAssistant( + @Path("assistantId") assistantId: String, + @Body request: AssistantUpdateRequest + ): AssistantDto + + @POST("v2/assistants/{assistantId}/generate") + suspend fun generateCopy( + @Path("assistantId") assistantId: String, + @Body request: GenerateCopyRequest + ): GenerateCopyResponseDto + + @GET("v2/admin/accounts/pending") + suspend fun pendingAccounts(): List + + @POST("v2/admin/accounts/{accountId}/approve") + suspend fun approveAccount(@Path("accountId") accountId: String): ApprovalDecisionDto + + @POST("v2/admin/accounts/{accountId}/reject") + suspend fun rejectAccount(@Path("accountId") accountId: String): ApprovalDecisionDto + + @GET("api/v1/app/update/latest") + suspend fun latestUpdate( + @Query("platform") platform: String = "android", + @Query("channel") channel: String = "stable", + @Query("currentVersionCode") currentVersionCode: Int? = null + ): AppUpdateLatestDto + + @POST("v2/admin/app/update/publish") + suspend fun publishAppUpdate(@Body request: PublishAppUpdateRequest): PublishAppUpdateResponseDto +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeModels.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeModels.kt new file mode 100644 index 0000000..663c0cf --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeModels.kt @@ -0,0 +1,249 @@ +package com.aiglasses.app.storyforge + +import kotlinx.serialization.Serializable + +@Serializable +data class RegisterAccountRequest( + val username: String, + val password: String, + val display_name: String +) + +@Serializable +data class LoginRequest( + val username: String, + val password: String +) + +@Serializable +data class AccountDto( + val id: String, + val username: String, + val display_name: String, + val role: String, + val approval_status: String, + val approved_by: String? = null, + val approved_at: String? = null, + val preferred_analysis_model_id: String = "", + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class AuthResponseDto( + val token: String, + val account: AccountDto, + val default_external_base_url: String = "" +) + +@Serializable +data class ModelProfileDto( + val id: String, + val owner_account_id: String? = null, + val name: String, + val provider: String, + val base_url: String, + val api_key_masked: String = "", + val model_name: String, + val is_system: Boolean = false, + val is_default: Boolean = false, + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class ModelProfileRequest( + val name: String, + val base_url: String, + val api_key: String, + val model_name: String, + val is_default: Boolean = false +) + +@Serializable +data class PreferredModelRequest( + val model_profile_id: String +) + +@Serializable +data class KnowledgeBaseDto( + val id: String, + val user_id: String, + val name: String, + val description: String = "", + val fastgpt_dataset_id: String? = null, + val sync_status: String = "pending", + val document_count: Int = 0, + val linked_assistant_count: Int = 0, + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class KnowledgeBaseCreateRequest( + val name: String, + val description: String = "" +) + +@Serializable +data class AssistantDto( + val id: String, + val user_id: String, + val name: String, + val description: String = "", + val system_prompt: String = "", + val generation_goal: String = "", + val knowledge_base_ids: List = emptyList(), + val fastgpt_app_key: String = "", + val model_profile_id: String = "", + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class AssistantCreateRequest( + val name: String, + val description: String = "", + val system_prompt: String = "", + val generation_goal: String = "", + val knowledge_base_ids: List = emptyList(), + val fastgpt_app_key: String = "", + val model_profile_id: String = "" +) + +@Serializable +data class AssistantUpdateRequest( + val name: String? = null, + val description: String? = null, + val system_prompt: String? = null, + val generation_goal: String? = null, + val knowledge_base_ids: List? = null, + val fastgpt_app_key: String? = null, + val model_profile_id: String? = null +) + +@Serializable +data class ExploreVideoLinkRequest( + val video_url: String, + val title: String? = null, + val knowledge_base_id: String? = null, + val assistant_id: String? = null, + val analysis_model_profile_id: String? = null, + val language: String = "auto" +) + +@Serializable +data class ExploreTextRequest( + val title: String, + val content: String, + val knowledge_base_id: String? = null, + val assistant_id: String? = null, + val analysis_model_profile_id: String? = null +) + +@Serializable +data class JobDto( + val id: String, + val user_id: String, + val assistant_id: String? = null, + val knowledge_base_id: String, + val source_type: String, + val source_url: String? = null, + val title: String, + val language: String, + val status: String, + val transcript_text: String = "", + val style_summary: String = "", + val fastgpt_collection_id: String = "", + val upload_status: String = "pending", + val error: String = "", + val artifacts: Map = emptyMap(), + val analysis_model_profile_id: String = "", + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class KnowledgeDocumentDto( + val id: String, + val knowledge_base_id: String, + val title: String, + val source_type: String, + val source_url: String = "", + val transcript_text: String = "", + val style_summary: String = "", + val combined_text: String = "", + val fastgpt_collection_id: String = "", + val analysis_model_profile_id: String = "", + val created_at: String = "", + val updated_at: String = "" +) + +@Serializable +data class GenerateCopyRequest( + val brief: String, + val platform: String = "抖音", + val audience: String = "创业者", + val extra_requirements: String = "", + val knowledge_base_ids: List = emptyList() +) + +@Serializable +data class GenerateCopyResponseDto( + val assistant_id: String, + val knowledge_base_ids: List, + val content: String, + val prompt_excerpt: String, + val used_documents: List = emptyList() +) + +@Serializable +data class DashboardDto( + val account: AccountDto, + val knowledge_bases: List = emptyList(), + val assistants: List = emptyList(), + val recent_jobs: List = emptyList(), + val model_profiles: List = emptyList() +) + +@Serializable +data class ApprovalDecisionDto( + val saved: Boolean, + val account: AccountDto +) + +@Serializable +data class PublishAppUpdateRequest( + val platform: String = "android", + val channel: String = "stable", + val versionCode: Int, + val versionName: String, + val minSupportedCode: Int, + val apkUrl: String, + val apkSha256: String = "", + val notes: String = "", + val forceUpdate: Boolean = false, + val isActive: Boolean = true +) + +@Serializable +data class PublishAppUpdateResponseDto( + val saved: Boolean, + val action: String, + val updateId: Int = 0 +) + +@Serializable +data class AppUpdateLatestDto( + val platform: String = "android", + val channel: String = "stable", + val hasUpdate: Boolean = false, + val latestVersionCode: Int = 0, + val latestVersionName: String = "", + val minSupportedCode: Int = 0, + val downloadUrl: String = "", + val apkSha256: String = "", + val releaseNotes: String = "", + val forceUpdate: Boolean = false, + val publishedAt: Long = 0L +) diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeRepository.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeRepository.kt new file mode 100644 index 0000000..0b902b2 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeRepository.kt @@ -0,0 +1,366 @@ +package com.aiglasses.app.storyforge + +import android.content.Context +import android.net.Uri +import android.provider.OpenableColumns +import com.aiglasses.app.BuildConfig +import com.jakewharton.retrofit2.converter.kotlinx.serialization.asConverterFactory +import java.io.File +import java.io.FileOutputStream +import java.net.InetAddress +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.MultipartBody +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.logging.HttpLoggingInterceptor +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import retrofit2.Retrofit +import retrofit2.create + +data class StoryForgeConnectionInfo( + val rawBaseUrl: String, + val requestBaseUrl: String, + val originalHostHeader: String, + val resolvedIp: String +) + +data class StoryForgeLoginResult( + val auth: AuthResponseDto, + val connection: StoryForgeConnectionInfo +) + +class StoryForgeRepository(private val context: Context) { + private val appContext = context.applicationContext + private val sessionStore = StoryForgeSessionStore(appContext) + + @OptIn(ExperimentalSerializationApi::class) + private val json = Json { + ignoreUnknownKeys = true + explicitNulls = false + } + + @Volatile + private var cachedService: StoryForgeApiService? = null + + @Volatile + private var cachedConnection: StoryForgeConnectionInfo? = null + + @Volatile + private var cachedToken: String = "" + + fun savedSession(): SavedStoryForgeSession = sessionStore.load() + + fun saveBaseUrl(baseUrl: String) { + sessionStore.saveBaseUrl(normalizeRawBaseUrl(baseUrl)) + } + + suspend fun resolveConnection(baseUrl: String): StoryForgeConnectionInfo = withContext(Dispatchers.IO) { + resolveConnectionInternal(baseUrl) + } + + suspend fun register(baseUrl: String, username: String, password: String, displayName: String): AccountDto { + sessionStore.saveBaseUrl(normalizeRawBaseUrl(baseUrl)) + return api(baseUrl = baseUrl, token = "").register( + RegisterAccountRequest( + username = username, + password = password, + display_name = displayName + ) + ) + } + + suspend fun login(baseUrl: String, username: String, password: String): StoryForgeLoginResult { + val auth = api(baseUrl = baseUrl, token = "").login(LoginRequest(username = username, password = password)) + val effectiveBaseUrl = auth.default_external_base_url.ifBlank { normalizeRawBaseUrl(baseUrl) } + sessionStore.save(effectiveBaseUrl, auth.token) + cachedService = null + val connection = apiConnection(baseUrl = effectiveBaseUrl, token = auth.token) + return StoryForgeLoginResult(auth = auth, connection = connection) + } + + suspend fun logout() { + runCatching { api().logout() } + sessionStore.clearToken() + cachedToken = "" + cachedService = null + } + + suspend fun me(): AccountDto = api().me() + + suspend fun dashboard(): DashboardDto = api().dashboard() + + suspend fun modelProfiles(): List = api().modelProfiles() + + suspend fun createModelProfile(request: ModelProfileRequest): ModelProfileDto = api().createModelProfile(request) + + suspend fun setPreferredAnalysisModel(modelProfileId: String): AccountDto = + api().setPreferredAnalysisModel(PreferredModelRequest(model_profile_id = modelProfileId)) + + suspend fun createKnowledgeBase(name: String, description: String): KnowledgeBaseDto = + api().createKnowledgeBase(KnowledgeBaseCreateRequest(name = name, description = description)) + + suspend fun knowledgeDocuments(knowledgeBaseId: String): List = + api().knowledgeDocuments(knowledgeBaseId) + + suspend fun jobs(): List = api().jobs() + + suspend fun job(jobId: String): JobDto = api().job(jobId) + + suspend fun createVideoLinkJob( + videoUrl: String, + title: String, + knowledgeBaseId: String, + assistantId: String, + analysisModelProfileId: String + ): JobDto = api().createVideoLinkJob( + ExploreVideoLinkRequest( + video_url = videoUrl, + title = title.ifBlank { null }, + knowledge_base_id = knowledgeBaseId.ifBlank { null }, + assistant_id = assistantId.ifBlank { null }, + analysis_model_profile_id = analysisModelProfileId.ifBlank { null } + ) + ) + + suspend fun createTextJob( + title: String, + content: String, + knowledgeBaseId: String, + assistantId: String, + analysisModelProfileId: String + ): JobDto = api().createTextJob( + ExploreTextRequest( + title = title, + content = content, + knowledge_base_id = knowledgeBaseId.ifBlank { null }, + assistant_id = assistantId.ifBlank { null }, + analysis_model_profile_id = analysisModelProfileId.ifBlank { null } + ) + ) + + suspend fun uploadVideo( + uri: Uri, + title: String, + knowledgeBaseId: String, + assistantId: String, + analysisModelProfileId: String + ): JobDto = withContext(Dispatchers.IO) { + val tempFile = copyUriToCache(uri) + try { + val filePart = MultipartBody.Part.createFormData( + name = "file", + filename = tempFile.name, + body = tempFile.asRequestBody(guessMimeType(tempFile.name).toMediaTypeOrNull()) + ) + api().uploadVideo( + file = filePart, + title = title.toRequestBody("text/plain".toMediaType()), + knowledgeBaseId = knowledgeBaseId.toRequestBody("text/plain".toMediaType()), + assistantId = assistantId.toRequestBody("text/plain".toMediaType()), + analysisModelProfileId = analysisModelProfileId.toRequestBody("text/plain".toMediaType()) + ) + } finally { + tempFile.delete() + } + } + + suspend fun createAssistant(request: AssistantCreateRequest): AssistantDto = api().createAssistant(request) + + suspend fun updateAssistant(assistantId: String, request: AssistantUpdateRequest): AssistantDto = + api().updateAssistant(assistantId, request) + + suspend fun generateCopy(assistantId: String, request: GenerateCopyRequest): GenerateCopyResponseDto = + api().generateCopy(assistantId, request) + + suspend fun pendingAccounts(): List = api().pendingAccounts() + + suspend fun approveAccount(accountId: String): ApprovalDecisionDto = api().approveAccount(accountId) + + suspend fun rejectAccount(accountId: String): ApprovalDecisionDto = api().rejectAccount(accountId) + + suspend fun latestUpdate(currentVersionCode: Int): AppUpdateLatestDto = + api().latestUpdate(currentVersionCode = currentVersionCode) + + suspend fun publishAppUpdate(request: PublishAppUpdateRequest): PublishAppUpdateResponseDto = + api().publishAppUpdate(request) + + suspend fun currentConnection(): StoryForgeConnectionInfo = apiConnection() + + private suspend fun api( + baseUrl: String? = null, + token: String? = null + ): StoryForgeApiService = withContext(Dispatchers.IO) { + val connection = apiConnection(baseUrl = baseUrl, token = token) + val authToken = token ?: sessionStore.load().token + if (cachedService != null && cachedConnection == connection && cachedToken == authToken) { + return@withContext cachedService!! + } + val client = buildClient(connection, authToken) + val retrofit = Retrofit.Builder() + .baseUrl(connection.requestBaseUrl) + .client(client) + .addConverterFactory(json.asConverterFactory("application/json".toMediaType())) + .build() + retrofit.create().also { + cachedService = it + cachedConnection = connection + cachedToken = authToken + } + } + + private suspend fun apiConnection( + baseUrl: String? = null, + token: String? = null + ): StoryForgeConnectionInfo = withContext(Dispatchers.IO) { + val saved = sessionStore.load() + val targetBaseUrl = normalizeRawBaseUrl(baseUrl ?: saved.baseUrl) + val resolved = resolveConnectionInternal(targetBaseUrl) + cachedConnection = resolved + if (token != null) { + cachedToken = token + } + resolved + } + + private fun buildClient(connection: StoryForgeConnectionInfo, token: String): OkHttpClient { + val logging = HttpLoggingInterceptor().apply { + level = HttpLoggingInterceptor.Level.BASIC + } + return OkHttpClient.Builder() + .protocols(listOf(Protocol.HTTP_1_1)) + .connectTimeout(12, TimeUnit.SECONDS) + .readTimeout(120, TimeUnit.SECONDS) + .writeTimeout(120, TimeUnit.SECONDS) + .callTimeout(150, TimeUnit.SECONDS) + .addInterceptor { chain -> + val builder: Request.Builder = chain.request().newBuilder() + if (token.isNotBlank()) { + builder.header("Authorization", "Bearer $token") + } + if (connection.originalHostHeader.isNotBlank()) { + builder.header("Host", connection.originalHostHeader) + } + builder.header("Connection", "close") + chain.proceed(builder.build()) + } + .addInterceptor(logging) + .build() + } + + private fun normalizeRawBaseUrl(baseUrl: String): String { + val trimmed = baseUrl.trim().ifBlank { BuildConfig.DEFAULT_STORYFORGE_BASE_URL } + val migrated = when { + trimmed.startsWith("http://test.hyzq.net:8081") -> BuildConfig.DEFAULT_STORYFORGE_BASE_URL + trimmed.startsWith("http://111.231.132.51:8081") -> BuildConfig.DEFAULT_STORYFORGE_BASE_URL + else -> trimmed + } + val withScheme = if (migrated.startsWith("http://") || migrated.startsWith("https://")) migrated else "http://$migrated" + return if (withScheme.endsWith('/')) withScheme else "$withScheme/" + } + + private fun resolveConnectionInternal(baseUrl: String): StoryForgeConnectionInfo { + val normalized = normalizeRawBaseUrl(baseUrl) + val httpUrl = normalized.toHttpUrlOrNull() ?: error("无效后端地址: $baseUrl") + val host = httpUrl.host + val scheme = httpUrl.scheme + if (scheme == "https" || isIpHost(host) || host == "localhost" || host == "10.0.2.2") { + return StoryForgeConnectionInfo( + rawBaseUrl = normalized, + requestBaseUrl = normalized, + originalHostHeader = "", + resolvedIp = if (isIpHost(host)) host else "" + ) + } + val resolvedIp = runCatching { + InetAddress.getAllByName(host).firstOrNull()?.hostAddress.orEmpty() + }.getOrDefault("") + .takeUnless { isInvalidResolvedIp(it) } + .orEmpty() + .ifBlank { + if (host.equals("test.hyzq.net", ignoreCase = true)) BuildConfig.DEFAULT_STORYFORGE_FALLBACK_IP else "" + } + if (resolvedIp.isBlank()) { + return StoryForgeConnectionInfo( + rawBaseUrl = normalized, + requestBaseUrl = normalized, + originalHostHeader = "", + resolvedIp = "" + ) + } + val rewritten = httpUrl.newBuilder().host(resolvedIp).build().toString() + return StoryForgeConnectionInfo( + rawBaseUrl = normalized, + requestBaseUrl = rewritten, + originalHostHeader = hostHeaderValue(httpUrl.host, httpUrl.port, scheme), + resolvedIp = resolvedIp + ) + } + + private fun hostHeaderValue(host: String, port: Int, scheme: String): String { + val isDefaultPort = (scheme == "http" && port == 80) || (scheme == "https" && port == 443) + return if (isDefaultPort) host else "$host:$port" + } + + private fun isIpHost(host: String): Boolean { + return IPV4_REGEX.matches(host) || host.contains(':') + } + + private fun isInvalidResolvedIp(ip: String): Boolean { + if (ip.isBlank()) return true + if (!IPV4_REGEX.matches(ip)) return false + val octets = ip.split('.').mapNotNull { it.toIntOrNull() } + if (octets.size != 4) return false + if (octets[0] == 127) return true + if (octets[0] == 0) return true + if (octets[0] == 169 && octets[1] == 254) return true + if (octets[0] == 198 && (octets[1] == 18 || octets[1] == 19)) return true + return false + } + + private fun copyUriToCache(uri: Uri): File { + val displayName = queryDisplayName(uri) + val safeName = displayName.ifBlank { "upload-${System.currentTimeMillis()}.mp4" } + val suffix = safeName.substringAfterLast('.', missingDelimiterValue = "mp4") + val target = File(appContext.cacheDir, "storyforge-${System.currentTimeMillis()}.$suffix") + appContext.contentResolver.openInputStream(uri).use { input -> + requireNotNull(input) { "无法读取所选视频" } + FileOutputStream(target).use { output -> + input.copyTo(output) + } + } + return target + } + + private fun queryDisplayName(uri: Uri): String { + if (uri.scheme == "file") { + return File(uri.path.orEmpty()).name + } + val cursor = appContext.contentResolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null) + cursor?.use { + val index = it.getColumnIndex(OpenableColumns.DISPLAY_NAME) + if (index >= 0 && it.moveToFirst()) { + return it.getString(index).orEmpty() + } + } + return uri.lastPathSegment.orEmpty() + } + + private fun guessMimeType(fileName: String): String = when { + fileName.endsWith(".mov", ignoreCase = true) -> "video/quicktime" + fileName.endsWith(".m4v", ignoreCase = true) -> "video/x-m4v" + else -> "video/mp4" + } + + private companion object { + private val IPV4_REGEX = Regex("""^\\d{1,3}(?:\\.\\d{1,3}){3}$""") + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeScreen.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeScreen.kt new file mode 100644 index 0000000..45068dc --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeScreen.kt @@ -0,0 +1,827 @@ +package com.aiglasses.app.storyforge + +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.ExperimentalLayoutApi +import androidx.compose.foundation.layout.FlowRow +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.navigationBarsPadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.FilterChip +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.NavigationBar +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Surface +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp + +@Composable +fun StoryForgeScreen( + state: StoryForgeUiState, + vm: StoryForgeViewModel, + onPickVideo: () -> Unit, + onInstallLatestUpdate: () -> Unit +) { + val heroBrush = Brush.linearGradient( + colors = listOf(Color(0xFF0B3C5D), Color(0xFF1F6E5F), Color(0xFFB97524)) + ) + + Scaffold( + bottomBar = { + if (state.isAuthenticated && state.isApproved) { + NavigationBar(modifier = Modifier.navigationBarsPadding()) { + BottomTabItem(label = "探索", tab = StoryForgeTab.Explore, state = state, onSelect = vm::selectTab) + BottomTabItem(label = "生产", tab = StoryForgeTab.Production, state = state, onSelect = vm::selectTab) + BottomTabItem(label = "我的", tab = StoryForgeTab.Mine, state = state, onSelect = vm::selectTab) + } + } + } + ) { innerPadding -> + Surface( + modifier = Modifier + .fillMaxSize() + .background(MaterialTheme.colorScheme.background) + .padding(innerPadding) + ) { + when { + !state.isAuthenticated -> AuthScreen(state = state, vm = vm, heroBrush = heroBrush) + !state.isApproved -> PendingApprovalScreen(state = state, vm = vm, heroBrush = heroBrush) + else -> AppShell( + state = state, + vm = vm, + heroBrush = heroBrush, + onPickVideo = onPickVideo, + onInstallLatestUpdate = onInstallLatestUpdate + ) + } + } + } +} + +@Composable +private fun BottomTabItem( + label: String, + tab: StoryForgeTab, + state: StoryForgeUiState, + onSelect: (StoryForgeTab) -> Unit +) { + val selected = state.currentTab == tab + Box( + modifier = Modifier + .clip(RoundedCornerShape(18.dp)) + .clickable { onSelect(tab) } + .background(if (selected) MaterialTheme.colorScheme.primaryContainer else Color.Transparent) + .padding(horizontal = 14.dp, vertical = 10.dp), + contentAlignment = Alignment.Center + ) { + Column(horizontalAlignment = Alignment.CenterHorizontally) { + Text(text = label.take(1), fontWeight = FontWeight.Bold) + Text(label, style = MaterialTheme.typography.labelSmall) + } + } +} + +@Composable +private fun AuthScreen( + state: StoryForgeUiState, + vm: StoryForgeViewModel, + heroBrush: Brush +) { + Box( + modifier = Modifier + .fillMaxSize() + .background(heroBrush) + .padding(18.dp), + contentAlignment = Alignment.Center + ) { + Card( + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surface), + shape = RoundedCornerShape(28.dp), + elevation = CardDefaults.cardElevation(defaultElevation = 4.dp), + modifier = Modifier.fillMaxWidth() + ) { + Column( + modifier = Modifier + .fillMaxWidth() + .padding(22.dp), + verticalArrangement = Arrangement.spacedBy(12.dp) + ) { + Text("StoryForge AI", style = MaterialTheme.typography.headlineSmall) + Text( + if (state.authMode == StoryForgeAuthMode.Login) "登录账号" else "注册新账号，提交后等待主管理员审批", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.7f) + ) + ChoiceRow( + options = listOf("登录" to (state.authMode == StoryForgeAuthMode.Login), "注册" to (state.authMode == StoryForgeAuthMode.Register)), + onSelect = { label -> vm.setAuthMode(if (label == "登录") StoryForgeAuthMode.Login else StoryForgeAuthMode.Register) } + ) + OutlinedTextField( + value = state.username, + onValueChange = vm::updateUsername, + modifier = Modifier.fillMaxWidth(), + label = { Text("账号") }, + singleLine = true + ) + OutlinedTextField( + value = state.password, + onValueChange = vm::updatePassword, + modifier = Modifier.fillMaxWidth(), + label = { Text("密码") }, + singleLine = true + ) + Button( + onClick = { if (state.authMode == StoryForgeAuthMode.Login) vm.login() else vm.registerAccount() }, + enabled = !state.busy, + modifier = Modifier.fillMaxWidth() + ) { + if (state.busy) { + CircularProgressIndicator(modifier = Modifier.size(18.dp), strokeWidth = 2.dp) + } else { + Text(if (state.authMode == StoryForgeAuthMode.Login) "登录" else "注册") + } + } + if (state.statusMessage.isNotBlank()) { + Text(state.statusMessage, style = MaterialTheme.typography.bodySmall) + } + if (state.errorMessage.isNotBlank()) { + Text(state.errorMessage, color = MaterialTheme.colorScheme.error) + } + } + } + } +} + +@Composable +private fun PendingApprovalScreen( + state: StoryForgeUiState, + vm: StoryForgeViewModel, + heroBrush: Brush +) { + val account = state.account + Column( + modifier = Modifier + .fillMaxSize() + .padding(18.dp), + verticalArrangement = Arrangement.spacedBy(16.dp) + ) { + HeroCard( + title = "等待审批", + subtitle = "${account?.display_name ?: account?.username ?: "当前账号"} 已登录，但尚未通过主管理员审批。", + heroBrush = heroBrush, + badges = listOf( + "审批状态：${account?.approval_status ?: "pending"}", + if (state.resolvedIp.isNotBlank()) "已解析到 ${state.resolvedIp}" else "" + ).filter { it.isNotBlank() } + ) + SectionCard(title = "当前说明", subtitle = state.statusMessage) { + Text("新注册账号在主管理员通过前，无法访问探索、生产和知识库功能。") + Spacer(modifier = Modifier.height(12.dp)) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Button(onClick = vm::refreshApprovalStatus, enabled = !state.busy) { + Text("刷新审批状态") + } + OutlinedButton(onClick = vm::logout) { + Text("退出登录") + } + } + if (state.errorMessage.isNotBlank()) { + Spacer(modifier = Modifier.height(10.dp)) + Text(state.errorMessage, color = MaterialTheme.colorScheme.error) + } + } + } +} + +@Composable +private fun AppShell( + state: StoryForgeUiState, + vm: StoryForgeViewModel, + heroBrush: Brush, + onPickVideo: () -> Unit, + onInstallLatestUpdate: () -> Unit +) { + val scroll = rememberScrollState() + Column( + modifier = Modifier + .fillMaxSize() + .verticalScroll(scroll) + .padding(18.dp), + verticalArrangement = Arrangement.spacedBy(16.dp) + ) { + HeroCard( + title = when (state.currentTab) { + StoryForgeTab.Explore -> "探索素材" + StoryForgeTab.Production -> "生产文案" + StoryForgeTab.Mine -> "我的工作台" + }, + subtitle = state.statusMessage, + heroBrush = heroBrush, + badges = listOf( + state.account?.display_name ?: state.account?.username.orEmpty(), + state.account?.role ?: "", + if (state.resolvedIp.isNotBlank()) "IP ${state.resolvedIp}" else "" + ).filter { it.isNotBlank() } + ) + StatusStrip(state = state, onRefresh = vm::refreshWorkspace) + when (state.currentTab) { + StoryForgeTab.Explore -> ExploreTab(state = state, vm = vm, onPickVideo = onPickVideo) + StoryForgeTab.Production -> ProductionTab(state = state, vm = vm) + StoryForgeTab.Mine -> MineTab(state = state, vm = vm, onInstallLatestUpdate = onInstallLatestUpdate) + } + } +} + +@Composable +private fun StatusStrip(state: StoryForgeUiState, onRefresh: () -> Unit) { + SectionCard(title = "连接状态", subtitle = if (state.busy) "正在同步" else "已连接") { + Text( + text = if (state.originalHost.isNotBlank()) { + "外网域名已解析为 ${state.resolvedIp}，请求会携带 Host=${state.originalHost}" + } else { + "当前使用地址：${state.baseUrl}" + }, + style = MaterialTheme.typography.bodySmall + ) + Spacer(modifier = Modifier.height(12.dp)) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), verticalAlignment = Alignment.CenterVertically) { + OutlinedButton(onClick = onRefresh) { + Text("刷新") + } + if (state.busy) { + CircularProgressIndicator(modifier = Modifier.size(18.dp), strokeWidth = 2.dp) + } + if (state.errorMessage.isNotBlank()) { + Text(state.errorMessage, color = MaterialTheme.colorScheme.error, style = MaterialTheme.typography.bodySmall) + } + } + } +} + +@Composable +private fun ExploreTab(state: StoryForgeUiState, vm: StoryForgeViewModel, onPickVideo: () -> Unit) { + SectionCard(title = "素材入口", subtitle = "视频链接、上传视频、输入文字都会转成文本并做风格分析") { + ChoiceRow( + options = listOf( + "视频链接" to (state.exploreInputMode == ExploreInputMode.VideoLink), + "上传视频" to (state.exploreInputMode == ExploreInputMode.UploadVideo), + "输入文字" to (state.exploreInputMode == ExploreInputMode.Text) + ), + onSelect = { label -> + vm.setExploreInputMode( + when (label) { + "视频链接" -> ExploreInputMode.VideoLink + "上传视频" -> ExploreInputMode.UploadVideo + else -> ExploreInputMode.Text + } + ) + } + ) + Spacer(modifier = Modifier.height(12.dp)) + KnowledgeBaseSelector(state = state, onSelect = vm::selectKnowledgeBase) + Spacer(modifier = Modifier.height(12.dp)) + AssistantSelector(state = state, onSelect = vm::selectAssistant) + Spacer(modifier = Modifier.height(12.dp)) + Text( + text = "当前分析模型：${state.modelProfiles.firstOrNull { it.id == state.account?.preferred_analysis_model_id }?.name ?: "本机默认模型"}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.7f) + ) + Spacer(modifier = Modifier.height(12.dp)) + when (state.exploreInputMode) { + ExploreInputMode.VideoLink -> { + OutlinedTextField( + value = state.videoUrl, + onValueChange = vm::updateVideoUrl, + modifier = Modifier.fillMaxWidth(), + label = { Text("短视频链接") }, + minLines = 2 + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.videoTitle, + onValueChange = vm::updateVideoTitle, + modifier = Modifier.fillMaxWidth(), + label = { Text("素材标题（可选）") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::submitVideoLink, enabled = !state.busy) { + Text("提交视频链接") + } + } + ExploreInputMode.UploadVideo -> { + OutlinedTextField( + value = state.videoTitle, + onValueChange = vm::updateVideoTitle, + modifier = Modifier.fillMaxWidth(), + label = { Text("素材标题（可选）") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), verticalAlignment = Alignment.CenterVertically) { + OutlinedButton(onClick = onPickVideo) { + Text(if (state.pickedVideoName.isBlank()) "选择视频文件" else "重新选择") + } + Text( + text = if (state.pickedVideoName.isBlank()) "未选择文件" else state.pickedVideoName, + modifier = Modifier.weight(1f), + maxLines = 2, + overflow = TextOverflow.Ellipsis + ) + } + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::submitUploadVideo, enabled = !state.busy && state.pickedVideoName.isNotBlank()) { + Text("上传并开始学习") + } + } + ExploreInputMode.Text -> { + OutlinedTextField( + value = state.textTitle, + onValueChange = vm::updateTextTitle, + modifier = Modifier.fillMaxWidth(), + label = { Text("素材标题") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.textContent, + onValueChange = vm::updateTextContent, + modifier = Modifier.fillMaxWidth(), + label = { Text("素材文字") }, + minLines = 5 + ) + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::submitText, enabled = !state.busy) { + Text("分析并沉淀到知识库") + } + } + } + } + + state.latestJob?.let { latestJob -> + SectionCard(title = "最新任务", subtitle = latestJob.title) { + KeyValueRow(label = "状态", value = latestJob.status) + KeyValueRow(label = "上传状态", value = latestJob.upload_status) + if (latestJob.transcript_text.isNotBlank()) { + KeyValueBlock(label = "文本转写", value = latestJob.transcript_text) + } + if (latestJob.style_summary.isNotBlank()) { + KeyValueBlock(label = "风格提炼", value = latestJob.style_summary) + } + if (latestJob.error.isNotBlank()) { + Text(latestJob.error, color = MaterialTheme.colorScheme.error) + } + } + } + + if (state.documents.isNotEmpty()) { + SectionCard(title = "当前知识库素材", subtitle = "已经沉淀到所选知识库的文本样本") { + state.documents.forEach { document -> + MiniCard(title = document.title, subtitle = document.style_summary.ifBlank { document.transcript_text.take(100) }) + Spacer(modifier = Modifier.height(10.dp)) + } + } + } +} + +@Composable +private fun ProductionTab(state: StoryForgeUiState, vm: StoryForgeViewModel) { + SectionCard(title = "智能体列表", subtitle = "一个智能体默认关联一个知识库，也可以关联多个知识库") { + ChoiceRow( + options = state.assistants.map { it.name to (state.selectedAssistantId == it.id) }, + onSelect = { label -> + state.assistants.firstOrNull { it.name == label }?.let { vm.selectAssistant(it.id) } + } + ) + Spacer(modifier = Modifier.height(12.dp)) + OutlinedButton(onClick = vm::startNewAssistant) { + Text("新建智能体") + } + } + + SectionCard(title = "编辑智能体", subtitle = "提示词由用户提供，可随时调整模型和知识库绑定") { + OutlinedTextField( + value = state.assistantName, + onValueChange = vm::updateAssistantName, + modifier = Modifier.fillMaxWidth(), + label = { Text("智能体名称") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.assistantDescription, + onValueChange = vm::updateAssistantDescription, + modifier = Modifier.fillMaxWidth(), + label = { Text("智能体说明") }, + minLines = 2 + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.assistantSystemPrompt, + onValueChange = vm::updateAssistantSystemPrompt, + modifier = Modifier.fillMaxWidth(), + label = { Text("系统提示词") }, + minLines = 5 + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.assistantGenerationGoal, + onValueChange = vm::updateAssistantGenerationGoal, + modifier = Modifier.fillMaxWidth(), + label = { Text("生成目标") }, + minLines = 3 + ) + Spacer(modifier = Modifier.height(12.dp)) + Text("选择生成模型", style = MaterialTheme.typography.titleSmall) + Spacer(modifier = Modifier.height(8.dp)) + ChoiceRow( + options = state.modelProfiles.map { it.name to (state.assistantModelProfileId == it.id) }, + onSelect = { label -> + state.modelProfiles.firstOrNull { it.name == label }?.let { vm.updateAssistantModelProfileId(it.id) } + } + ) + Spacer(modifier = Modifier.height(12.dp)) + Text("选择要关联的知识库", style = MaterialTheme.typography.titleSmall) + Spacer(modifier = Modifier.height(8.dp)) + ChoiceRow( + options = state.knowledgeBases.map { it.name to state.selectedAssistantKnowledgeBaseIds.contains(it.id) }, + onSelect = { label -> + state.knowledgeBases.firstOrNull { it.name == label }?.let { vm.toggleAssistantKnowledgeBase(it.id) } + } + ) + Spacer(modifier = Modifier.height(14.dp)) + Button(onClick = vm::saveAssistant, enabled = !state.busy) { + Text(if (state.assistantEditorId.isNullOrBlank()) "创建智能体" else "保存智能体配置") + } + } + + SectionCard(title = "生成文案", subtitle = "选择智能体后，直接基于关联知识库输出文案") { + OutlinedTextField( + value = state.generationBrief, + onValueChange = vm::updateGenerationBrief, + modifier = Modifier.fillMaxWidth(), + label = { Text("文案需求") }, + minLines = 4 + ) + Spacer(modifier = Modifier.height(10.dp)) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + OutlinedTextField( + value = state.generationPlatform, + onValueChange = vm::updateGenerationPlatform, + modifier = Modifier.weight(1f), + label = { Text("平台") }, + singleLine = true + ) + OutlinedTextField( + value = state.generationAudience, + onValueChange = vm::updateGenerationAudience, + modifier = Modifier.weight(1f), + label = { Text("目标受众") }, + singleLine = true + ) + } + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.generationExtraRequirements, + onValueChange = vm::updateGenerationExtraRequirements, + modifier = Modifier.fillMaxWidth(), + label = { Text("额外要求") }, + minLines = 3 + ) + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::generateCopy, enabled = !state.generateBusy) { + if (state.generateBusy) { + CircularProgressIndicator(modifier = Modifier.size(18.dp), strokeWidth = 2.dp) + } else { + Text("开始生成") + } + } + if (state.generationOutput.isNotBlank()) { + Spacer(modifier = Modifier.height(16.dp)) + KeyValueBlock(label = "生成结果", value = state.generationOutput) + } + } +} + +@Composable +private fun MineTab(state: StoryForgeUiState, vm: StoryForgeViewModel, onInstallLatestUpdate: () -> Unit) { + SectionCard(title = "我的账号", subtitle = state.account?.display_name ?: state.account?.username.orEmpty()) { + KeyValueRow(label = "用户名", value = state.account?.username ?: "-") + KeyValueRow(label = "角色", value = state.account?.role ?: "-") + KeyValueRow(label = "审批", value = state.account?.approval_status ?: "-") + KeyValueRow(label = "Base URL", value = state.baseUrl) + if (state.resolvedIp.isNotBlank()) { + KeyValueRow(label = "解析 IP", value = state.resolvedIp) + } + Spacer(modifier = Modifier.height(12.dp)) + OutlinedButton(onClick = vm::logout) { + Text("退出登录") + } + } + + SectionCard(title = "分析模型", subtitle = "探索页默认使用这里选中的模型") { + ChoiceRow( + options = state.modelProfiles.map { it.name to (state.account?.preferred_analysis_model_id == it.id) }, + onSelect = { label -> + state.modelProfiles.firstOrNull { it.name == label }?.let { vm.setPreferredModel(it.id) } + } + ) + Spacer(modifier = Modifier.height(14.dp)) + HorizontalDivider() + Spacer(modifier = Modifier.height(14.dp)) + OutlinedTextField( + value = state.newModelName, + onValueChange = vm::updateNewModelName, + modifier = Modifier.fillMaxWidth(), + label = { Text("模型别名") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.newModelBaseUrl, + onValueChange = vm::updateNewModelBaseUrl, + modifier = Modifier.fillMaxWidth(), + label = { Text("Base URL") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.newModelModelName, + onValueChange = vm::updateNewModelModelName, + modifier = Modifier.fillMaxWidth(), + label = { Text("模型名称") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.newModelApiKey, + onValueChange = vm::updateNewModelApiKey, + modifier = Modifier.fillMaxWidth(), + label = { Text("API Key") }, + minLines = 2 + ) + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::createModelProfile) { + Text("保存为默认分析模型") + } + } + + SectionCard(title = "OTA 更新", subtitle = state.otaStatus.ifBlank { "检查新版本并执行安装" }) { + Row(horizontalArrangement = Arrangement.spacedBy(12.dp), verticalAlignment = Alignment.CenterVertically) { + Button(onClick = vm::checkForUpdates) { + Text("检查更新") + } + OutlinedButton(onClick = onInstallLatestUpdate, enabled = state.otaInfo?.hasUpdate == true) { + Text("安装最新版本") + } + } + state.otaInfo?.let { ota -> + Spacer(modifier = Modifier.height(12.dp)) + KeyValueRow(label = "最新版本", value = "${ota.latestVersionName} (${ota.latestVersionCode})") + if (ota.releaseNotes.isNotBlank()) { + KeyValueBlock(label = "更新说明", value = ota.releaseNotes) + } + } + } + + if (state.account?.role == "super_admin") { + SectionCard(title = "主管理员审批", subtitle = "新注册账号需要你审批后才能正常使用全部功能") { + if (state.pendingAccounts.isEmpty()) { + Text("当前没有待审批账号") + } else { + state.pendingAccounts.forEach { account -> + Card(colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant)) { + Column(modifier = Modifier.fillMaxWidth().padding(14.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text(account.display_name, fontWeight = FontWeight.Bold) + Text(account.username, style = MaterialTheme.typography.bodySmall) + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Button(onClick = { vm.approveAccount(account.id) }) { + Text("通过") + } + OutlinedButton(onClick = { vm.rejectAccount(account.id) }) { + Text("拒绝") + } + } + } + } + Spacer(modifier = Modifier.height(10.dp)) + } + } + } + + SectionCard(title = "发布 OTA", subtitle = "主管理员可直接更新在线版本号和下载地址") { + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + OutlinedTextField( + value = state.publishVersionCode, + onValueChange = vm::updatePublishVersionCode, + modifier = Modifier.weight(1f), + label = { Text("VersionCode") }, + singleLine = true + ) + OutlinedTextField( + value = state.publishMinSupportedCode, + onValueChange = vm::updatePublishMinSupportedCode, + modifier = Modifier.weight(1f), + label = { Text("最低支持") }, + singleLine = true + ) + } + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.publishVersionName, + onValueChange = vm::updatePublishVersionName, + modifier = Modifier.fillMaxWidth(), + label = { Text("VersionName") }, + singleLine = true + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.publishApkUrl, + onValueChange = vm::updatePublishApkUrl, + modifier = Modifier.fillMaxWidth(), + label = { Text("APK 下载地址") }, + minLines = 2 + ) + Spacer(modifier = Modifier.height(10.dp)) + OutlinedTextField( + value = state.publishNotes, + onValueChange = vm::updatePublishNotes, + modifier = Modifier.fillMaxWidth(), + label = { Text("更新说明") }, + minLines = 3 + ) + Spacer(modifier = Modifier.height(12.dp)) + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Text("强制更新") + Switch(checked = state.publishForceUpdate, onCheckedChange = vm::setPublishForceUpdate) + } + Spacer(modifier = Modifier.height(12.dp)) + Button(onClick = vm::publishUpdate) { + Text("发布 OTA") + } + } + } + + SectionCard(title = "最近日志", subtitle = "用于确认审批、解析、任务和 OTA 状态") { + state.timeline.forEach { item -> + Text(item, style = MaterialTheme.typography.bodySmall) + Spacer(modifier = Modifier.height(6.dp)) + } + } +} + +@OptIn(ExperimentalLayoutApi::class) +@Composable +private fun ChoiceRow( + options: List>, + onSelect: (String) -> Unit +) { + FlowRow(horizontalArrangement = Arrangement.spacedBy(10.dp), verticalArrangement = Arrangement.spacedBy(10.dp)) { + options.forEach { (label, selected) -> + FilterChip( + selected = selected, + onClick = { onSelect(label) }, + label = { Text(label) } + ) + } + } +} + +@Composable +private fun KnowledgeBaseSelector(state: StoryForgeUiState, onSelect: (String) -> Unit) { + Text("选择知识库", style = MaterialTheme.typography.titleSmall) + Spacer(modifier = Modifier.height(8.dp)) + ChoiceRow( + options = state.knowledgeBases.map { it.name to (state.selectedKnowledgeBaseId == it.id) }, + onSelect = { label -> + state.knowledgeBases.firstOrNull { it.name == label }?.let { onSelect(it.id) } + } + ) +} + +@Composable +private fun AssistantSelector(state: StoryForgeUiState, onSelect: (String) -> Unit) { + Text("选择关联智能体", style = MaterialTheme.typography.titleSmall) + Spacer(modifier = Modifier.height(8.dp)) + ChoiceRow( + options = state.assistants.map { it.name to (state.selectedAssistantId == it.id) }, + onSelect = { label -> + state.assistants.firstOrNull { it.name == label }?.let { onSelect(it.id) } + } + ) +} + +@Composable +private fun HeroCard(title: String, subtitle: String, heroBrush: Brush, badges: List) { + Box( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(28.dp)) + .background(heroBrush) + .padding(20.dp) + ) { + Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + Text(title, style = MaterialTheme.typography.headlineLarge, color = Color.White) + Text(subtitle, style = MaterialTheme.typography.bodyLarge, color = Color(0xFFF8F5EF)) + if (badges.isNotEmpty()) { + ChoiceRow(options = badges.map { it to true }, onSelect = {}) + } + } + } +} + +@Composable +private fun SectionCard(title: String, subtitle: String, content: @Composable () -> Unit) { + Card( + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surface), + shape = RoundedCornerShape(22.dp), + elevation = CardDefaults.cardElevation(defaultElevation = 1.dp) + ) { + Column( + modifier = Modifier + .fillMaxWidth() + .padding(18.dp), + verticalArrangement = Arrangement.spacedBy(6.dp) + ) { + Text(title, style = MaterialTheme.typography.headlineSmall) + if (subtitle.isNotBlank()) { + Text( + subtitle, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.72f) + ) + } + Spacer(modifier = Modifier.height(6.dp)) + content() + } + } +} + +@Composable +private fun KeyValueRow(label: String, value: String) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically + ) { + Text(label, color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.7f)) + Spacer(modifier = Modifier.width(12.dp)) + Text(value, modifier = Modifier.weight(1f), maxLines = 2, overflow = TextOverflow.Ellipsis) + } +} + +@Composable +private fun KeyValueBlock(label: String, value: String) { + Text(label, style = MaterialTheme.typography.titleSmall) + Spacer(modifier = Modifier.height(6.dp)) + Box( + modifier = Modifier + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)) + .border(1.dp, MaterialTheme.colorScheme.outline.copy(alpha = 0.2f), RoundedCornerShape(16.dp)) + .padding(14.dp) + ) { + Text(value) + } +} + +@Composable +private fun MiniCard(title: String, subtitle: String) { + Card(colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant)) { + Column(modifier = Modifier.fillMaxWidth().padding(14.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) { + Text(title, fontWeight = FontWeight.Bold) + Text(subtitle, maxLines = 4, overflow = TextOverflow.Ellipsis, style = MaterialTheme.typography.bodySmall) + } + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeSessionStore.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeSessionStore.kt new file mode 100644 index 0000000..e308fd4 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeSessionStore.kt @@ -0,0 +1,59 @@ +package com.aiglasses.app.storyforge + +import android.content.Context +import com.aiglasses.app.BuildConfig + +data class SavedStoryForgeSession( + val baseUrl: String, + val token: String +) + +class StoryForgeSessionStore(context: Context) { + private val prefs = context.applicationContext.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + + fun load(): SavedStoryForgeSession = SavedStoryForgeSession( + baseUrl = migrateBaseUrl(prefs.getString(KEY_BASE_URL, BuildConfig.DEFAULT_STORYFORGE_BASE_URL).orEmpty()), + token = prefs.getString(KEY_TOKEN, "").orEmpty() + ) + + fun saveBaseUrl(baseUrl: String) { + prefs.edit().putString(KEY_BASE_URL, migrateBaseUrl(baseUrl)).apply() + } + + fun saveToken(token: String) { + prefs.edit().putString(KEY_TOKEN, token).apply() + } + + fun save(baseUrl: String, token: String) { + prefs.edit() + .putString(KEY_BASE_URL, migrateBaseUrl(baseUrl)) + .putString(KEY_TOKEN, token) + .apply() + } + + fun clearToken() { + prefs.edit().remove(KEY_TOKEN).apply() + } + + fun clearAll() { + prefs.edit().remove(KEY_BASE_URL).remove(KEY_TOKEN).apply() + } + + private companion object { + private const val PREFS_NAME = "storyforge_session" + private const val KEY_BASE_URL = "base_url" + private const val KEY_TOKEN = "token" + private const val LEGACY_DOMAIN_URL = "http://test.hyzq.net:8081" + private const val LEGACY_IP_URL = "http://111.231.132.51:8081" + } + + private fun migrateBaseUrl(baseUrl: String): String { + val trimmed = baseUrl.trim() + return when { + trimmed.isBlank() -> BuildConfig.DEFAULT_STORYFORGE_BASE_URL + trimmed.startsWith(LEGACY_DOMAIN_URL) -> BuildConfig.DEFAULT_STORYFORGE_BASE_URL + trimmed.startsWith(LEGACY_IP_URL) -> BuildConfig.DEFAULT_STORYFORGE_BASE_URL + else -> trimmed + } + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeViewModel.kt b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeViewModel.kt new file mode 100644 index 0000000..ebb7c6b --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/storyforge/StoryForgeViewModel.kt @@ -0,0 +1,907 @@ +package com.aiglasses.app.storyforge + +import android.app.Application +import android.net.Uri +import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.viewModelScope +import com.aiglasses.app.BuildConfig +import com.aiglasses.app.update.AppOtaUpdater +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import retrofit2.HttpException + +enum class StoryForgeTab { + Explore, + Production, + Mine +} + +enum class StoryForgeAuthMode { + Login, + Register +} + +enum class ExploreInputMode { + VideoLink, + UploadVideo, + Text +} + +private const val DEFAULT_SYSTEM_PROMPT = "你是一个擅长学习短视频口播风格的 AI 文案助手，请优先保留素材中的钩子、节奏、转折和行动号召。" +private const val DEFAULT_GENERATION_GOAL = "为不同渠道生成稳定风格的短视频标题、口播脚本和收尾行动号召。" + +private fun nextVersionName(current: String): String { + val parts = current.split('.').toMutableList() + val last = parts.lastOrNull()?.toIntOrNull() + if (last != null) { + parts[parts.lastIndex] = (last + 1).toString() + return parts.joinToString(".") + } + return current +} + +data class StoryForgeUiState( + val authMode: StoryForgeAuthMode = StoryForgeAuthMode.Login, + val baseUrl: String = BuildConfig.DEFAULT_STORYFORGE_BASE_URL, + val resolvedBaseUrl: String = "", + val resolvedIp: String = "", + val originalHost: String = "", + val isAuthenticated: Boolean = false, + val isApproved: Boolean = false, + val currentTab: StoryForgeTab = StoryForgeTab.Explore, + val busy: Boolean = false, + val generateBusy: Boolean = false, + val statusMessage: String = "准备连接 StoryForge", + val errorMessage: String = "", + val account: AccountDto? = null, + val knowledgeBases: List = emptyList(), + val assistants: List = emptyList(), + val modelProfiles: List = emptyList(), + val jobs: List = emptyList(), + val documents: List = emptyList(), + val selectedKnowledgeBaseId: String = "", + val selectedAssistantId: String = "", + val selectedAssistantKnowledgeBaseIds: Set = emptySet(), + val assistantEditorId: String? = null, + val username: String = "", + val password: String = "", + val createKnowledgeBaseName: String = "", + val createKnowledgeBaseDescription: String = "", + val exploreInputMode: ExploreInputMode = ExploreInputMode.VideoLink, + val videoUrl: String = "", + val videoTitle: String = "", + val textTitle: String = "", + val textContent: String = "", + val pickedVideoName: String = "", + val latestJobId: String = "", + val latestJob: JobDto? = null, + val assistantName: String = "", + val assistantDescription: String = "", + val assistantSystemPrompt: String = DEFAULT_SYSTEM_PROMPT, + val assistantGenerationGoal: String = DEFAULT_GENERATION_GOAL, + val assistantModelProfileId: String = "", + val generationBrief: String = "围绕 AI 创业做一条 60 秒短视频口播文案", + val generationPlatform: String = "抖音", + val generationAudience: String = "创业者", + val generationExtraRequirements: String = "开头结论先行，结尾给一个明确行动建议。", + val generationOutput: String = "", + val generationPromptExcerpt: String = "", + val newModelName: String = "", + val newModelBaseUrl: String = BuildConfig.DEFAULT_LOCAL_MODEL_BASE_URL, + val newModelApiKey: String = "", + val newModelModelName: String = "GLM-5", + val pendingAccounts: List = emptyList(), + val otaInfo: AppUpdateLatestDto? = null, + val otaStatus: String = "", + val publishVersionCode: String = (BuildConfig.VERSION_CODE + 1).toString(), + val publishVersionName: String = nextVersionName(BuildConfig.VERSION_NAME), + val publishMinSupportedCode: String = BuildConfig.VERSION_CODE.toString(), + val publishApkUrl: String = "", + val publishNotes: String = "", + val publishForceUpdate: Boolean = false, + val timeline: List = listOf("应用已启动，等待连接") +) + +class StoryForgeViewModel(application: Application) : AndroidViewModel(application) { + private val repository = StoryForgeRepository(application.applicationContext) + private val _state = MutableStateFlow(StoryForgeUiState(baseUrl = repository.savedSession().baseUrl)) + val state: StateFlow = _state.asStateFlow() + + private var jobPollingJob: Job? = null + private var pickedVideoUri: Uri? = null + + init { + restoreSession() + } + + fun updateBaseUrl(value: String) { + _state.value = _state.value.copy(baseUrl = value) + repository.saveBaseUrl(value) + } + + fun updateUsername(value: String) { + _state.value = _state.value.copy(username = value) + } + + fun updatePassword(value: String) { + _state.value = _state.value.copy(password = value) + } + + + fun setAuthMode(mode: StoryForgeAuthMode) { + _state.value = _state.value.copy(authMode = mode, errorMessage = "") + } + + fun selectTab(tab: StoryForgeTab) { + _state.value = _state.value.copy(currentTab = tab) + if (tab == StoryForgeTab.Mine && state.value.account?.role == "super_admin") { + loadPendingAccounts() + } + } + + fun updateCreateKnowledgeBaseName(value: String) { + _state.value = _state.value.copy(createKnowledgeBaseName = value) + } + + fun updateCreateKnowledgeBaseDescription(value: String) { + _state.value = _state.value.copy(createKnowledgeBaseDescription = value) + } + + fun updateVideoUrl(value: String) { + _state.value = _state.value.copy(videoUrl = value) + } + + fun updateVideoTitle(value: String) { + _state.value = _state.value.copy(videoTitle = value) + } + + fun updateTextTitle(value: String) { + _state.value = _state.value.copy(textTitle = value) + } + + fun updateTextContent(value: String) { + _state.value = _state.value.copy(textContent = value) + } + + fun setExploreInputMode(mode: ExploreInputMode) { + _state.value = _state.value.copy(exploreInputMode = mode, errorMessage = "") + } + + fun setPickedVideo(uri: Uri?, fileName: String) { + pickedVideoUri = uri + _state.value = _state.value.copy(pickedVideoName = fileName) + } + + fun selectKnowledgeBase(knowledgeBaseId: String) { + _state.value = _state.value.copy(selectedKnowledgeBaseId = knowledgeBaseId) + refreshDocuments() + } + + fun selectAssistant(assistantId: String) { + val assistant = _state.value.assistants.firstOrNull { it.id == assistantId } + _state.value = _state.value.copy( + selectedAssistantId = assistantId, + selectedAssistantKnowledgeBaseIds = assistant?.knowledge_base_ids?.toSet() ?: emptySet(), + assistantEditorId = assistant?.id, + assistantName = assistant?.name.orEmpty(), + assistantDescription = assistant?.description.orEmpty(), + assistantSystemPrompt = assistant?.system_prompt ?: DEFAULT_SYSTEM_PROMPT, + assistantGenerationGoal = assistant?.generation_goal ?: DEFAULT_GENERATION_GOAL, + assistantModelProfileId = assistant?.model_profile_id.orEmpty(), + generationOutput = "", + generationPromptExcerpt = "" + ) + } + + fun startNewAssistant() { + _state.value = _state.value.copy( + assistantEditorId = null, + assistantName = "", + assistantDescription = "", + assistantSystemPrompt = DEFAULT_SYSTEM_PROMPT, + assistantGenerationGoal = DEFAULT_GENERATION_GOAL, + assistantModelProfileId = preferredModelId(), + selectedAssistantKnowledgeBaseIds = listOfNotNull(state.value.selectedKnowledgeBaseId.takeIf { it.isNotBlank() }).toSet() + ) + } + + fun toggleAssistantKnowledgeBase(knowledgeBaseId: String) { + val updated = _state.value.selectedAssistantKnowledgeBaseIds.toMutableSet() + if (!updated.add(knowledgeBaseId)) { + updated.remove(knowledgeBaseId) + } + _state.value = _state.value.copy(selectedAssistantKnowledgeBaseIds = updated) + } + + fun updateAssistantName(value: String) { + _state.value = _state.value.copy(assistantName = value) + } + + fun updateAssistantDescription(value: String) { + _state.value = _state.value.copy(assistantDescription = value) + } + + fun updateAssistantSystemPrompt(value: String) { + _state.value = _state.value.copy(assistantSystemPrompt = value) + } + + fun updateAssistantGenerationGoal(value: String) { + _state.value = _state.value.copy(assistantGenerationGoal = value) + } + + fun updateAssistantModelProfileId(value: String) { + _state.value = _state.value.copy(assistantModelProfileId = value) + } + + fun updateGenerationBrief(value: String) { + _state.value = _state.value.copy(generationBrief = value) + } + + fun updateGenerationPlatform(value: String) { + _state.value = _state.value.copy(generationPlatform = value) + } + + fun updateGenerationAudience(value: String) { + _state.value = _state.value.copy(generationAudience = value) + } + + fun updateGenerationExtraRequirements(value: String) { + _state.value = _state.value.copy(generationExtraRequirements = value) + } + + fun updateNewModelName(value: String) { + _state.value = _state.value.copy(newModelName = value) + } + + fun updateNewModelBaseUrl(value: String) { + _state.value = _state.value.copy(newModelBaseUrl = value) + } + + fun updateNewModelApiKey(value: String) { + _state.value = _state.value.copy(newModelApiKey = value) + } + + fun updateNewModelModelName(value: String) { + _state.value = _state.value.copy(newModelModelName = value) + } + + fun updatePublishVersionCode(value: String) { + _state.value = _state.value.copy(publishVersionCode = value) + } + + fun updatePublishVersionName(value: String) { + _state.value = _state.value.copy(publishVersionName = value) + } + + fun updatePublishMinSupportedCode(value: String) { + _state.value = _state.value.copy(publishMinSupportedCode = value) + } + + fun updatePublishApkUrl(value: String) { + _state.value = _state.value.copy(publishApkUrl = value) + } + + fun updatePublishNotes(value: String) { + _state.value = _state.value.copy(publishNotes = value) + } + + fun setPublishForceUpdate(value: Boolean) { + _state.value = _state.value.copy(publishForceUpdate = value) + } + + fun registerAccount() { + val current = state.value + if (current.username.isBlank() || current.password.isBlank()) { + setError("请填写用户名和密码") + return + } + runBusy(message = "正在提交注册申请...", task = { + repository.register( + baseUrl = current.baseUrl, + username = current.username.trim(), + password = current.password, + displayName = current.username.trim() + ) + }) { account -> + appendTimeline("账号 ${account.username} 已注册，等待主管理员审批") + _state.value = _state.value.copy( + authMode = StoryForgeAuthMode.Login, + statusMessage = "注册成功，请等待主管理员审批", + errorMessage = "" + ) + } + } + + fun login() { + val current = state.value + if (current.username.isBlank() || current.password.isBlank()) { + setError("请先填写用户名和密码") + return + } + runBusy(message = "正在登录 StoryForge...", task = { + repository.login( + baseUrl = current.baseUrl, + username = current.username.trim(), + password = current.password + ) + }) { result -> + applyConnection(result.connection) + appendTimeline("账号 ${result.auth.account.username} 登录成功") + val account = result.auth.account + _state.value = _state.value.copy( + isAuthenticated = true, + isApproved = account.approval_status == "approved", + account = account, + statusMessage = if (account.approval_status == "approved") "登录成功，正在同步工作台" else "账号待主管理员审批", + errorMessage = "" + ) + if (account.approval_status == "approved") { + refreshWorkspace() + } + } + } + + fun refreshApprovalStatus() { + runBusy(message = "正在刷新审批状态...", task = { + repository.me() to repository.currentConnection() + }) { (account, connection) -> + applyConnection(connection) + _state.value = _state.value.copy( + isAuthenticated = true, + isApproved = account.approval_status == "approved", + account = account, + statusMessage = if (account.approval_status == "approved") "审批已通过，正在同步工作台" else "当前账号仍在等待审批", + errorMessage = "" + ) + appendTimeline("审批状态更新为 ${account.approval_status}") + if (account.approval_status == "approved") { + refreshWorkspace() + } + } + } + + fun logout() { + viewModelScope.launch { + repository.logout() + jobPollingJob?.cancel() + pickedVideoUri = null + appendTimeline("已退出当前账号") + _state.value = StoryForgeUiState(baseUrl = repository.savedSession().baseUrl) + } + } + + fun refreshWorkspace() { + viewModelScope.launch { + val current = state.value + _state.value = current.copy(busy = true, errorMessage = "", statusMessage = "正在同步工作台数据...") + runCatching { + val me = repository.me() + val connection = repository.currentConnection() + if (me.approval_status != "approved") { + Triple(me, connection, null) + } else { + Triple(me, connection, repository.dashboard()) + } + }.onSuccess { (account, connection, dashboard) -> + applyConnection(connection) + if (dashboard == null) { + _state.value = state.value.copy( + busy = false, + isAuthenticated = true, + isApproved = false, + account = account, + statusMessage = "账号待主管理员审批" + ) + } else { + applyDashboard(account, dashboard) + } + }.onFailure { throwable -> + if (throwable is HttpException && throwable.code() == 401) { + repository.logout() + _state.value = StoryForgeUiState(baseUrl = repository.savedSession().baseUrl).copy( + errorMessage = "登录已失效，请重新登录", + statusMessage = "请重新登录 StoryForge" + ) + } else { + _state.value = state.value.copy( + busy = false, + errorMessage = throwable.toReadableMessage(), + statusMessage = "同步失败，请检查网络或稍后重试" + ) + appendTimeline("同步失败: ${throwable.toReadableMessage()}") + } + } + } + } + + fun createKnowledgeBase() { + val current = state.value + if (current.createKnowledgeBaseName.isBlank()) { + setError("请先填写知识库名称") + return + } + runBusy(message = "正在创建知识库...", task = { + repository.createKnowledgeBase(current.createKnowledgeBaseName.trim(), current.createKnowledgeBaseDescription.trim()) + }) { knowledgeBase -> + appendTimeline("已创建知识库 ${knowledgeBase.name}") + _state.value = state.value.copy( + createKnowledgeBaseName = "", + createKnowledgeBaseDescription = "", + selectedKnowledgeBaseId = knowledgeBase.id + ) + refreshWorkspace() + } + } + + fun submitVideoLink() { + val current = state.value + if (current.videoUrl.isBlank()) { + setError("请先输入视频链接") + return + } + val knowledgeBaseId = selectedKnowledgeBaseIdOrFallback() + if (knowledgeBaseId.isBlank()) { + setError("请先选择知识库") + return + } + runBusy(message = "正在提交视频学习任务...", task = { + repository.createVideoLinkJob( + videoUrl = current.videoUrl.trim(), + title = current.videoTitle.trim(), + knowledgeBaseId = knowledgeBaseId, + assistantId = current.selectedAssistantId, + analysisModelProfileId = preferredModelId() + ) + }) { job -> + appendTimeline("视频链接任务已创建: ${job.title}") + _state.value = state.value.copy(videoUrl = "", videoTitle = "") + afterJobCreated(job) + } + } + + fun submitText() { + val current = state.value + if (current.textTitle.isBlank() || current.textContent.isBlank()) { + setError("请输入素材标题和文字内容") + return + } + val knowledgeBaseId = selectedKnowledgeBaseIdOrFallback() + if (knowledgeBaseId.isBlank()) { + setError("请先选择知识库") + return + } + runBusy(message = "正在提交文字分析任务...", task = { + repository.createTextJob( + title = current.textTitle.trim(), + content = current.textContent.trim(), + knowledgeBaseId = knowledgeBaseId, + assistantId = current.selectedAssistantId, + analysisModelProfileId = preferredModelId() + ) + }) { job -> + appendTimeline("文字素材已进入分析队列: ${job.title}") + _state.value = state.value.copy(textTitle = "", textContent = "") + afterJobCreated(job) + } + } + + fun submitUploadVideo() { + val current = state.value + val uri = pickedVideoUri + if (uri == null) { + setError("请先选择本地视频文件") + return + } + val knowledgeBaseId = selectedKnowledgeBaseIdOrFallback() + if (knowledgeBaseId.isBlank()) { + setError("请先选择知识库") + return + } + runBusy(message = "正在上传视频并创建学习任务...", task = { + repository.uploadVideo( + uri = uri, + title = current.videoTitle.trim(), + knowledgeBaseId = knowledgeBaseId, + assistantId = current.selectedAssistantId, + analysisModelProfileId = preferredModelId() + ) + }) { job -> + appendTimeline("视频上传成功，任务已创建: ${job.title}") + pickedVideoUri = null + _state.value = state.value.copy(videoTitle = "", pickedVideoName = "") + afterJobCreated(job) + } + } + + fun saveAssistant() { + val current = state.value + if (current.assistantName.isBlank()) { + setError("请先填写智能体名称") + return + } + if (current.selectedAssistantKnowledgeBaseIds.isEmpty()) { + setError("请至少关联一个知识库") + return + } + val request = AssistantCreateRequest( + name = current.assistantName.trim(), + description = current.assistantDescription.trim(), + system_prompt = current.assistantSystemPrompt.trim(), + generation_goal = current.assistantGenerationGoal.trim(), + knowledge_base_ids = current.selectedAssistantKnowledgeBaseIds.toList(), + model_profile_id = current.assistantModelProfileId.ifBlank { preferredModelId() } + ) + if (current.assistantEditorId.isNullOrBlank()) { + runBusy(message = "正在创建智能体...", task = { + repository.createAssistant(request) + }) { assistant -> + appendTimeline("已创建智能体 ${assistant.name}") + _state.value = state.value.copy(selectedAssistantId = assistant.id) + refreshWorkspace() + } + } else { + runBusy(message = "正在保存智能体配置...", task = { + repository.updateAssistant( + current.assistantEditorId, + AssistantUpdateRequest( + name = request.name, + description = request.description, + system_prompt = request.system_prompt, + generation_goal = request.generation_goal, + knowledge_base_ids = request.knowledge_base_ids, + model_profile_id = request.model_profile_id + ) + ) + }) { assistant -> + appendTimeline("已更新智能体 ${assistant.name}") + _state.value = state.value.copy(selectedAssistantId = assistant.id) + refreshWorkspace() + } + } + } + + fun generateCopy() { + val current = state.value + val assistantId = current.selectedAssistantId.ifBlank { current.assistantEditorId.orEmpty() } + if (assistantId.isBlank()) { + setError("请先选择一个智能体") + return + } + if (current.generationBrief.isBlank()) { + setError("请先填写文案需求") + return + } + viewModelScope.launch { + _state.value = state.value.copy(generateBusy = true, errorMessage = "", statusMessage = "正在生成文案，请稍候...") + runCatching { + repository.generateCopy( + assistantId, + GenerateCopyRequest( + brief = current.generationBrief.trim(), + platform = current.generationPlatform.trim(), + audience = current.generationAudience.trim(), + extra_requirements = current.generationExtraRequirements.trim(), + knowledge_base_ids = current.selectedAssistantKnowledgeBaseIds.toList() + ) + ) + }.onSuccess { result -> + _state.value = state.value.copy( + generateBusy = false, + generationOutput = result.content, + generationPromptExcerpt = result.prompt_excerpt, + statusMessage = "文案生成完成" + ) + appendTimeline("智能体已生成一条新文案") + }.onFailure { throwable -> + _state.value = state.value.copy( + generateBusy = false, + errorMessage = throwable.toReadableMessage(), + statusMessage = "文案生成失败" + ) + appendTimeline("文案生成失败: ${throwable.toReadableMessage()}") + } + } + } + + fun createModelProfile() { + val current = state.value + if (current.newModelName.isBlank() || current.newModelBaseUrl.isBlank() || current.newModelApiKey.isBlank() || current.newModelModelName.isBlank()) { + setError("请完整填写模型名称、Base URL、API Key 和模型名") + return + } + runBusy(message = "正在保存模型配置...", task = { + repository.createModelProfile( + ModelProfileRequest( + name = current.newModelName.trim(), + base_url = current.newModelBaseUrl.trim(), + api_key = current.newModelApiKey.trim(), + model_name = current.newModelModelName.trim(), + is_default = true + ) + ) + }) { profile -> + appendTimeline("已新增模型配置 ${profile.name}") + _state.value = state.value.copy( + newModelName = "", + newModelApiKey = "", + newModelModelName = current.newModelModelName, + assistantModelProfileId = profile.id + ) + refreshWorkspace() + } + } + + fun setPreferredModel(modelProfileId: String) { + runBusy(message = "正在切换默认分析模型...", task = { + repository.setPreferredAnalysisModel(modelProfileId) + }) { account -> + _state.value = state.value.copy(account = account) + appendTimeline("已切换默认分析模型") + refreshWorkspace() + } + } + + fun loadPendingAccounts() { + if (state.value.account?.role != "super_admin") return + viewModelScope.launch { + runCatching { repository.pendingAccounts() } + .onSuccess { pending -> + _state.value = state.value.copy(pendingAccounts = pending) + } + .onFailure { throwable -> + _state.value = state.value.copy(errorMessage = throwable.toReadableMessage()) + } + } + } + + fun approveAccount(accountId: String) { + runBusy(message = "正在通过账号审批...", task = { + repository.approveAccount(accountId) + }) { + appendTimeline("已通过一条账号审批") + refreshWorkspace() + } + } + + fun rejectAccount(accountId: String) { + runBusy(message = "正在拒绝账号申请...", task = { + repository.rejectAccount(accountId) + }) { + appendTimeline("已拒绝一条账号申请") + refreshWorkspace() + } + } + + fun checkForUpdates() { + viewModelScope.launch { + _state.value = state.value.copy(otaStatus = "正在检查更新...") + runCatching { repository.latestUpdate(BuildConfig.VERSION_CODE) } + .onSuccess { latest -> + _state.value = state.value.copy( + otaInfo = latest, + otaStatus = if (latest.hasUpdate) { + "发现新版本 ${latest.latestVersionName} (${latest.latestVersionCode})" + } else { + "当前已经是最新版本" + } + ) + appendTimeline("OTA 检查完成") + } + .onFailure { throwable -> + _state.value = state.value.copy(otaStatus = throwable.toReadableMessage(), errorMessage = throwable.toReadableMessage()) + } + } + } + + fun publishUpdate() { + val current = state.value + val versionCode = current.publishVersionCode.toIntOrNull() + val minSupportedCode = current.publishMinSupportedCode.toIntOrNull() + if (versionCode == null || minSupportedCode == null || current.publishVersionName.isBlank() || current.publishApkUrl.isBlank()) { + setError("请完整填写 OTA 的版本号、最小支持版本、下载地址") + return + } + runBusy(message = "正在发布 OTA 配置...", task = { + repository.publishAppUpdate( + PublishAppUpdateRequest( + versionCode = versionCode, + versionName = current.publishVersionName.trim(), + minSupportedCode = minSupportedCode, + apkUrl = current.publishApkUrl.trim(), + notes = current.publishNotes.trim(), + forceUpdate = current.publishForceUpdate + ) + ) + }) { response -> + _state.value = state.value.copy(otaStatus = "已发布 OTA: ${response.action}") + appendTimeline("主管理员已发布 OTA ${current.publishVersionName}") + checkForUpdates() + } + } + + fun onOtaLog(message: String) { + appendTimeline(message) + _state.value = state.value.copy(otaStatus = message) + } + + fun installLatestUpdate(otaUpdater: AppOtaUpdater) { + val latest = state.value.otaInfo + if (latest == null || !latest.hasUpdate || latest.downloadUrl.isBlank()) { + setError("当前没有可安装的更新") + return + } + val started = otaUpdater.downloadAndInstall( + apkUrl = latest.downloadUrl, + versionName = latest.latestVersionName.ifBlank { "${latest.latestVersionCode}" }, + expectedSha256 = latest.apkSha256 + ) + _state.value = state.value.copy(otaStatus = if (started) "OTA 下载已启动" else "OTA 下载启动失败") + } + + private fun restoreSession() { + val saved = repository.savedSession() + _state.value = state.value.copy(baseUrl = saved.baseUrl) + if (saved.token.isBlank()) { + viewModelScope.launch { + runCatching { repository.resolveConnection(saved.baseUrl) } + .onSuccess { applyConnection(it) } + } + return + } + refreshWorkspace() + } + + private fun refreshDocuments() { + val knowledgeBaseId = state.value.selectedKnowledgeBaseId + if (knowledgeBaseId.isBlank() || !state.value.isApproved) return + viewModelScope.launch { + runCatching { repository.knowledgeDocuments(knowledgeBaseId) } + .onSuccess { documents -> + _state.value = state.value.copy(documents = documents) + } + .onFailure { throwable -> + _state.value = state.value.copy(errorMessage = throwable.toReadableMessage()) + } + } + } + + private fun afterJobCreated(job: JobDto) { + _state.value = state.value.copy( + latestJob = job, + latestJobId = job.id, + currentTab = StoryForgeTab.Explore + ) + refreshWorkspace() + startJobPolling(job.id) + } + + private fun startJobPolling(jobId: String) { + jobPollingJob?.cancel() + jobPollingJob = viewModelScope.launch { + repeat(30) { + delay(5000) + runCatching { repository.job(jobId) } + .onSuccess { job -> + _state.value = state.value.copy(latestJob = job, latestJobId = job.id) + if (job.status == "completed" || job.status == "failed") { + appendTimeline("素材任务 ${job.title} 已${if (job.status == "completed") "完成" else "失败"}") + refreshWorkspace() + return@launch + } + } + } + } + } + + private fun applyDashboard(account: AccountDto, dashboard: DashboardDto) { + val selectedKbId = state.value.selectedKnowledgeBaseId.takeIf { id -> dashboard.knowledge_bases.any { it.id == id } } + ?: dashboard.knowledge_bases.firstOrNull()?.id.orEmpty() + val selectedAssistantId = state.value.selectedAssistantId.takeIf { id -> dashboard.assistants.any { it.id == id } } + ?: dashboard.assistants.firstOrNull()?.id.orEmpty() + val selectedAssistant = dashboard.assistants.firstOrNull { it.id == selectedAssistantId } + _state.value = state.value.copy( + busy = false, + isAuthenticated = true, + isApproved = true, + account = account, + knowledgeBases = dashboard.knowledge_bases, + assistants = dashboard.assistants, + modelProfiles = dashboard.model_profiles, + jobs = dashboard.recent_jobs, + documents = emptyList(), + selectedKnowledgeBaseId = selectedKbId, + selectedAssistantId = selectedAssistantId, + selectedAssistantKnowledgeBaseIds = selectedAssistant?.knowledge_base_ids?.toSet() + ?: listOfNotNull(selectedKbId.takeIf { it.isNotBlank() }).toSet(), + assistantEditorId = selectedAssistant?.id, + assistantName = selectedAssistant?.name.orEmpty(), + assistantDescription = selectedAssistant?.description.orEmpty(), + assistantSystemPrompt = selectedAssistant?.system_prompt ?: DEFAULT_SYSTEM_PROMPT, + assistantGenerationGoal = selectedAssistant?.generation_goal ?: DEFAULT_GENERATION_GOAL, + assistantModelProfileId = (selectedAssistant?.model_profile_id ?: "").ifBlank { preferredModelId(dashboard, account) }, + latestJob = dashboard.recent_jobs.firstOrNull(), + latestJobId = dashboard.recent_jobs.firstOrNull()?.id.orEmpty(), + pendingAccounts = if (account.role == "super_admin") state.value.pendingAccounts else emptyList(), + statusMessage = "工作台已同步完成", + errorMessage = "" + ) + refreshDocuments() + if (account.role == "super_admin") { + loadPendingAccounts() + } + } + + private fun preferredModelId( + dashboard: DashboardDto? = null, + account: AccountDto? = state.value.account + ): String { + val currentDashboard = dashboard + val accountPreferred = account?.preferred_analysis_model_id.orEmpty() + if (accountPreferred.isNotBlank()) return accountPreferred + val profiles = currentDashboard?.model_profiles ?: state.value.modelProfiles + return profiles.firstOrNull { it.is_default }?.id.orEmpty() + } + + private fun selectedKnowledgeBaseIdOrFallback(): String { + return state.value.selectedKnowledgeBaseId.ifBlank { + state.value.knowledgeBases.firstOrNull()?.id.orEmpty() + } + } + + private fun applyConnection(connection: StoryForgeConnectionInfo) { + _state.value = state.value.copy( + baseUrl = connection.rawBaseUrl, + resolvedBaseUrl = connection.requestBaseUrl, + resolvedIp = connection.resolvedIp, + originalHost = connection.originalHostHeader + ) + } + + private fun setError(message: String) { + _state.value = state.value.copy(errorMessage = message, statusMessage = message) + } + + private fun appendTimeline(message: String) { + val next = (listOf(message) + state.value.timeline).distinct().take(16) + _state.value = state.value.copy(timeline = next) + } + + private fun runBusy( + message: String, + task: suspend () -> T, + onSuccess: (T) -> Unit + ) { + viewModelScope.launch { + _state.value = state.value.copy(busy = true, errorMessage = "", statusMessage = message) + runCatching { task() } + .onSuccess { result -> + _state.value = state.value.copy(busy = false, errorMessage = "") + onSuccess(result) + } + .onFailure { throwable -> + _state.value = state.value.copy( + busy = false, + errorMessage = throwable.toReadableMessage(), + statusMessage = throwable.toReadableMessage() + ) + appendTimeline(throwable.toReadableMessage()) + } + } + } +} + +private fun Throwable.toReadableMessage(): String { + if (this is HttpException) { + val body = response()?.errorBody()?.string().orEmpty() + return if (body.isNotBlank()) { + body.take(240) + } else { + "请求失败 (${code()})" + } + } + return message ?: "发生未知错误" +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/ui/MainViewModel.kt b/android-app/app/src/main/java/com/aiglasses/app/ui/MainViewModel.kt new file mode 100644 index 0000000..53da594 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/ui/MainViewModel.kt @@ -0,0 +1,1387 @@ +package com.aiglasses.app.ui + +import android.app.Application +import android.content.Context +import android.util.Base64 +import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.viewModelScope +import com.aiglasses.app.BuildConfig +import com.aiglasses.app.ble.BleManager +import com.aiglasses.app.ble.GlassesBleEvent +import com.aiglasses.app.data.AdminStats +import com.aiglasses.app.data.BaiduInfo +import com.aiglasses.app.data.BackendRepository +import com.aiglasses.app.data.ClientEventRequest +import com.baidu.rtc.ndk.NdkLoader +import com.baidu.rtc.ndk.NdkPhoenix +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import retrofit2.HttpException + +data class MainUiState( + val deviceId: String = "dev-test-001", + val userId: String = "user-1", + val sessionId: String = "", + val messageText: String = "你好，请做个自我介绍。", + val sceneId: String = "voice_assistant", + val roleId: String = "assistant", + val activationSummary: String = "Activation: -", + val adminSummary: String = "Admin: loading", + val adminStats: AdminStats = AdminStats(), + val baiduInfo: BaiduInfo = BaiduInfo(), + val bleSummary: String = "BLE: idle", + val conversationRunning: Boolean = false, + val softwareConversationRunning: Boolean = false, + val softwareConversationStarting: Boolean = false, + val softwareSummary: String = "Software: idle", + val audioFrameCount: Int = 0, + val photoChunkCount: Int = 0, + val softwareSpeechCount: Int = 0, + val softwarePhotoCount: Int = 0, + val softwareMicFrameCount: Int = 0, + val softwareVoiceUploadCount: Int = 0, + val softwareLogs: List = listOf("software-ready"), + val softwareSpeakText: String = "", + val softwareSpeakSeq: Long = 0L, + val softwareAudioBase64: String = "", + val softwareAudioUrl: String = "", + val softwareAudioSeq: Long = 0L, + val softwareRealtimeWsUrl: String = "", + val softwareLicenseKey: String = "", + val softwareAppId: String = "", + val softwareCid: String = "", + val softwareToken: String = "", + val softwareContext: String = "", + val otaSummary: String = "OTA: not checked", + val otaChecking: Boolean = false, + val otaHasUpdate: Boolean = false, + val otaLatestVersionCode: Int = 0, + val otaLatestVersionName: String = "", + val otaDownloadUrl: String = "", + val otaApkSha256: String = "", + val otaReleaseNotes: String = "", + val otaForceUpdate: Boolean = false, + val otaInstallSeq: Long = 0L, + val lastMessage: String = "Ready", + val isLoading: Boolean = false, + val timeline: List = listOf("System ready") +) + +@Serializable +private data class PendingRemoteLog( + val localId: Long, + val sessionId: String? = null, + val deviceId: String = "", + val eventType: String, + val eventLevel: String = "INFO", + val payload: Map = emptyMap(), + val ts: Long +) + +class MainViewModel(application: Application) : AndroidViewModel(application) { + companion object { + private const val BACKEND_BASE_URL = "https://test.hyzq.net" + private const val REMOTE_LOG_PREFS = "remote_log_queue" + private const val REMOTE_LOG_KEY_QUEUE = "pending_events_json" + private const val REMOTE_LOG_MAX_PENDING = 400 + private const val REMOTE_LOG_BATCH_SIZE = 25 + private const val REMOTE_LOG_FLUSH_INTERVAL_MS = 2_500L + } + + private val _state = MutableStateFlow(MainUiState()) + val state: StateFlow = _state.asStateFlow() + + private var repository = BackendRepository(BACKEND_BASE_URL) + private val bleManager = BleManager(application.applicationContext) + private val remoteLogPrefs = application.applicationContext.getSharedPreferences( + REMOTE_LOG_PREFS, + Context.MODE_PRIVATE + ) + private val remoteLogJson = Json { + ignoreUnknownKeys = true + encodeDefaults = true + } + + private var metricsJob: Job? = null + private var cameraPollingJob: Job? = null + private var softwareStartJob: Job? = null + private var softwareHeartbeatJob: Job? = null + private var remoteLogFlushJob: Job? = null + private var bootstrapInProgress = false + private var softwareSpeakSeqCounter = 0L + private var softwareAudioSeqCounter = 0L + private var lastMicFallbackReportMs = 0L + private var lastVoiceUploadMs = 0L + private var voiceMaxRms = 0 + private val voiceBuffer = ByteArrayOutputStream() + private var softwareRemoteLogSeq = 0L + private var otaRemoteLogSeq = 0L + private var otaInstallSeqCounter = 0L + private var pendingAudioFrames = 0 + private var pendingAudioBytes = 0 + private var pendingPhotoChunks = 0 + private var latestCameraImageBase64 = "" + private var latestCameraImageBytes = 0 + private var latestCameraImageWidth = 0 + private var latestCameraImageHeight = 0 + private var latestCameraImageAt = 0L + private var lastVisionTriggerAt = 0L + private var lastVisionTriggerText = "" + private var pendingVisionQueryText = "" + private var pendingVisionQueryAt = 0L + private var lastVisionSendAt = 0L + private var lastVisionSentText = "" + private var lastBargeInDispatchAt = 0L + private val pendingRemoteLogsLock = Any() + private val pendingRemoteLogs = mutableListOf() + private var remoteLogFlushInProgress = false + private var remoteLogLocalId = 0L + + init { + loadPendingRemoteLogs() + startRemoteLogFlushJob() + observeBle() + } + + fun updateDeviceId(value: String) { + _state.value = _state.value.copy(deviceId = value) + } + + fun updateUserId(value: String) { + _state.value = _state.value.copy(userId = value) + } + + fun updateMessageText(value: String) { + _state.value = _state.value.copy(messageText = value) + } + + fun updateSceneId(value: String) { + _state.value = _state.value.copy(sceneId = value) + } + + fun updateRoleId(value: String) { + _state.value = _state.value.copy(roleId = value) + } + + fun checkAppUpdate(currentVersionCode: Int, currentVersionName: String, autoInstall: Boolean) { + _state.value = _state.value.copy( + otaChecking = true, + otaSummary = "OTA: checking (current=$currentVersionName/$currentVersionCode)" + ) + viewModelScope.launch { + runCatching { + repository.appUpdateLatest(currentVersionCode) + }.onSuccess { data -> + val hasUpdate = data.hasUpdate && data.downloadUrl.isNotBlank() + _state.value = _state.value.copy( + otaChecking = false, + otaHasUpdate = hasUpdate, + otaLatestVersionCode = data.latestVersionCode, + otaLatestVersionName = data.latestVersionName, + otaDownloadUrl = data.downloadUrl, + otaApkSha256 = data.apkSha256, + otaReleaseNotes = data.releaseNotes, + otaForceUpdate = data.forceUpdate, + otaSummary = if (hasUpdate) { + "OTA: new ${data.latestVersionName}/${data.latestVersionCode}" + } else { + "OTA: already latest" + } + ) + logResult( + if (hasUpdate) { + "Update found: ${data.latestVersionName} (${data.latestVersionCode})" + } else { + "App is up to date" + } + ) + if (hasUpdate && autoInstall) { + triggerOtaInstall() + } + }.onFailure { + _state.value = _state.value.copy( + otaChecking = false, + otaSummary = "OTA: check failed ${it.message}" + ) + logResult("OTA check failed: ${it.message}") + } + } + } + + fun triggerOtaInstall() { + if (_state.value.otaDownloadUrl.isBlank()) { + _state.value = _state.value.copy(otaSummary = "OTA: download url missing") + return + } + otaInstallSeqCounter += 1 + _state.value = _state.value.copy( + otaInstallSeq = otaInstallSeqCounter, + otaSummary = "OTA: downloading..." + ) + appendSoftwareLog("OTA: 开始下载安装") + } + + fun consumeOtaInstallTrigger() { + if (_state.value.otaInstallSeq == 0L) return + _state.value = _state.value.copy(otaInstallSeq = 0L) + } + + fun onOtaStatus(status: String) { + val msg = status.trim() + if (msg.isBlank()) return + _state.value = _state.value.copy(otaSummary = msg.take(120)) + appendSoftwareLog(msg) + pushOtaLogRemote(msg) + logResult(msg) + } + + fun bindDevice() { + perform("Bind") { + val data = repository.bindDevice(_state.value.deviceId, _state.value.userId) + "Bind success: ${data.bindStatus}, license=${data.licenseKeyMasked}" + } + } + + fun createSession() { + perform("CreateSession") { + val data = repository.createSession(_state.value.deviceId, _state.value.userId) + _state.value = _state.value.copy(sessionId = data.sessionId) + "Session created: ${data.sessionId}" + } + } + + fun stopSession() { + val sid = _state.value.sessionId + if (sid.isBlank()) { + _state.value = _state.value.copy(lastMessage = "No sessionId to stop") + return + } + perform("StopSession") { + val data = repository.stopSession(sid) + "Session stop result: ${data.sessionStatus}" + } + } + + fun refreshStatus() { + perform("Status") { + val data = repository.getDeviceStatus(_state.value.deviceId) + "Status: bind=${data.bindStatus}, active=${data.activeSessionStatus ?: "NONE"}" + } + } + + fun heartbeat() { + val sid = _state.value.sessionId + if (sid.isBlank()) { + _state.value = _state.value.copy(lastMessage = "No sessionId for heartbeat") + return + } + perform("Heartbeat") { + val data = repository.heartbeat(sid) + "Heartbeat ok: ${data.heartbeatAt}" + } + } + + fun postEvent() { + perform("PostEvent") { + val data = repository.postDemoEvent(_state.value.deviceId, _state.value.sessionId.ifBlank { null }) + "Event saved: ${data.saved}" + } + } + + fun sendMessage() { + val sid = _state.value.sessionId + if (sid.isBlank()) { + logResult("No sessionId for send-message") + return + } + val text = _state.value.messageText.trim() + if (text.isBlank()) { + logResult("Message cannot be empty") + return + } + perform("SendMessage") { + val data = sendMessageWithOptionalVision( + sessionId = sid, + message = text, + source = "manual_input" + ) + handleProviderAction(data) + "Send message: ${data.status}" + } + } + + fun switchRole() { + val sid = _state.value.sessionId + if (sid.isBlank()) { + logResult("No sessionId for switch-role") + return + } + val sceneId = _state.value.sceneId.trim() + val roleId = _state.value.roleId.trim() + if (sceneId.isBlank() || roleId.isBlank()) { + logResult("SceneId/RoleId cannot be empty") + return + } + perform("SwitchRole") { + val data = repository.switchRole(sid, sceneId, roleId) + "Switch role: ${data.status}" + } + } + + fun interrupt() { + toggleInterrupt(true) + } + + fun resumeInterrupt() { + toggleInterrupt(false) + } + + private fun toggleInterrupt(interrupt: Boolean) { + val sid = _state.value.sessionId + if (sid.isBlank()) { + logResult("No sessionId for interrupt") + return + } + perform(if (interrupt) "Interrupt" else "Resume") { + val data = repository.interrupt(sid, interrupt) + "Interrupt=$interrupt: ${data.status}" + } + } + + fun queryActivation() { + perform("ActivationQuery") { + val data = repository.activationQuery(_state.value.deviceId) + _state.value = _state.value.copy( + activationSummary = "Activation: ${data.status} (${data.detail})" + ) + "Activation fetched for ${data.deviceId}" + } + } + + fun reloadLicenses() { + perform("ReloadLicenses") { + val data = repository.reloadLicenses() + "License reloaded: inserted=${data.inserted}" + } + } + + fun loadAdminOverview() { + perform("AdminOverview") { + val data = repository.adminOverview() + val stats = data.stats + val baidu = data.baidu + _state.value = _state.value.copy( + adminSummary = "Admin: devices=${stats.totalDevices}, running=${stats.runningSessions}, baidu=${baidu.mode}", + adminStats = stats, + baiduInfo = baidu + ) + "Admin overview refreshed" + } + } + + fun startHardwareConversation() { + if (_state.value.conversationRunning) { + logResult("Conversation already running") + return + } + if (_state.value.softwareConversationRunning) { + stopSoftwareConversation() + } + val appUuid = buildAppUuid(_state.value.userId) + _state.value = _state.value.copy( + conversationRunning = true, + bleSummary = "BLE: scanning..." + ) + logResult("Starting hardware conversation") + bleManager.connectAndHandshake(appUuid = appUuid) + } + + fun stopHardwareConversation() { + _state.value = _state.value.copy(conversationRunning = false) + cameraPollingJob?.cancel() + metricsJob?.cancel() + bleManager.stopWakeUpAudio() + bleManager.disconnect() + val sid = _state.value.sessionId + if (sid.isNotBlank()) { + viewModelScope.launch { + runCatching { repository.stopSession(sid) } + } + } + logResult("Hardware conversation stopped") + } + + fun startSoftwareConversation() { + if (_state.value.softwareConversationRunning || _state.value.softwareConversationStarting) { + logResult("Software conversation already running") + return + } + if (_state.value.conversationRunning) { + stopHardwareConversation() + } + _state.value = _state.value.copy( + softwareConversationStarting = true, + softwareSummary = "Software: bootstrapping..." + ) + voiceBuffer.reset() + voiceMaxRms = 0 + lastVoiceUploadMs = 0L + appendSoftwareLog("Bootstrapping software conversation") + softwareStartJob?.cancel() + softwareStartJob = viewModelScope.launch { + var startStep = "resolve_device_id" + suspend fun runStartStep(name: String, block: suspend () -> T): T { + startStep = name + return block() + } + try { + val deviceId = resolveSoftwareBaiduDeviceId(_state.value.userId) + appendSoftwareLog("Baidu device id: $deviceId") + appendSoftwareLog("Health check: $BACKEND_BASE_URL/healthz") + val health = runStartStep("healthz") { repository.healthz() } + appendSoftwareLog("Health OK: status=${health.status}, env=${health.env}") + appendSoftwareLog("Bind device: $deviceId") + val bind = runStartStep("bind_device") { + repository.bindDevice(deviceId, _state.value.userId) + } + appendSoftwareLog("Bind success: bind=${bind.bindStatus}, license=${bind.licenseStatus}") + appendSoftwareLog("Create session start") + val session = runStartStep("create_session") { + repository.createSession(deviceId, _state.value.userId) + } + _state.value = _state.value.copy( + deviceId = deviceId, + sessionId = session.sessionId, + softwareConversationRunning = true, + softwareConversationStarting = false, + softwareSummary = "Software: running / session=${session.sessionId.takeLast(8)}", + softwareRealtimeWsUrl = session.realtimeWsUrl, + softwareLicenseKey = bind.licenseKey, + softwareAppId = session.appId, + softwareCid = session.cid, + softwareToken = session.token, + softwareContext = session.context + ) + startSoftwareHeartbeatJob() + appendSoftwareLog("Session created: ${session.sessionId}") + queueSoftwareSpeak("软件对话已连接") + logResult("Software conversation started: ${session.sessionId}") + } catch (_: kotlinx.coroutines.CancellationException) { + _state.value = _state.value.copy( + softwareConversationStarting = false, + softwareConversationRunning = false, + softwareSummary = "Software: cancelled" + ) + appendSoftwareLog("Start cancelled") + logResult("Software conversation start cancelled") + } catch (e: Exception) { + _state.value = _state.value.copy( + softwareConversationStarting = false, + softwareConversationRunning = false, + softwareSummary = "Software: start failed" + ) + val detail = describeStartFailure(startStep, e) + appendSoftwareLog("Start failed($startStep): $detail") + queueSoftwareSpeak("软件对话启动失败") + logResult("Software conversation failed($startStep): $detail") + } finally { + softwareStartJob = null + } + } + } + + fun stopSoftwareConversation() { + val sid = _state.value.sessionId + softwareStartJob?.cancel() + softwareStartJob = null + _state.value = _state.value.copy( + softwareConversationStarting = false, + softwareConversationRunning = false, + softwareSummary = "Software: stopped", + softwareRealtimeWsUrl = "", + softwareLicenseKey = "", + softwareAppId = "", + softwareCid = "", + softwareToken = "", + softwareContext = "" + ) + voiceBuffer.reset() + voiceMaxRms = 0 + latestCameraImageBase64 = "" + latestCameraImageBytes = 0 + latestCameraImageWidth = 0 + latestCameraImageHeight = 0 + latestCameraImageAt = 0L + lastVisionTriggerAt = 0L + lastVisionTriggerText = "" + pendingVisionQueryText = "" + pendingVisionQueryAt = 0L + lastVisionSendAt = 0L + lastVisionSentText = "" + lastBargeInDispatchAt = 0L + softwareHeartbeatJob?.cancel() + if (sid.isNotBlank()) { + viewModelScope.launch { + runCatching { repository.stopSession(sid) } + } + } + appendSoftwareLog("Software conversation stopped") + logResult("Software conversation stopped") + } + + fun onSoftwareSpeechText(text: String) { + val message = text.trim() + if (message.isBlank()) return + _state.value = _state.value.copy( + softwareSpeechCount = _state.value.softwareSpeechCount + 1, + softwareSummary = "Software: last speech=${message.take(16)}" + ) + appendSoftwareLog("Speech recognized: ${message.take(80)}") + val sid = _state.value.sessionId + if (sid.isBlank()) { + appendSoftwareLog("Drop speech: session not ready") + logResult("Software speech dropped: session not ready") + return + } + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_PHONE_MIC_TEXT", + payload = mapOf("text" to message.take(120)) + ) + sendMessageWithOptionalVision( + sessionId = sid, + message = message, + source = "speech_recognizer" + ) + }.onSuccess { data -> + appendSoftwareLog("Send message success: ${data.status}, detail=${data.detail.take(60)}") + handleProviderAction(data) + logResult("Software message forwarded") + }.onFailure { + appendSoftwareLog("Send message failed: ${it.message}") + queueSoftwareSpeak("消息发送失败") + logResult("Software message failed: ${it.message}") + } + } + } + + fun onSoftwareRealtimeAsrText(text: String) { + val message = text.trim() + if (message.isBlank()) return + _state.value = _state.value.copy( + softwareSpeechCount = _state.value.softwareSpeechCount + 1, + softwareSummary = "Software: realtime asr=${message.take(16)}" + ) + appendSoftwareLog("Realtime ASR text: ${message.take(80)}") + maybeTriggerVisionByRealtimeAsr(message) + } + + fun onSoftwareRealtimeVisionFrameReady( + imageBase64: String, + bytes: Int, + width: Int, + height: Int, + source: String + ) { + val sid = _state.value.sessionId + if (sid.isBlank()) { + appendSoftwareLog("Vision skipped($source): session not ready") + return + } + val now = System.currentTimeMillis() + if (pendingVisionQueryAt > 0L && now - pendingVisionQueryAt > 20_000L) { + pendingVisionQueryText = "" + pendingVisionQueryAt = 0L + } + val message = pendingVisionQueryText.ifBlank { + "请结合刚上传的图片回答用户当前画面内容。" + } + if (message == lastVisionSentText && now - lastVisionSendAt < 2000L) { + appendSoftwareLog("Vision skipped($source): duplicate send suppressed") + return + } + appendSoftwareLog( + "Vision send($source): text=${message.take(40)}, frame=${width}x${height}, bytes=$bytes" + ) + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_PHONE_VISION_FRAME_READY", + payload = mapOf( + "source" to source, + "text" to message.take(120), + "width" to width.toString(), + "height" to height.toString(), + "bytes" to bytes.toString() + ) + ) + repository.sendVisionMessage( + sessionId = sid, + message = message, + imageBase64 = imageBase64, + width = width, + height = height, + bytes = bytes + ) + }.onSuccess { data -> + lastVisionSendAt = now + lastVisionSentText = message + if (pendingVisionQueryText == message) { + pendingVisionQueryText = "" + pendingVisionQueryAt = 0L + } + appendSoftwareLog( + "Vision send result($source): status=${data.status}, detail=${data.detail.take(60)}" + ) + if (data.audioBase64.isNotBlank() || data.audioUrl.isNotBlank()) { + handleProviderAction(data) + } + }.onFailure { + appendSoftwareLog("Vision send failed($source): ${it.message}") + } + } + } + + fun onSoftwareBargeInDetected() { + val sid = _state.value.sessionId + if (sid.isBlank() || !_state.value.softwareConversationRunning) return + val now = System.currentTimeMillis() + if (now - lastBargeInDispatchAt < 1500L) return + lastBargeInDispatchAt = now + appendSoftwareLog("Barge-in detected: realtime break only") + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_BARGE_IN", + payload = mapOf( + "source" to "android_app", + "mode" to "realtime_break_only" + ) + ) + }.onSuccess { + appendSoftwareLog("Barge-in break acknowledged") + }.onFailure { + appendSoftwareLog("Barge-in event upload failed: ${it.message}") + } + } + } + + fun onSoftwareCameraCapture(imageBase64: String, bytes: Int, width: Int, height: Int) { + _state.value = _state.value.copy( + softwarePhotoCount = _state.value.softwarePhotoCount + 1, + softwareSummary = "Software: camera ${width}x${height}" + ) + appendSoftwareLog("Camera capture: ${width}x${height}, bytes=$bytes") + latestCameraImageBase64 = imageBase64 + latestCameraImageBytes = bytes + latestCameraImageWidth = width + latestCameraImageHeight = height + latestCameraImageAt = System.currentTimeMillis() + val sid = _state.value.sessionId + if (sid.isBlank()) return + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_PHONE_CAMERA_CAPTURE", + payload = mapOf( + "bytes" to bytes.toString(), + "width" to width.toString(), + "height" to height.toString(), + "base64Size" to imageBase64.length.toString() + ) + ) + } + } + } + + fun onSoftwareMicFallbackFrame(pcmBytes: ByteArray, rms: Int, sampleRate: Int) { + val bytes = pcmBytes.size + _state.value = _state.value.copy( + softwareMicFrameCount = _state.value.softwareMicFrameCount + 1, + softwareSummary = "Software: mic fallback rms=$rms" + ) + if (bytes > 0) { + voiceBuffer.write(pcmBytes, 0, bytes) + if (voiceBuffer.size() > 320_000) { + val all = voiceBuffer.toByteArray() + voiceBuffer.reset() + val tail = all.copyOfRange(all.size - 240_000, all.size) + voiceBuffer.write(tail) + } + } + if (rms > voiceMaxRms) voiceMaxRms = rms + val now = System.currentTimeMillis() + if (now - lastMicFallbackReportMs < 3_000) return + lastMicFallbackReportMs = now + appendSoftwareLog("Mic fallback frame: bytes=$bytes, rms=$rms, sampleRate=$sampleRate") + val sid = _state.value.sessionId + if (sid.isBlank()) return + maybeUploadVoiceChunk(sessionId = sid, sampleRate = sampleRate, now = now) + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_PHONE_MIC_FALLBACK", + payload = mapOf( + "bytes" to bytes.toString(), + "rms" to rms.toString() + ) + ) + } + } + } + + fun onSoftwareLog(message: String) { + _state.value = _state.value.copy(softwareSummary = "Software: $message") + appendSoftwareLog(message) + logResult("Software: $message") + } + + private fun maybeUploadVoiceChunk(sessionId: String, sampleRate: Int, now: Long) { + if (now - lastVoiceUploadMs < 2200) return + if (voiceBuffer.size() < 24_000) return + if (voiceMaxRms < 300) { + voiceBuffer.reset() + voiceMaxRms = 0 + return + } + val pcm = voiceBuffer.toByteArray() + voiceBuffer.reset() + val peakRms = voiceMaxRms + voiceMaxRms = 0 + lastVoiceUploadMs = now + val durationMs = ((pcm.size / 2.0) / sampleRate * 1000.0).toInt().coerceAtLeast(200) + val payload = Base64.encodeToString(pcm, Base64.NO_WRAP) + if (_state.value.softwareRealtimeWsUrl.isNotBlank()) { + appendSoftwareLog("Realtime PCM window: bytes=${pcm.size}, dur=${durationMs}ms, peakRms=$peakRms") + _state.value = _state.value.copy( + softwareVoiceUploadCount = _state.value.softwareVoiceUploadCount + 1 + ) + return + } + appendSoftwareLog("Voice chunk upload: bytes=${pcm.size}, dur=${durationMs}ms, peakRms=$peakRms") + viewModelScope.launch { + runCatching { + repository.sendVoiceMessage( + sessionId = sessionId, + pcmBase64 = payload, + sampleRate = sampleRate, + durationMs = durationMs, + rms = peakRms + ) + }.onSuccess { data -> + _state.value = _state.value.copy( + softwareVoiceUploadCount = _state.value.softwareVoiceUploadCount + 1 + ) + appendSoftwareLog( + "Voice response: status=${data.status}, asr=${data.asrText.take(30)}, tts=${data.ttsText.take(30)}" + ) + handleProviderAction(data) + }.onFailure { + appendSoftwareLog("Voice upload failed: ${it.message}") + } + } + } + + private suspend fun sendMessageWithOptionalVision( + sessionId: String, + message: String, + source: String + ): com.aiglasses.app.data.ProviderActionData { + val now = System.currentTimeMillis() + if (!isVisionIntent(message)) { + return repository.sendMessage(sessionId, message) + } + if (!hasRecentVisionFrame(now)) { + appendSoftwareLog("Vision skipped($source): no recent camera frame") + return repository.sendMessage(sessionId, message) + } + appendSoftwareLog( + "Vision request($source): text=${message.take(40)}, frame=${latestCameraImageWidth}x${latestCameraImageHeight}" + ) + return repository.sendVisionMessage( + sessionId = sessionId, + message = message, + imageBase64 = latestCameraImageBase64, + width = latestCameraImageWidth, + height = latestCameraImageHeight, + bytes = latestCameraImageBytes + ) + } + + private fun maybeTriggerVisionByRealtimeAsr(message: String) { + if (!isVisionIntent(message)) return + val sid = _state.value.sessionId + if (sid.isBlank()) { + appendSoftwareLog("Vision skipped(realtime): session not ready") + return + } + val now = System.currentTimeMillis() + if (message == lastVisionTriggerText && now - lastVisionTriggerAt < 2500L) { + return + } + if (!hasRecentVisionFrame(now)) { + appendSoftwareLog("Vision skipped(realtime): no recent camera frame") + return + } + lastVisionTriggerAt = now + lastVisionTriggerText = message + pendingVisionQueryText = message + pendingVisionQueryAt = now + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "APP_PHONE_VISION_QUERY", + payload = mapOf( + "source" to "realtime_asr", + "text" to message.take(120), + "width" to latestCameraImageWidth.toString(), + "height" to latestCameraImageHeight.toString(), + "bytes" to latestCameraImageBytes.toString() + ) + ) + }.onSuccess { + appendSoftwareLog("Vision intent queued(realtime): waiting for [E]:[UPLOAD_IMAGE]") + }.onFailure { + appendSoftwareLog("Vision request failed: ${it.message}") + } + } + } + + private fun hasRecentVisionFrame(now: Long): Boolean { + if (latestCameraImageBase64.isBlank()) return false + if (latestCameraImageBytes <= 0) return false + return now - latestCameraImageAt <= 25_000L + } + + private fun isVisionIntent(text: String): Boolean { + val normalized = text.trim().lowercase(Locale.ROOT) + if (normalized.isBlank()) return false + val keywords = listOf( + "看到什么", + "看到了什么", + "看见什么", + "看眼前", + "眼前", + "眼前的东西", + "面前是什么", + "周围是什么", + "前面是什么", + "前面有", + "画面", + "镜头", + "摄像头", + "图里", + "图中", + "视频", + "识别", + "这是什么", + "帮我看看", + "看一下", + "look", + "what do you see", + "what can you see", + "camera" + ) + return keywords.any { normalized.contains(it) } + } + + private fun handleProviderAction(data: com.aiglasses.app.data.ProviderActionData) { + if (data.audioBase64.isNotBlank() || data.audioUrl.isNotBlank()) { + queueSoftwareAudio(data.audioBase64, data.audioUrl) + return + } + if (data.ttsText.isNotBlank()) { + queueSoftwareSpeak(data.ttsText) + return + } + val ackOnly = data.status.equals("ACCEPTED", ignoreCase = true) && + data.detail.contains("accepted", ignoreCase = true) + if (ackOnly) { + return + } + if (data.detail.isNotBlank()) { + queueSoftwareSpeak(data.detail) + } + } + + fun consumeSoftwareSpeakText() { + if (_state.value.softwareSpeakText.isBlank()) return + _state.value = _state.value.copy(softwareSpeakText = "") + } + + fun consumeSoftwareAudio() { + if (_state.value.softwareAudioBase64.isBlank() && _state.value.softwareAudioUrl.isBlank()) return + _state.value = _state.value.copy( + softwareAudioBase64 = "", + softwareAudioUrl = "" + ) + } + + private fun perform(action: String, block: suspend () -> String) { + _state.value = _state.value.copy(isLoading = true, lastMessage = "$action running...") + viewModelScope.launch { + try { + val message = block() + logResult(message) + } catch (e: Exception) { + logResult("$action failed: ${e.message}") + } + } + } + + private fun observeBle() { + viewModelScope.launch { + bleManager.state.collect { ble -> + val summary = buildString { + append("BLE: ") + if (ble.connected) append("connected") else append("disconnected") + if (ble.notificationsReady) append(" / notify") + if (ble.handshaked) append(" / handshaked") + if (ble.devUuid.isNotBlank()) append(" / ${ble.devUuid.takeLast(6)}") + } + _state.value = _state.value.copy( + bleSummary = summary, + deviceId = if (ble.devUuid.isNotBlank()) ble.devUuid else _state.value.deviceId + ) + } + } + + viewModelScope.launch { + bleManager.events.collect { event -> + when (event) { + is GlassesBleEvent.Log -> { + logResult("BLE: ${event.message}") + } + is GlassesBleEvent.HandshakeOk -> { + _state.value = _state.value.copy(deviceId = event.devUuid) + if (_state.value.conversationRunning) { + bootstrapConversation(event.devUuid) + } + } + is GlassesBleEvent.AudioFrame -> { + pendingAudioFrames += 1 + pendingAudioBytes += event.bytes.size + _state.value = _state.value.copy( + audioFrameCount = _state.value.audioFrameCount + 1 + ) + } + is GlassesBleEvent.CameraThumbInfo -> { + val sid = _state.value.sessionId + if (sid.isNotBlank()) { + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = if (event.isVideo) "VIDEO_THUMB_INFO" else "PHOTO_THUMB_INFO", + payload = mapOf("sourceFileName" to event.sourceFileName) + ) + } + } + } + } + is GlassesBleEvent.CameraThumbData -> { + pendingPhotoChunks += 1 + _state.value = _state.value.copy( + photoChunkCount = _state.value.photoChunkCount + 1 + ) + } + is GlassesBleEvent.StatusUpdate -> { + val sid = _state.value.sessionId + if (sid.isNotBlank()) { + viewModelScope.launch { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "DEVICE_STATUS_UPDATE", + payload = mapOf("rawJson" to event.payloadJson.take(512)) + ) + } + } + } + } + } + } + } + } + + private fun bootstrapConversation(devUuid: String) { + if (bootstrapInProgress) return + bootstrapInProgress = true + perform("HardwareBootstrap") { + try { + repository.bindDevice(devUuid, _state.value.userId) + val session = repository.createSession(devUuid, _state.value.userId) + _state.value = _state.value.copy(sessionId = session.sessionId) + bleManager.startWakeUpAudio() + bleManager.triggerPhotoCapture() + startReporterJobs() + "Hardware conversation started: ${session.sessionId}" + } finally { + bootstrapInProgress = false + } + } + } + + private fun startReporterJobs() { + metricsJob?.cancel() + metricsJob = viewModelScope.launch { + while (_state.value.conversationRunning) { + delay(1500) + val sid = _state.value.sessionId + if (sid.isBlank()) continue + val audioFrames = pendingAudioFrames + val audioBytes = pendingAudioBytes + val photoChunks = pendingPhotoChunks + pendingAudioFrames = 0 + pendingAudioBytes = 0 + pendingPhotoChunks = 0 + if (audioFrames > 0) { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "BLE_AUDIO_STATS", + payload = mapOf( + "frames" to audioFrames.toString(), + "bytes" to audioBytes.toString() + ) + ) + } + } + if (photoChunks > 0) { + runCatching { + repository.postEvent( + deviceId = _state.value.deviceId, + sessionId = sid, + eventType = "CAMERA_THUMB_CHUNKS", + payload = mapOf("chunks" to photoChunks.toString()) + ) + } + } + } + } + + cameraPollingJob?.cancel() + cameraPollingJob = viewModelScope.launch { + while (_state.value.conversationRunning) { + delay(12000) + bleManager.triggerPhotoCapture() + } + } + } + + private fun startSoftwareHeartbeatJob() { + softwareHeartbeatJob?.cancel() + softwareHeartbeatJob = viewModelScope.launch { + while (_state.value.softwareConversationRunning) { + delay(20_000) + val sid = _state.value.sessionId + if (sid.isBlank()) continue + runCatching { repository.heartbeat(sid) } + .onFailure { appendSoftwareLog("Heartbeat failed: ${it.message}") } + } + } + } + + private fun startRemoteLogFlushJob() { + remoteLogFlushJob?.cancel() + remoteLogFlushJob = viewModelScope.launch { + while (isActive) { + delay(REMOTE_LOG_FLUSH_INTERVAL_MS) + flushPendingRemoteLogs() + } + } + } + + private fun appendSoftwareLog(message: String) { + val ts = SimpleDateFormat("HH:mm:ss", Locale.getDefault()).format(Date()) + val logs = listOf("[$ts] $message") + _state.value.softwareLogs.take(79) + _state.value = _state.value.copy(softwareLogs = logs) + pushSoftwareLogRemote("[$ts] $message") + } + + private fun pushSoftwareLogRemote(line: String) { + softwareRemoteLogSeq += 1 + enqueueRemoteLog( + eventType = "APP_SOFTWARE_LOG", + payload = mapOf( + "seq" to softwareRemoteLogSeq.toString(), + "line" to line.take(300), + "channel" to "software" + ) + ) + } + + private fun pushOtaLogRemote(line: String) { + if (!line.contains("OTA:", ignoreCase = true)) return + otaRemoteLogSeq += 1 + enqueueRemoteLog( + eventType = "APP_OTA_LOG", + payload = mapOf( + "seq" to otaRemoteLogSeq.toString(), + "line" to line.take(300), + "latestVersionName" to _state.value.otaLatestVersionName, + "latestVersionCode" to _state.value.otaLatestVersionCode.toString(), + "channel" to "ota" + ) + ) + } + + private fun queueSoftwareSpeak(text: String) { + val msg = text.trim() + if (msg.isBlank()) return + softwareSpeakSeqCounter += 1 + _state.value = _state.value.copy( + softwareSpeakText = msg.take(80), + softwareSpeakSeq = softwareSpeakSeqCounter + ) + } + + private fun queueSoftwareAudio(audioBase64: String, audioUrl: String) { + softwareAudioSeqCounter += 1 + _state.value = _state.value.copy( + softwareAudioBase64 = audioBase64, + softwareAudioUrl = audioUrl, + softwareAudioSeq = softwareAudioSeqCounter + ) + } + + private fun logResult(message: String) { + val ts = SimpleDateFormat("HH:mm:ss", Locale.getDefault()).format(Date()) + val history = listOf("[$ts] $message") + _state.value.timeline.take(11) + _state.value = _state.value.copy( + isLoading = false, + lastMessage = message, + timeline = history + ) + enqueueRemoteLog( + eventType = "APP_TIMELINE_LOG", + payload = mapOf( + "line" to "[$ts] $message".take(300), + "channel" to "timeline" + ) + ) + } + + private fun enqueueRemoteLog( + eventType: String, + eventLevel: String = "INFO", + payload: Map = emptyMap(), + sessionId: String? = _state.value.sessionId.takeIf { it.isNotBlank() }, + ts: Long = System.currentTimeMillis() + ) { + synchronized(pendingRemoteLogsLock) { + remoteLogLocalId += 1 + pendingRemoteLogs += PendingRemoteLog( + localId = remoteLogLocalId, + sessionId = sessionId, + deviceId = _state.value.deviceId.trim(), + eventType = eventType, + eventLevel = eventLevel, + payload = payload, + ts = ts + ) + if (pendingRemoteLogs.size > REMOTE_LOG_MAX_PENDING) { + val overflow = pendingRemoteLogs.size - REMOTE_LOG_MAX_PENDING + repeat(overflow) { + pendingRemoteLogs.removeAt(0) + } + } + } + persistPendingRemoteLogs() + if (pendingRemoteLogCount() >= REMOTE_LOG_BATCH_SIZE) { + flushPendingRemoteLogs() + } + } + + private fun flushPendingRemoteLogs() { + if (remoteLogFlushInProgress) return + val batchItems = synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.take(REMOTE_LOG_BATCH_SIZE) + } + if (batchItems.isEmpty()) return + val fallbackDeviceId = _state.value.deviceId.trim() + if (fallbackDeviceId.isBlank()) return + remoteLogFlushInProgress = true + viewModelScope.launch { + runCatching { + repository.postEventsBatch( + batchItems.map { item -> + ClientEventRequest( + sessionId = item.sessionId, + deviceId = item.deviceId.ifBlank { fallbackDeviceId }, + eventType = item.eventType, + eventLevel = item.eventLevel, + payload = item.payload + mapOf( + "appVersionName" to BuildConfig.VERSION_NAME, + "appVersionCode" to BuildConfig.VERSION_CODE.toString() + ), + ts = item.ts + ) + } + ) + }.onSuccess { + val sentIds = batchItems.map { item -> item.localId }.toSet() + synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.removeAll { item -> item.localId in sentIds } + } + persistPendingRemoteLogs() + }.also { + remoteLogFlushInProgress = false + if (it.isSuccess && pendingRemoteLogCount() > 0) { + flushPendingRemoteLogs() + } + } + } + } + + private fun loadPendingRemoteLogs() { + val raw = remoteLogPrefs.getString(REMOTE_LOG_KEY_QUEUE, null) ?: return + runCatching { + remoteLogJson.decodeFromString>(raw) + }.onSuccess { restored -> + synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.clear() + pendingRemoteLogs.addAll(restored.takeLast(REMOTE_LOG_MAX_PENDING)) + remoteLogLocalId = pendingRemoteLogs.maxOfOrNull { it.localId } ?: 0L + } + }.onFailure { + remoteLogPrefs.edit().remove(REMOTE_LOG_KEY_QUEUE).apply() + synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.clear() + remoteLogLocalId = 0L + } + } + } + + private fun persistPendingRemoteLogs() { + val snapshot = synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.takeLast(REMOTE_LOG_MAX_PENDING) + } + if (snapshot.isEmpty()) { + remoteLogPrefs.edit().remove(REMOTE_LOG_KEY_QUEUE).apply() + return + } + remoteLogPrefs.edit() + .putString( + REMOTE_LOG_KEY_QUEUE, + remoteLogJson.encodeToString(snapshot) + ) + .apply() + } + + private fun pendingRemoteLogCount(): Int { + return synchronized(pendingRemoteLogsLock) { + pendingRemoteLogs.size + } + } + + private fun buildAppUuid(userId: String): String { + val suffix = System.currentTimeMillis().toString().takeLast(10) + val raw = "${userId}_$suffix" + return raw.take(32) + } + + private fun buildPhoneDeviceId(userId: String): String { + val ts = System.currentTimeMillis().toString().takeLast(8) + return "phone_${userId.take(8)}_$ts" + } + + private fun resolveSoftwareBaiduDeviceId(userId: String): String { + val fallback = _state.value.deviceId.ifBlank { buildPhoneDeviceId(userId) } + val loaded = runCatching { NdkLoader.loadLibrary() }.getOrDefault(false) + if (!loaded) { + appendSoftwareLog("Baidu device id fallback: NdkLoader.loadLibrary=false") + return fallback + } + val raw = runCatching { + NdkPhoenix.getDeviceInfo(getApplication().applicationContext) + }.onFailure { + appendSoftwareLog("Baidu device id fallback: ${it.javaClass.simpleName}") + }.getOrDefault("").trim() + if (raw.isBlank()) { + appendSoftwareLog("Baidu device id fallback: deviceInfo empty") + return fallback + } + return md5Hex(raw).ifBlank { fallback } + } + + private fun md5Hex(value: String): String { + return runCatching { + val digest = MessageDigest.getInstance("MD5").digest(value.toByteArray(Charsets.UTF_8)) + digest.joinToString("") { "%02x".format(it) } + }.getOrDefault("") + } + + private fun describeStartFailure(step: String, error: Exception): String { + val base = when (error) { + is HttpException -> { + val body = runCatching { error.response()?.errorBody()?.string().orEmpty() }.getOrDefault("") + buildString { + append("HttpException(") + append(error.code()) + append(")") + val httpMessage = error.message().orEmpty() + if (httpMessage.isNotBlank()) { + append(": ") + append(httpMessage) + } + if (body.isNotBlank()) { + append(", body=") + append(body.take(180)) + } + } + } + is IOException -> "IOException: ${error.message ?: "network transport failure"}" + else -> "${error.javaClass.simpleName}: ${error.message ?: "unknown error"}" + } + val cause = error.cause?.let { "${it.javaClass.simpleName}: ${it.message}" }.orEmpty() + return buildString { + append(base) + if (cause.isNotBlank()) { + append(", cause=") + append(cause.take(180)) + } + append(", baseUrl=") + append(BACKEND_BASE_URL) + append(", step=") + append(step) + } + } + + override fun onCleared() { + super.onCleared() + softwareStartJob?.cancel() + remoteLogFlushJob?.cancel() + } +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/ui/theme/AppTheme.kt b/android-app/app/src/main/java/com/aiglasses/app/ui/theme/AppTheme.kt new file mode 100644 index 0000000..47e1d8c --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/ui/theme/AppTheme.kt @@ -0,0 +1,74 @@ +package com.aiglasses.app.ui.theme + +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.darkColorScheme +import androidx.compose.material3.lightColorScheme +import androidx.compose.material3.Typography +import androidx.compose.runtime.Composable +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +private val LightColors = lightColorScheme( + primary = Color(0xFF0E4B43), + secondary = Color(0xFF9C6427), + tertiary = Color(0xFF2A5B8A), + background = Color(0xFFF7F3EC), + surface = Color(0xFFFFFCF8), + onPrimary = Color.White, + onSecondary = Color.White, + onBackground = Color(0xFF1A1713), + onSurface = Color(0xFF1A1713) +) + +private val DarkColors = darkColorScheme( + primary = Color(0xFF7FD6C7), + secondary = Color(0xFFFFC27A), + tertiary = Color(0xFF98C7FF), + background = Color(0xFF101714), + surface = Color(0xFF18211D), + onPrimary = Color(0xFF062D29), + onSecondary = Color(0xFF4B2B00), + onBackground = Color(0xFFF0E8DB), + onSurface = Color(0xFFF0E8DB) +) + +private val AppTypography = Typography( + headlineLarge = TextStyle( + fontFamily = FontFamily.Serif, + fontWeight = FontWeight.Bold, + fontSize = 34.sp, + lineHeight = 40.sp + ), + headlineSmall = TextStyle( + fontFamily = FontFamily.Serif, + fontWeight = FontWeight.SemiBold, + fontSize = 22.sp, + lineHeight = 28.sp + ), + bodyLarge = TextStyle( + fontFamily = FontFamily.SansSerif, + fontSize = 16.sp, + lineHeight = 24.sp + ), + labelLarge = TextStyle( + fontFamily = FontFamily.SansSerif, + fontWeight = FontWeight.Medium, + fontSize = 14.sp + ) +) + +@Composable +fun AIGlassesTheme( + darkTheme: Boolean = isSystemInDarkTheme(), + content: @Composable () -> Unit +) { + MaterialTheme( + colorScheme = if (darkTheme) DarkColors else LightColors, + typography = AppTypography, + content = content + ) +} diff --git a/android-app/app/src/main/java/com/aiglasses/app/update/AppOtaUpdater.kt b/android-app/app/src/main/java/com/aiglasses/app/update/AppOtaUpdater.kt new file mode 100644 index 0000000..6235597 --- /dev/null +++ b/android-app/app/src/main/java/com/aiglasses/app/update/AppOtaUpdater.kt @@ -0,0 +1,559 @@ +package com.aiglasses.app.update + +import android.app.DownloadManager +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.net.Uri +import android.os.Build +import android.os.Environment +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.provider.Settings +import androidx.core.content.FileProvider +import java.io.File +import java.io.FileOutputStream +import java.security.MessageDigest + +class AppOtaUpdater( + context: Context, + private val onLog: (String) -> Unit +) { + private val appContext = context.applicationContext + private val downloadManager = appContext.getSystemService(Context.DOWNLOAD_SERVICE) as DownloadManager + private val prefs = appContext.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + private val mainHandler = Handler(Looper.getMainLooper()) + private var receiverRegistered = false + private var activeDownloadId = -1L + private var activeDownloadUrl = "" + private var activeExpectedSha256 = "" + private var activeFileName = "" + private var progressTask: Runnable? = null + private var lastProgressPercent = -1 + private var lastProgressLogAt = 0L + private var lastProgressBytes = -1L + private var lastProgressBytesAt = 0L + + private data class DownloadSnapshot( + val exists: Boolean = false, + val status: Int = 0, + val reason: Int = -1, + val soFar: Long = 0L, + val total: Long = 0L, + val url: String = "" + ) + + private val downloadReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + if (intent?.action != DownloadManager.ACTION_DOWNLOAD_COMPLETE) return + val id = intent.getLongExtra(DownloadManager.EXTRA_DOWNLOAD_ID, -1L) + if (id <= 0 || id != activeDownloadId) return + handleDownloadComplete(id) + } + } + + fun register() { + if (receiverRegistered) return + val filter = IntentFilter(DownloadManager.ACTION_DOWNLOAD_COMPLETE) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + appContext.registerReceiver(downloadReceiver, filter, Context.RECEIVER_NOT_EXPORTED) + } else { + appContext.registerReceiver(downloadReceiver, filter) + } + receiverRegistered = true + recoverTrackedDownload() + } + + fun release() { + if (!receiverRegistered) return + runCatching { appContext.unregisterReceiver(downloadReceiver) } + receiverRegistered = false + stopProgressPolling() + } + + fun downloadAndInstall(apkUrl: String, versionName: String, expectedSha256: String = ""): Boolean { + val url = apkUrl.trim() + if (url.isBlank()) { + onLog("OTA: missing apk url") + return false + } + val expected = expectedSha256.trim().lowercase() + recoverTrackedDownload() + val existing = findDownloadByUrl(url) + if (existing > 0) { + val snapshot = queryDownload(existing) + when (snapshot.status) { + DownloadManager.STATUS_SUCCESSFUL -> { + onLog("OTA: 发现已下载完成任务，直接安装 id=$existing") + activeDownloadId = existing + activeDownloadUrl = url + activeExpectedSha256 = expected + persistTrackedDownload() + handleDownloadComplete(existing) + return true + } + DownloadManager.STATUS_PENDING, + DownloadManager.STATUS_PAUSED, + DownloadManager.STATUS_RUNNING -> { + activeDownloadId = existing + activeDownloadUrl = url + activeExpectedSha256 = expected + if (activeFileName.isBlank()) { + activeFileName = buildStableFileName(versionName) + } + persistTrackedDownload() + onLog("OTA: 继续已有下载任务 id=$existing") + startProgressPolling(existing) + return true + } + } + if (snapshot.status == DownloadManager.STATUS_FAILED) { + onLog("OTA: 清理失败下载任务 id=$existing 后重试") + runCatching { downloadManager.remove(existing) } + if (activeDownloadId == existing) { + clearTrackedDownload() + } + } + } + val fileName = buildStableFileName(versionName) + val req = DownloadManager.Request(Uri.parse(url)) + .setTitle("AI Glasses 更新包") + .setDescription("下载并安装 $versionName") + .setAllowedOverMetered(true) + .setAllowedOverRoaming(true) + .setMimeType("application/vnd.android.package-archive") + .setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED) + .setDestinationInExternalFilesDir(appContext, Environment.DIRECTORY_DOWNLOADS, fileName) + if (activeDownloadId > 0 && activeDownloadUrl != url) { + onLog("OTA: 切换到新下载地址，取消旧任务 id=$activeDownloadId") + runCatching { downloadManager.remove(activeDownloadId) } + } + stopProgressPolling() + resetProgressTracking() + activeDownloadUrl = url + activeExpectedSha256 = expected + activeFileName = fileName + activeDownloadId = runCatching { downloadManager.enqueue(req) } + .onFailure { onLog("OTA: download enqueue failed: ${it.message}") } + .getOrDefault(-1L) + if (activeDownloadId <= 0) return false + persistTrackedDownload() + onLog("OTA: 开始下载更新包 id=$activeDownloadId") + onLog("OTA: 下载地址 ${url.take(120)}") + startProgressPolling(activeDownloadId) + return true + } + + private fun handleDownloadComplete(downloadId: Long) { + stopProgressPolling() + val cursor = downloadManager.query(DownloadManager.Query().setFilterById(downloadId)) + cursor.use { c -> + if (!c.moveToFirst()) { + onLog("OTA: 下载任务不存在 id=$downloadId") + clearTrackedDownload() + return + } + val statusIdx = c.getColumnIndex(DownloadManager.COLUMN_STATUS) + if (statusIdx < 0) { + onLog("OTA: 无法读取下载状态") + clearTrackedDownload() + return + } + val status = c.getInt(statusIdx) + if (status != DownloadManager.STATUS_SUCCESSFUL) { + val reasonIdx = c.getColumnIndex(DownloadManager.COLUMN_REASON) + val reason = if (reasonIdx >= 0) c.getInt(reasonIdx) else -1 + onLog("OTA: 下载失败 status=$status reason=${reasonToText(reason)}($reason)") + clearTrackedDownload() + return + } + } + onLog("OTA: 下载完成 id=$downloadId") + val uri = downloadManager.getUriForDownloadedFile(downloadId) + if (uri == null) { + onLog("OTA: 找不到已下载文件 URI") + clearTrackedDownload() + return + } + if (!verifyDownloadedApkSha256(uri, activeExpectedSha256)) { + clearTrackedDownload() + return + } + if (!canInstallPackages()) { + openInstallPermissionSettings() + onLog("OTA: 下载完成，请允许本应用安装未知来源后再次点击更新") + persistTrackedDownload() + return + } + val installUri = materializeInstallUri(uri, activeFileName) + if (installUri == null) { + onLog("OTA: 无法准备安装包") + clearTrackedDownload() + return + } + val ok = installApk(installUri) + onLog(if (ok) "OTA: 已拉起安装流程" else "OTA: 拉起安装失败") + clearTrackedDownload() + } + + private fun startProgressPolling(downloadId: Long) { + stopProgressPolling() + val task = object : Runnable { + override fun run() { + if (activeDownloadId != downloadId || activeDownloadId <= 0) return + val keep = emitDownloadProgress(downloadId) + if (!keep) return + mainHandler.postDelayed(this, 1000L) + } + } + progressTask = task + mainHandler.post(task) + } + + private fun stopProgressPolling() { + progressTask?.let { mainHandler.removeCallbacks(it) } + progressTask = null + } + + private fun emitDownloadProgress(downloadId: Long): Boolean { + val cursor = downloadManager.query(DownloadManager.Query().setFilterById(downloadId)) + cursor.use { c -> + if (!c.moveToFirst()) { + onLog("OTA: 下载任务丢失 id=$downloadId") + clearTrackedDownload() + return false + } + val statusIdx = c.getColumnIndex(DownloadManager.COLUMN_STATUS) + val soFarIdx = c.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR) + val totalIdx = c.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES) + val reasonIdx = c.getColumnIndex(DownloadManager.COLUMN_REASON) + if (statusIdx < 0 || soFarIdx < 0 || totalIdx < 0) { + return true + } + val status = c.getInt(statusIdx) + val soFar = c.getLong(soFarIdx).coerceAtLeast(0L) + val total = c.getLong(totalIdx).coerceAtLeast(0L) + val percent = if (total > 0L) { + ((soFar * 100L) / total).toInt().coerceIn(0, 100) + } else { + -1 + } + val now = SystemClock.elapsedRealtime() + when { + soFar > lastProgressBytes -> { + lastProgressBytes = soFar + lastProgressBytesAt = now + } + lastProgressBytes < 0L -> { + lastProgressBytes = soFar + lastProgressBytesAt = now + } + } + val shouldLog = when { + status == DownloadManager.STATUS_RUNNING && percent >= 0 -> + (percent != lastProgressPercent && (percent % 2 == 0 || percent >= 98)) || + (now - lastProgressLogAt >= 4_000L) + status == DownloadManager.STATUS_RUNNING -> + now - lastProgressLogAt >= 3_000L + status == DownloadManager.STATUS_PENDING || status == DownloadManager.STATUS_PAUSED -> + now - lastProgressLogAt >= 3000L + else -> false + } + if (shouldLog) { + lastProgressLogAt = now + if (percent >= 0) { + lastProgressPercent = percent + onLog( + "OTA: 下载进度 $percent% (${formatBytes(soFar)}/${formatBytes(total)}) status=${statusToText(status)}" + ) + } else { + val reason = if (reasonIdx >= 0) c.getInt(reasonIdx) else -1 + onLog( + if (status == DownloadManager.STATUS_RUNNING) { + "OTA: 下载中 ${formatBytes(soFar)} (总大小未知)" + } else { + "OTA: 下载状态=${statusToText(status)} reason=${reasonToText(reason)} ${formatBytes(soFar)}" + } + ) + } + } + return when (status) { + DownloadManager.STATUS_PENDING, DownloadManager.STATUS_PAUSED, DownloadManager.STATUS_RUNNING -> true + DownloadManager.STATUS_SUCCESSFUL -> { + handleDownloadComplete(downloadId) + false + } + DownloadManager.STATUS_FAILED -> { + val reason = if (reasonIdx >= 0) c.getInt(reasonIdx) else -1 + onLog("OTA: 下载失败 reason=${reasonToText(reason)}($reason)") + clearTrackedDownload() + false + } + else -> true + } + } + } + + private fun recoverTrackedDownload() { + if (activeDownloadId <= 0L) { + activeDownloadId = prefs.getLong(KEY_DOWNLOAD_ID, -1L) + activeDownloadUrl = prefs.getString(KEY_DOWNLOAD_URL, "") ?: "" + activeExpectedSha256 = prefs.getString(KEY_EXPECTED_SHA256, "") ?: "" + activeFileName = prefs.getString(KEY_FILE_NAME, "") ?: "" + } + if (activeDownloadId <= 0L) return + val snapshot = queryDownload(activeDownloadId) + if (!snapshot.exists) { + clearTrackedDownload() + return + } + if (activeDownloadUrl.isBlank()) { + activeDownloadUrl = snapshot.url + } + when (snapshot.status) { + DownloadManager.STATUS_PENDING, + DownloadManager.STATUS_PAUSED, + DownloadManager.STATUS_RUNNING -> { + onLog("OTA: 恢复下载任务 id=$activeDownloadId") + persistTrackedDownload() + resetProgressTracking(snapshot.soFar) + startProgressPolling(activeDownloadId) + } + DownloadManager.STATUS_SUCCESSFUL -> { + onLog("OTA: 检测到已完成下载任务，继续安装") + handleDownloadComplete(activeDownloadId) + } + DownloadManager.STATUS_FAILED -> { + onLog( + "OTA: 上次下载任务已失败 reason=${reasonToText(snapshot.reason)}(${snapshot.reason})" + ) + clearTrackedDownload() + } + else -> { + persistTrackedDownload() + } + } + } + + private fun findDownloadByUrl(url: String): Long { + if (activeDownloadId > 0L && activeDownloadUrl == url) { + val active = queryDownload(activeDownloadId) + if (active.exists) return activeDownloadId + } + val savedId = prefs.getLong(KEY_DOWNLOAD_ID, -1L) + val savedUrl = prefs.getString(KEY_DOWNLOAD_URL, "") ?: "" + if (savedId > 0L && savedUrl == url) { + val saved = queryDownload(savedId) + if (saved.exists) return savedId + } + val query = DownloadManager.Query().setFilterByStatus( + DownloadManager.STATUS_PENDING or + DownloadManager.STATUS_PAUSED or + DownloadManager.STATUS_RUNNING or + DownloadManager.STATUS_SUCCESSFUL + ) + val cursor = downloadManager.query(query) + var latestId = -1L + cursor.use { c -> + val idIdx = c.getColumnIndex(DownloadManager.COLUMN_ID) + val urlIdx = c.getColumnIndex(DownloadManager.COLUMN_URI) + if (idIdx < 0 || urlIdx < 0) return@use + while (c.moveToNext()) { + val itemUrl = c.getString(urlIdx).orEmpty() + if (itemUrl != url) continue + val id = c.getLong(idIdx) + if (id > latestId) latestId = id + } + } + return latestId + } + + private fun queryDownload(downloadId: Long): DownloadSnapshot { + if (downloadId <= 0L) return DownloadSnapshot() + val cursor = downloadManager.query(DownloadManager.Query().setFilterById(downloadId)) + cursor.use { c -> + if (!c.moveToFirst()) return DownloadSnapshot() + val statusIdx = c.getColumnIndex(DownloadManager.COLUMN_STATUS) + val reasonIdx = c.getColumnIndex(DownloadManager.COLUMN_REASON) + val soFarIdx = c.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR) + val totalIdx = c.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES) + val urlIdx = c.getColumnIndex(DownloadManager.COLUMN_URI) + return DownloadSnapshot( + exists = true, + status = if (statusIdx >= 0) c.getInt(statusIdx) else 0, + reason = if (reasonIdx >= 0) c.getInt(reasonIdx) else -1, + soFar = if (soFarIdx >= 0) c.getLong(soFarIdx) else 0L, + total = if (totalIdx >= 0) c.getLong(totalIdx) else 0L, + url = if (urlIdx >= 0) c.getString(urlIdx).orEmpty() else "" + ) + } + } + + private fun persistTrackedDownload() { + if (activeDownloadId <= 0L) return + prefs.edit() + .putLong(KEY_DOWNLOAD_ID, activeDownloadId) + .putString(KEY_DOWNLOAD_URL, activeDownloadUrl) + .putString(KEY_EXPECTED_SHA256, activeExpectedSha256) + .putString(KEY_FILE_NAME, activeFileName) + .apply() + } + + private fun clearTrackedDownload() { + activeDownloadId = -1L + activeDownloadUrl = "" + activeExpectedSha256 = "" + activeFileName = "" + resetProgressTracking() + prefs.edit() + .remove(KEY_DOWNLOAD_ID) + .remove(KEY_DOWNLOAD_URL) + .remove(KEY_EXPECTED_SHA256) + .remove(KEY_FILE_NAME) + .apply() + } + + private fun buildStableFileName(versionName: String): String { + val safeName = versionName.ifBlank { "latest" }.replace(Regex("[^A-Za-z0-9._-]"), "_") + return "ai-glasses-$safeName.apk" + } + + private fun resetProgressTracking(initialBytes: Long = -1L) { + lastProgressPercent = -1 + lastProgressLogAt = 0L + lastProgressBytes = initialBytes + lastProgressBytesAt = if (initialBytes >= 0L) SystemClock.elapsedRealtime() else 0L + } + + private fun verifyDownloadedApkSha256(uri: Uri, expectedSha256: String): Boolean { + if (expectedSha256.isBlank()) return true + val digest = runCatching { + val md = MessageDigest.getInstance("SHA-256") + appContext.contentResolver.openInputStream(uri)?.use { input -> + val buffer = ByteArray(16 * 1024) + while (true) { + val n = input.read(buffer) + if (n <= 0) break + md.update(buffer, 0, n) + } + } ?: return false + md.digest().joinToString("") { "%02x".format(it) } + }.onFailure { + onLog("OTA: 校验失败 ${it.message}") + }.getOrNull() ?: return false + if (digest != expectedSha256) { + onLog("OTA: 文件校验不匹配 expected=${expectedSha256.take(10)} actual=${digest.take(10)}") + return false + } + onLog("OTA: 文件校验通过") + return true + } + + private fun installApk(uri: Uri): Boolean { + return runCatching { + val intent = Intent(Intent.ACTION_INSTALL_PACKAGE).apply { + data = uri + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION) + putExtra(Intent.EXTRA_NOT_UNKNOWN_SOURCE, true) + putExtra(Intent.EXTRA_RETURN_RESULT, false) + } + intent.resolveActivity(appContext.packageManager) + ?: throw IllegalStateException("no package installer activity") + appContext.startActivity(intent) + true + }.onFailure { + onLog("OTA: 安装 Intent 失败 ${it.message}") + }.getOrDefault(false) + } + + private fun materializeInstallUri(sourceUri: Uri, fileName: String): Uri? { + return runCatching { + val otaDir = File(appContext.cacheDir, "ota").apply { mkdirs() } + val apkFile = File(otaDir, fileName.ifBlank { "ai-glasses-update.apk" }) + appContext.contentResolver.openInputStream(sourceUri)?.use { input -> + FileOutputStream(apkFile, false).use { output -> + input.copyTo(output) + } + } ?: return null + FileProvider.getUriForFile( + appContext, + "${appContext.packageName}.fileprovider", + apkFile + ) + }.onFailure { + onLog("OTA: 准备安装包失败 ${it.message}") + }.getOrNull() + } + + private fun formatBytes(value: Long): String { + if (value < 1024L) return "${value}B" + val kb = value / 1024.0 + if (kb < 1024.0) return String.format("%.1fKB", kb) + val mb = kb / 1024.0 + if (mb < 1024.0) return String.format("%.1fMB", mb) + val gb = mb / 1024.0 + return String.format("%.2fGB", gb) + } + + private fun reasonToText(reason: Int): String { + return when (reason) { + DownloadManager.ERROR_CANNOT_RESUME -> "CANNOT_RESUME" + DownloadManager.ERROR_DEVICE_NOT_FOUND -> "DEVICE_NOT_FOUND" + DownloadManager.ERROR_FILE_ALREADY_EXISTS -> "FILE_ALREADY_EXISTS" + DownloadManager.ERROR_FILE_ERROR -> "FILE_ERROR" + DownloadManager.ERROR_HTTP_DATA_ERROR -> "HTTP_DATA_ERROR" + DownloadManager.ERROR_INSUFFICIENT_SPACE -> "INSUFFICIENT_SPACE" + DownloadManager.ERROR_TOO_MANY_REDIRECTS -> "TOO_MANY_REDIRECTS" + DownloadManager.ERROR_UNHANDLED_HTTP_CODE -> "UNHANDLED_HTTP_CODE" + DownloadManager.ERROR_UNKNOWN -> "UNKNOWN" + DownloadManager.PAUSED_QUEUED_FOR_WIFI -> "PAUSED_QUEUED_FOR_WIFI" + DownloadManager.PAUSED_WAITING_FOR_NETWORK -> "PAUSED_WAITING_FOR_NETWORK" + DownloadManager.PAUSED_WAITING_TO_RETRY -> "PAUSED_WAITING_TO_RETRY" + DownloadManager.PAUSED_UNKNOWN -> "PAUSED_UNKNOWN" + else -> "OTHER" + } + } + + private fun statusToText(status: Int): String { + return when (status) { + DownloadManager.STATUS_PENDING -> "PENDING" + DownloadManager.STATUS_RUNNING -> "RUNNING" + DownloadManager.STATUS_PAUSED -> "PAUSED" + DownloadManager.STATUS_SUCCESSFUL -> "SUCCESSFUL" + DownloadManager.STATUS_FAILED -> "FAILED" + else -> "UNKNOWN" + } + } + + private fun canInstallPackages(): Boolean { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + appContext.packageManager.canRequestPackageInstalls() + } else { + true + } + } + + private fun openInstallPermissionSettings() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + runCatching { + val intent = Intent( + Settings.ACTION_MANAGE_UNKNOWN_APP_SOURCES, + Uri.parse("package:${appContext.packageName}") + ).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + appContext.startActivity(intent) + } + } + + private companion object { + const val PREFS_NAME = "ota_updater_prefs" + const val KEY_DOWNLOAD_ID = "download_id" + const val KEY_DOWNLOAD_URL = "download_url" + const val KEY_EXPECTED_SHA256 = "expected_sha256" + const val KEY_FILE_NAME = "file_name" + } +} diff --git a/android-app/app/src/main/res/values/strings.xml b/android-app/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..7a31f37 --- /dev/null +++ b/android-app/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + StoryForge AI + diff --git a/android-app/app/src/main/res/values/themes.xml b/android-app/app/src/main/res/values/themes.xml new file mode 100644 index 0000000..3b1e9d4 --- /dev/null +++ b/android-app/app/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + +