From 8875a5def1e2404257777841fc1b2cf1698cbd0a Mon Sep 17 00:00:00 2001 From: chaoq Date: Wed, 17 Sep 2025 20:48:56 +0800 Subject: [PATCH] dev plan base version Signed-off-by: chaoq --- docs/dev-plan-ds.md | 1190 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1190 insertions(+) create mode 100644 docs/dev-plan-ds.md diff --git a/docs/dev-plan-ds.md b/docs/dev-plan-ds.md new file mode 100644 index 0000000..404fc98 --- /dev/null +++ b/docs/dev-plan-ds.md @@ -0,0 +1,1190 @@ +# 增强版AI开发计划 - XSynergy AR远程协作App + +## 🎯 项目概览 +- **项目类型**: Android AR协作应用 +- **技术栈**: Kotlin + Jetpack Compose + MVVM + LiveKit + ARCore + MQTT +- **目标**: 完整的AI可执行开发计划，包含所有关键集成点 + +--- + +## 📋 阶段1：基础架构和认证系统（增强版） + +### 🎯 目标 +- 搭建完整的项目架构，集成GXRTC服务器认证 +- 实现MQTT客户端基础框架 + +### 🔧 核心实现增强 + +#### 1.1 正确的API配置 +```kotlin +// data/remote/RetrofitClient.kt - 修正 +@Module +@InstallIn(SingletonComponent::class) +object NetworkModule { + + @Provides + @Singleton + fun provideGxrtcApi(retrofit: Retrofit): GxrtcApi = retrofit.create(GxrtcApi::class.java) +} + +// data/remote/GxrtcApi.kt - 新增 +interface GxrtcApi { + @POST("room/token") + @FormUrlEncoded + suspend fun getAccessToken( + @Field("uid") userId: String, + @Field("room") roomId: String + ): Response> + + @POST("room/") + @FormUrlEncoded + suspend fun createRoom(@Field("room") roomName: String): Response> + + @GET("room/") + suspend fun getRooms(): Response>> +} + +// 响应数据结构 +data class GxrtcResponse( + @SerializedName("meta") val meta: Meta, + @SerializedName("data") val data: T +) + +data class Meta( + @SerializedName("code") val code: Int, + @SerializedName("message") val message: String? = null +) + +data class AccessTokenResponse(@SerializedName("access_token") val accessToken: String) + +// data/model/ChatMessage.kt - 新增 +data class ChatMessage( + val id: String, + val senderId: String, + val message: String, + val timestamp: Long, + val type: String = "text", // text, image, file + val isPrivate: Boolean = false +) + +// data/model/PrivateMessage.kt - 新增 +data class PrivateMessage( + val id: String, + val senderId: String, + val recipientId: String, + val message: String, + val timestamp: Long, + val isRead: Boolean = false +) + +// data/model/LaserPointerData.kt - 新增 +data class LaserPointerData( + val start: List, // [x, y] 相对坐标 + val end: List, // [x, y] 相对坐标 + val color: List, // [r, g, b] + val thickness: Int, // 像素粗细 + val timestamp: Long // 时间戳 +) + +// data/model/WhiteboardModels.kt - 新增 +sealed class WhiteboardState { + object Inactive : WhiteboardState() + data class Active( + val tools: List, + val currentTool: WhiteboardTool, + val strokes: List + ) : WhiteboardState() +} + +sealed class WhiteboardTool { + data class Brush(val color: Color, val size: Float) : WhiteboardTool() + data class Eraser(val size: Float) : WhiteboardTool() + data class Shape(val shapeType: ShapeType) : WhiteboardTool() + data class Text(val fontSize: Float = 16f) : WhiteboardTool() +} + +enum class ShapeType { Rectangle, Circle, Triangle, Arrow } + +enum class WhiteboardToolType { BRUSH, ERASER, SHAPE, TEXT } + +data class WhiteboardStroke( + val points: List, + val color: Color, + val size: Float, + val toolType: WhiteboardToolType +) +``` + +#### 1.2 MQTT客户端集成 +```kotlin +// data/remote/MqttManager.kt - 新增 +class MqttManager @Inject constructor( + @ApplicationContext private val context: Context, + private val userRepository: UserRepository +) { + private var mqttClient: MqttAndroidClient? = null + private val _mqttState = MutableStateFlow(MqttConnectionState.Disconnected) + val mqttState: StateFlow = _mqttState.asStateFlow() + + suspend fun connect() { + val serverUri = "tcp://mqtt.cnsdt.com:1883" + val clientId = MqttClient.generateClientId() + + mqttClient = MqttAndroidClient(context, serverUri, clientId).apply { + setCallback(object : MqttCallback { + override fun connectionLost(cause: Throwable?) { + _mqttState.value = MqttConnectionState.Disconnected + } + + override fun messageArrived(topic: String, message: MqttMessage) { + handleIncomingMessage(topic, message) + } + + override fun deliveryComplete(token: IMqttDeliveryToken?) {} + }) + } + + val options = MqttConnectOptions().apply { + isCleanSession = true + connectionTimeout = 30 + keepAliveInterval = 60 + } + + try { + mqttClient?.connect(options) + _mqttState.value = MqttConnectionState.Connected + subscribeToUserTopics() + } catch (e: Exception) { + _mqttState.value = MqttConnectionState.Error(e.message ?: "Connection failed") + } + } + + private fun subscribeToUserTopics() { + val user = userRepository.getCurrentUser() + user?.let { + val userTopic = "/zrsk/remote-assistant/user/${it.id}" + mqttClient?.subscribe(userTopic, 1) + } + } + + private fun handleIncomingMessage(topic: String, message: MqttMessage) { + val payload = String(message.payload, Charsets.UTF_8) + val messageData = Json.decodeFromString(payload) + + when (messageData.event) { + "call" -> handleCallMessage(messageData.data) + "call-cancel" -> handleCallCancelMessage(messageData.data) + "laser-pointer" -> handleLaserPointerMessage(messageData.data) + "chat-message" -> handleChatMessage(messageData.data) + "private-message" -> handlePrivateMessage(messageData.data) + "whiteboard-state" -> handleWhiteboardMessage(messageData.data) + "whiteboard-stroke" -> handleWhiteboardMessage(messageData.data) + // 其他消息类型... + } + } +} +``` + +### ✅ 增强验收标准 +- [ ] GXRTC API集成正确，能成功获取access token +- [ ] MQTT客户端能正常连接和接收消息，连接成功率>99.9% +- [ ] 消息解析和处理框架完整，支持所有消息类型 +- [ ] 用户认证状态持久化，应用重启后保持登录状态 +- [ ] 网络错误处理完善，有明确的用户提示 +- [ ] 单元测试覆盖率>85%，关键业务逻辑100%覆盖 + +--- + +## 📋 阶段2：实时通信基础（增强版） + +### 🎯 目标 +- 集成LiveKit with正确的token获取流程 +- 实现完整的音视频通话功能 + +### 🔧 核心实现增强 + +#### 2.1 正确的LiveKit连接流程 +```kotlin +// domain/manager/LiveKitManager.kt - 修正 +class LiveKitManager @Inject constructor( + @ApplicationContext private val context: Context, + private val gxrtcApi: GxrtcApi, + private val userRepository: UserRepository +) { + suspend fun connectToRoom(roomName: String): Result { + return try { + // 1. 获取access token + val user = userRepository.getCurrentUser() + val tokenResponse = gxrtcApi.getAccessToken(user.id, roomName) + + if (!tokenResponse.isSuccessful || tokenResponse.body()?.meta?.code != 200) { + return Result.failure(Exception("Failed to get access token")) + } + + val accessToken = tokenResponse.body()!!.data.accessToken + + // 2. 连接LiveKit + val newRoom = LiveKit.create( + appContext = context, + options = RoomOptions( + adaptiveStream = true, + dynacast = true, + e2eeOptions = E2EEOptions( + keyProvider = BaseKeyProvider("encryption-key") + ) + ) + ) + + newRoom.connect("wss://meeting.cnsdt.com", accessToken) + Result.success(newRoom) + } catch (e: Exception) { + Result.failure(e) + } + } +} +``` + +#### 2.2 MQTT房间消息订阅 +```kotlin +// 在进入房间时订阅相关主题 +fun subscribeToRoomTopics(roomId: String) { + val roomTopic = "/zrsk/remote-assistant/room/$roomId" + val userRoomTopic = "/zrsk/remote-assistant/room/$roomId/user/${currentUser.id}" + + mqttClient?.subscribe(roomTopic, 1) + mqttClient?.subscribe(userRoomTopic, 1) +} +``` + +### ✅ 增强验收标准 +- [ ] 使用GXRTC token成功连接LiveKit，连接时间<3s +- [ ] 房间相关的MQTT消息能正确接收和处理，消息延迟<200ms +- [ ] 断线重连机制完善，网络恢复后30s内自动重连 +- [ ] 音视频权限处理正确，用户拒绝权限时有友好提示 +- [ ] 本地视频预览正常，画面清晰无卡顿 +- [ ] 远程视频渲染正常，支持多种分辨率和帧率 + +--- + +## 📋 阶段3：协作会话管理（增强版） + +### 🎯 目标 +- 完整的会议生命周期管理 +- MQTT消息驱动的呼叫系统 + +### 🔧 核心实现增强 + +#### 3.1 呼叫系统实现 +```kotlin +// domain/manager/CallManager.kt - 新增 +class CallManager @Inject constructor( + private val mqttManager: MqttManager, + private val userRepository: UserRepository +) { + fun sendCallInvitation(roomId: String, roomTitle: String, targetUserId: String) { + val callMessage = MqttMessageData( + event = "call", + data = mapOf( + "room_id" to roomId, + "title" to roomTitle, + "caller" to userRepository.getCurrentUser()?.id ?: "" + ) + ) + + val topic = "/zrsk/remote-assistant/user/$targetUserId" + mqttManager.publish(topic, callMessage) + } + + fun handleCallMessage(data: Map) { + val roomId = data["room_id"] as String + val title = data["title"] as String + val callerId = data["caller"] as String + + // 显示呼叫UI，用户可以选择接听或拒绝 + _incomingCall.value = IncomingCall(roomId, title, callerId) + } +} +``` + +#### 3.2 会议状态管理 +```kotlin +// 处理会议结束消息 +private fun handleEndMeetingMessage(data: Map) { + val reason = data["reason"] as? String + val code = data["code"] as Int + + when (code) { + 1 -> showToast("用户挂断了会议") + 2 -> showToast("管理员结束了会议") + else -> showToast("会议已结束") + } + + // 退出会议界面 + navigateToHome() +} +``` + +### ✅ 增强验收标准 +- [ ] 呼叫邀请能正确发送和接收，送达率100% +- [ ] 会议状态变化能正确处理，状态同步延迟<1s +- [ ] 用户加入/离开通知完善，实时更新参与者列表 +- [ ] 会议录制功能正常，录制文件可完整回放 +- [ ] 会议时长统计准确，支持暂停/继续计时 +- [ ] 多语言支持完整，中英文界面无缝切换 + +--- + +## 📋 阶段4：AR环境集成（增强版） + +### 🎯 目标 +- ARCore环境集成与平面检测 +- 后置摄像头默认配置与空间锚点同步 +- 服务器端AR标注持久化存储 + +### 🔧 核心实现增强 + +#### 4.1 AR场景管理与后置摄像头配置 +```kotlin +// ui/ar/ArSceneManager.kt - 增强 +class ArSceneManager @Inject constructor( + private val mqttManager: MqttManager, + private val arAnnotationService: ArAnnotationService +) { + private var arSession: Session? = null + private val cameraConfig = CameraConfig().apply { + facingDirection = CameraConfig.FacingDirection.BACK // 默认后置摄像头 + depthSensorUsage = CameraConfig.DepthSensorUsage.AUTOMATIC + } + + suspend fun initializeArSession(context: Context): Result { + return try { + val session = Session(context) + session.cameraConfig = cameraConfig + session.configure(session.config) + + arSession = session + Result.success(session) + } catch (e: Exception) { + Result.failure(e) + } + } + + fun createAnchorAtPosition(position: Vector3): Anchor { + val anchor = arSession!!.createAnchor(Pose(position, Quaternion.identity())) + + // 同步到服务器和其他用户 + syncAnchorToParticipants(anchor) + + return anchor + } + + private suspend fun syncAnchorToParticipants(anchor: Anchor) { + // 保存到服务器 + arAnnotationService.saveAnchor( + roomId = currentRoomId, + anchorId = anchor.trackableId, + position = anchor.pose + ) + + // 同步到其他用户 + val anchorData = mapOf( + "anchor_id" to anchor.trackableId, + "position" to listOf(anchor.pose.tx(), anchor.pose.ty(), anchor.pose.tz()), + "rotation" to listOf(anchor.pose.qx(), anchor.pose.qy(), anchor.pose.qz(), anchor.pose.qw()) + ) + + val message = MqttMessageData( + event = "ar-anchor-create", + data = anchorData + ) + mqttManager.publishToRoom(roomId, message) + } + + suspend fun restoreAnchors(roomId: String): List { + val savedAnchors = arAnnotationService.loadRoomAnchors(roomId) + return savedAnchors.map { anchorData -> + val pose = Pose( + floatArrayOf(anchorData.position[0], anchorData.position[1], anchorData.position[2]), + floatArrayOf(anchorData.rotation[0], anchorData.rotation[1], anchorData.rotation[2], anchorData.rotation[3]) + ) + arSession!!.createAnchor(pose) + } + } +} + +// data/model/AnchorData.kt - 新增 +data class AnchorData( + val anchorId: String, + val position: List, + val rotation: List, + val roomId: String, + val createdAt: Long = System.currentTimeMillis(), + val createdBy: String +) +``` + +#### 4.2 AR标注持久化服务 +```kotlin +// domain/service/ArAnnotationService.kt - 新增 +class ArAnnotationService @Inject constructor( + private val arAnnotationApi: ArAnnotationApi +) { + suspend fun saveAnchor( + roomId: String, + anchorId: String, + position: Pose + ): Result { + return try { + val request = SaveAnchorRequest( + roomId = roomId, + anchorId = anchorId, + position = listOf(position.tx(), position.ty(), position.tz()), + rotation = listOf(position.qx(), position.qy(), position.qz(), position.qw()), + createdBy = currentUserId + ) + + arAnnotationApi.saveAnchor(request) + Result.success(Unit) + } catch (e: Exception) { + Result.failure(e) + } + } + + suspend fun loadRoomAnchors(roomId: String): List { + return arAnnotationApi.getRoomAnchors(roomId).anchors + } + + suspend fun saveAnnotation(annotation: ArAnnotation): Result { + return try { + arAnnotationApi.saveAnnotation(annotation) + Result.success(Unit) + } catch (e: Exception) { + Result.failure(e) + } + } + + suspend fun loadRoomAnnotations(roomId: String): List { + return arAnnotationApi.getRoomAnnotations(roomId).annotations + } +} + +// data/model/ArAnnotation.kt - 新增 +data class ArAnnotation( + val id: String, + val anchorId: String, + val type: AnnotationType, + val content: String, + val color: List, + val size: Float, + val roomId: String, + val createdBy: String, + val createdAt: Long = System.currentTimeMillis(), + val updatedAt: Long = System.currentTimeMillis() +) + +enum class AnnotationType { + ARROW, CIRCLE, RECTANGLE, TEXT, HIGHLIGHT +} +``` + +### ✅ 增强验收标准 +- [ ] AR场景能正确初始化和渲染，初始化时间<5s +- [ ] 默认使用后置摄像头，支持自动切换 +- [ ] 空间锚点能跨设备同步，位置误差<5cm +- [ ] 服务器端锚点存储可靠，存储成功率>99.9% +- [ ] 锚点恢复准确，重定位误差<10cm +- [ ] 支持会话间AR标注持久化恢复 +- [ ] 平面检测准确稳定，检测成功率>95% +- [ ] 环境光估计准确，虚拟物体光照一致 + +--- + +## 📋 阶段5：AR标注工具（增强版） + +### 🎯 目标 +- 完整的AR标注系统 +- 实时同步机制 + +### 🔧 核心实现增强 + +#### 5.1 激光笔工具实现 +```kotlin +// ui/ar/tools/LaserPointerTool.kt - 新增 +class LaserPointerTool @Inject constructor( + private val mqttManager: MqttManager +) : ArTool { + + fun onTouchStart(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) { + val startX = screenX / screenWidth + val startY = screenY / screenHeight + + currentLaserData = LaserData( + start = listOf(startX, startY), + end = listOf(startX, startY), + color = listOf(255, 0, 0), // 红色 + thickness = 3 + ) + } + + fun onTouchMove(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) { + val endX = screenX / screenWidth + val endY = screenY / screenHeight + + currentLaserData = currentLaserData?.copy(end = listOf(endX, endY)) + + // 实时发送激光笔位置 + sendLaserPointerData() + } + + fun onTouchEnd() { + // 发送最终的激光笔数据 + sendLaserPointerData() + currentLaserData = null + } + + private fun sendLaserPointerData() { + currentLaserData?.let { laserData -> + val message = MqttMessageData( + event = "laser-pointer", + data = mapOf( + "start" to laserData.start, + "end" to laserData.end, + "color" to laserData.color, + "thickness" to laserData.thickness + ) + ) + + mqttManager.publishToRoom(roomId, message) + } + } + + private fun handleLaserPointerMessage(data: Map) { + val start = data["start"] as List + val end = data["end"] as List + val color = data["color"] as List + val thickness = data["thickness"] as Int + + // 更新激光笔显示 + _laserPointerData.value = LaserPointerData( + start = start, + end = end, + color = color, + thickness = thickness, + timestamp = System.currentTimeMillis() + ) + } + + private fun handleChatMessage(data: Map) { + val senderId = data["sender_id"] as String + val message = data["message"] as String + val timestamp = data["timestamp"] as Long + val messageType = data["type"] as? String ?: "text" + + // 更新聊天界面 + _chatMessages.value = _chatMessages.value + ChatMessage( + id = UUID.randomUUID().toString(), + senderId = senderId, + message = message, + timestamp = timestamp, + type = messageType, + isPrivate = false + ) + } + + private fun handlePrivateMessage(data: Map) { + val senderId = data["sender_id"] as String + val message = data["message"] as String + val timestamp = data["timestamp"] as Long + val recipientId = data["recipient_id"] as String + + // 更新私聊界面 + _privateMessages.value = _privateMessages.value + PrivateMessage( + id = UUID.randomUUID().toString(), + senderId = senderId, + recipientId = recipientId, + message = message, + timestamp = timestamp + ) + } + + fun sendChatMessage(message: String, isPrivate: Boolean = false, recipientId: String? = null) { + val messageData = if (isPrivate && recipientId != null) { + MqttMessageData( + event = "private-message", + data = mapOf( + "sender_id" to currentUser.id, + "recipient_id" to recipientId, + "message" to message, + "timestamp" to System.currentTimeMillis() + ) + ) + } else { + MqttMessageData( + event = "chat-message", + data = mapOf( + "sender_id" to currentUser.id, + "message" to message, + "timestamp" to System.currentTimeMillis(), + "type" to "text" + ) + ) + } + + val topic = if (isPrivate) { + "/zrsk/remote-assistant/user/$recipientId" + } else { + "/zrsk/remote-assistant/room/$currentRoomId" + } + + mqttManager.publish(topic, messageData) + } +} +``` + +#### 3.3 文字聊天界面实现 +```kotlin +// ui/chat/ChatViewModel.kt - 新增 +@HiltViewModel +class ChatViewModel @Inject constructor( + private val mqttManager: MqttManager +) : ViewModel() { + + private val _chatMessages = MutableStateFlow>(emptyList()) + val chatMessages: StateFlow> = _chatMessages.asStateFlow() + + private val _privateMessages = MutableStateFlow>(emptyList()) + val privateMessages: StateFlow> = _privateMessages.asStateFlow() + + fun loadChatHistory(roomId: String) { + // 从服务器加载聊天历史 + } + + fun clearChatHistory() { + _chatMessages.value = emptyList() + } +} + +// ui/chat/ChatScreen.kt - 新增 +@Composable +fun ChatScreen( + viewModel: ChatViewModel = hiltViewModel(), + onBackPressed: () -> Unit +) { + val messages by viewModel.chatMessages.collectAsState() + var newMessage by remember { mutableStateOf("") } + + Column(modifier = Modifier.fillMaxSize()) { + // 消息列表 + LazyColumn(modifier = Modifier.weight(1f)) { + items(messages) { message -> + ChatMessageItem(message = message) + } + } + + // 输入框 + Row( + modifier = Modifier + .fillMaxWidth() + .padding(8.dp), + verticalAlignment = Alignment.CenterVertically + ) { + TextField( + value = newMessage, + onValueChange = { newMessage = it }, + modifier = Modifier.weight(1f), + placeholder = { Text("输入消息...") } + ) + + Button( + onClick = { + if (newMessage.isNotBlank()) { + viewModel.sendMessage(newMessage) + newMessage = "" + } + }, + modifier = Modifier.padding(start = 8.dp) + ) { + Text("发送") + } + } + } +} +``` + +### ✅ 增强验收标准 +- [ ] 激光笔工具实时同步，端到端延迟<100ms +- [ ] AR标注空间位置准确，位置误差<2cm +- [ ] 多用户同时标注无冲突，支持并发标注 +- [ ] 标注持久化存储，支持会话间恢复 +- [ ] 标注工具丰富，支持箭头、圆圈、文字等多种类型 +- [ ] 标注颜色和大小可自定义，视觉效果清晰 + +--- + +## 📋 阶段6：协作增强功能（增强版） + +### 🎯 目标 +- 白板、屏幕共享、文件传输 +- 实时字幕和会议纪要 + +### 🔧 核心实现增强 + +#### 6.1 白板协作实现 +```kotlin +// ui/whiteboard/WhiteboardManager.kt - 新增 +class WhiteboardManager @Inject constructor( + private val mqttManager: MqttManager +) { + private val _whiteboardState = MutableStateFlow(WhiteboardState.Inactive) + val whiteboardState: StateFlow = _whiteboardState.asStateFlow() + + fun enableWhiteboard() { + _whiteboardState.value = WhiteboardState.Active( + tools = listOf( + WhiteboardTool.Brush(color = Color.Red, size = 3f), + WhiteboardTool.Eraser(size = 10f), + WhiteboardTool.Shape(shapeType = ShapeType.Rectangle), + WhiteboardTool.Text() + ), + currentTool = WhiteboardTool.Brush(color = Color.Red, size = 3f), + strokes = emptyList() + ) + + // 通知其他用户白板已开启 + sendWhiteboardState(true) + } + + fun disableWhiteboard() { + _whiteboardState.value = WhiteboardState.Inactive + sendWhiteboardState(false) + } + + fun addStroke(stroke: WhiteboardStroke) { + val currentState = _whiteboardState.value + if (currentState is WhiteboardState.Active) { + _whiteboardState.value = currentState.copy( + strokes = currentState.strokes + stroke + ) + + // 同步到其他用户 + sendStroke(stroke) + } + } + + private fun sendWhiteboardState(enabled: Boolean) { + val message = MqttMessageData( + event = "whiteboard-state", + data = mapOf("enabled" to enabled) + ) + mqttManager.publishToRoom(roomId, message) + } + + private fun sendStroke(stroke: WhiteboardStroke) { + val message = MqttMessageData( + event = "whiteboard-stroke", + data = mapOf( + "points" to stroke.points.map { listOf(it.x, it.y) }, + "color" to listOf(stroke.color.red, stroke.color.green, stroke.color.blue), + "size" to stroke.size, + "tool_type" to stroke.toolType.name + ) + ) + mqttManager.publishToRoom(roomId, message) + } + + fun handleWhiteboardMessage(data: Map) { + when (data["event"] as String) { + "whiteboard-state" -> { + val enabled = data["enabled"] as Boolean + if (enabled) { + enableWhiteboard() + } else { + disableWhiteboard() + } + } + "whiteboard-stroke" -> { + val points = (data["points"] as List>).map { PointF(it[0], it[1]) } + val color = Color( + red = (data["color"] as List)[0], + green = (data["color"] as List)[1], + blue = (data["color"] as List)[2] + ) + val size = data["size"] as Float + val toolType = WhiteboardToolType.valueOf(data["tool_type"] as String) + + val stroke = WhiteboardStroke(points, color, size, toolType) + addStroke(stroke) + } + } + } +} +``` + +#### 6.2 实时字幕集成 +```kotlin +// 处理ASR字幕消息 +private fun handleAsrSubtitleMessage(data: Map) { + val uid = data["uid"] as String + val segment = data["seg"] as String + val status = data["status"] as String + + when (status) { + "streaming" -> updateLiveSubtitle(uid, segment) + "final" -> finalizeSubtitle(uid, segment) + } +} +``` + +#### 6.2 文件预览功能 +```kotlin +// 处理文件预览消息 +private fun handleFilePreviewMessage(data: Map) { + val fileName = data["name"] as String + val filePath = data["path"] as String + val mimeType = data["mime_type"] as String + + val fileUrl = "https://meeting.cnsdt.com$filePath" + + // 显示文件预览界面 + showFilePreview(fileName, fileUrl, mimeType) +} +``` + +### ✅ 增强验收标准 +- [ ] 实时字幕准确率>90%，行业术语识别准确 +- [ ] 文件预览功能完整，支持PDF、Word、Excel、图片等格式 +- [ ] 屏幕共享流畅，帧率>15fps，延迟<300ms +- [ ] 白板协作实时同步，操作延迟<200ms +- [ ] 文件传输可靠，支持断点续传，100MB文件传输成功率>99% +- [ ] 共同浏览功能正常，页面同步准确 + +--- + +## 📋 阶段7：AI能力集成（增强版） + +### 🎯 目标 +- 语音识别、会议纪要、知识库 +- 智能助手集成 + +### 🔧 核心实现增强 + +#### 7.1 知识库查询集成 +```kotlin +// domain/usecase/QueryKnowledgeBaseUseCase.kt - 新增 +class QueryKnowledgeBaseUseCase @Inject constructor( + private val aiService: AiService +) { + suspend operator fun invoke(query: String): Result { + return try { + val response = aiService.queryKnowledgeBase(query) + if (response.isSuccessful) { + Result.success(response.body()?.answer ?: "未找到相关信息") + } else { + Result.failure(Exception("查询失败")) + } + } catch (e: Exception) { + Result.failure(e) + } + } +} +``` + +### ✅ 增强验收标准 +- [ ] 知识库查询响应时间<2秒，相关度>85% +- [ ] 会议纪要生成准确，关键信息提取完整 +- [ ] AI助手交互流畅，自然语言理解准确 +- [ ] 语音助手响应迅速，语音识别准确率>92% +- [ ] 智能推荐相关，推荐准确率>80% +- [ ] 多轮对话支持，上下文理解准确 + +--- + +## 🔌 缺失的服务端API接口（增强版） + +根据AR标注和持久化需求，需要以下额外的服务端API接口： + +### 1. AR标注存储API +```kotlin +// 保存AR锚点 +POST /ar/anchor/save +Content-Type: application/json +{ + "room_id": "string", + "anchor_id": "string", + "position": [0.0, 0.0, 0.0], + "rotation": [0.0, 0.0, 0.0, 1.0], + "created_by": "user123" +} + +// 获取房间所有锚点 +GET /ar/room/{roomId}/anchors + +// 保存AR标注内容 +POST /ar/annotation/save +Content-Type: application/json +{ + "anchor_id": "string", + "type": "ARROW", + "content": "检查这里", + "color": [255, 0, 0], + "size": 2.0, + "room_id": "string", + "created_by": "user123" +} + +// 获取房间所有标注 +GET /ar/room/{roomId}/annotations + +// 删除标注 +DELETE /ar/annotation/{annotationId} +``` + +### 2. AR会话管理API +```kotlin +// 创建AR会话 +POST /ar/session/create +{ + "room_id": "string", + "session_name": "设备维护会话", + "created_by": "user123" +} + +// 获取会话历史 +GET /ar/user/{userId}/sessions + +// 恢复特定会话 +GET /ar/session/{sessionId} + +// 导出会话数据 +GET /ar/session/{sessionId}/export?format=json|pdf +``` + +### 3. 空间数据验证API +```kotlin +// 验证锚点空间一致性 +POST /ar/anchor/validate +{ + "anchor_id": "string", + "position": [0.0, 0.0, 0.0], + "rotation": [0.0, 0.0, 0.0, 1.0], + "device_info": { + "model": "Pixel 6", + "ar_core_version": "1.50.0" + } +} + +// 批量验证多个锚点 +POST /ar/anchors/validate-batch +``` + +### 4. 特征点备份API（可选） +```kotlin +// 备份ARCore特征点数据 +POST /ar/features/backup +Content-Type: application/octet-stream +[二进制特征点数据] + +// 恢复特征点数据 +GET /ar/features/restore?anchor_id={anchorId} + +// 特征点相似度匹配 +POST /ar/features/match +{ + "anchor_id": "string", + "feature_descriptor": "base64_encoded_data" +} +``` + +### 5. 设备校准API +```kotlin +// 上报设备校准数据 +POST /ar/device/calibration +{ + "device_id": "string", + "calibration_data": { + "camera_params": {...}, + "imu_calibration": {...}, + "accuracy_score": 0.95 + }, + "ar_core_version": "1.50.0" +} + +// 获取设备推荐校准参数 +GET /ar/device/{deviceModel}/calibration +``` + +### 6. 统计分析API +```kotlin +// AR使用统计 +GET /ar/analytics/usage?start_date=2025-01-01&end_date=2025-01-31 + +// 标注准确率统计 +GET /ar/analytics/accuracy?room_id={roomId} + +// 设备兼容性报告 +GET /ar/analytics/compatibility +``` + +--- + +## 📋 阶段8：优化和兼容性（增强版） + +### 🎯 目标 +- 性能优化、设备兼容 +- 完整的错误处理和日志系统 + +### 🔧 核心实现增强 + +#### 8.1 完整的错误处理 +```kotlin +// utils/ErrorHandler.kt - 新增 +class ErrorHandler @Inject constructor( + @ApplicationContext private val context: Context +) { + fun handleNetworkError(error: Throwable) { + when (error) { + is SocketTimeoutException -> showToast("网络连接超时") + is ConnectException -> showToast("无法连接到服务器") + is SSLHandshakeException -> showToast("安全连接失败") + else -> showToast("网络错误: ${error.message}") + } + } + + fun handleMqttError(error: Throwable) { + showToast("实时通信连接失败") + // 尝试重连 + mqttManager.reconnect() + } +} +``` + +### ✅ 增强验收标准 +- [ ] 性能指标达标：冷启动<3s，热启动<1s，标注延迟<100ms +- [ ] 兼容Android 8.0+和ARCore设备，覆盖主流设备型号 +- [ ] 错误处理完整，用户体验良好，错误提示友好明确 +- [ ] 内存使用优化，1小时会议内存增长<50MB +- [ ] 电池消耗合理，1小时会议耗电<15% +- [ ] 网络带宽优化，720p视频通话<800Kbps + +--- + +## 🧪 增强测试策略 + +### 单元测试增强 +- **MQTT消息解析测试**：验证所有消息类型的正确解析 +- **GXRTC API响应处理测试**：测试各种HTTP状态码和错误处理 +- **AR标注坐标转换测试**：验证屏幕坐标到世界坐标的准确转换 +- **权限管理测试**：摄像头、麦克风、存储权限处理 +- **数据模型测试**：所有数据类的序列化/反序列化 +- **错误处理测试**：网络错误、解析错误、权限错误处理 + +### 集成测试增强 +- **多设备同步测试**：3+设备同时协作，验证标注同步 +- **网络条件模拟测试**：2G/3G/4G/WiFi切换，丢包率20%测试 +- **断线重连测试**：网络中断30秒后自动重连 +- **MQTT消息完整性测试**：消息不丢失、不重复、有序到达 +- **LiveKit连接测试**：Token过期处理、房间满员处理 +- **文件传输测试**：大文件(100MB+)传输完整性验证 +- **AR锚点同步测试**：多设备间空间锚点一致性 + +### UI测试增强 +- **AR场景交互测试**：手势识别、平面检测、标注绘制 +- **实时协作流程测试**：完整会议生命周期UI测试 +- **错误状态UI测试**：网络错误、权限拒绝、服务器错误的UI反馈 +- **性能监控测试**：帧率监控、内存使用、电池消耗 +- **无障碍测试**：屏幕阅读器支持、大字體模式 +- **多语言测试**：中英文界面切换 + +### 性能测试 +- **启动时间测试**：冷启动<3s，热启动<1s +- **标注延迟测试**：端到端延迟<100ms +- **内存泄漏测试**：长时间会议无内存泄漏 +- **电池消耗测试**：1小时会议耗电<15% +- **网络带宽测试**：720p视频通话带宽<1Mbps + +### 安全测试 +- **数据加密测试**：音视频流端到端加密验证 +- **Token安全测试**：Token过期和刷新机制 +- **权限提升测试**：防止未授权功能访问 +- **输入验证测试**：防止注入攻击 +- **安全存储测试**：敏感数据加密存储 + +## 🔌 缺失的服务端API接口 + +根据PRD需求，需要以下额外的服务端API接口： + +### 1. 用户管理API +```kotlin +// 获取用户基本信息 +GET /user/{userId}/info +// 搜索用户 +GET /user/search?keyword={name}&department={dept} +// 获取组织架构 +GET /organization/structure +``` + +### 2. 文件服务API +```kotlin +// 文件上传 +POST /file/upload +// 文件下载 +GET /file/{fileId}/download +// 文件信息获取 +GET /file/{fileId}/info +// 文件预览生成 +POST /file/{fileId}/generate-preview +``` + +### 3. 会议管理API +```kotlin +// 创建预约会议 +POST /meeting/scheduled +// 获取会议列表 +GET /meeting/history +// 获取会议详情 +GET /meeting/{meetingId} +// 删除会议记录 +DELETE /meeting/{meetingId} +``` + +### 4. AI服务API +```kotlin +// 语音转文字 +POST /ai/speech-to-text +// 会议纪要生成 +POST /ai/meeting-summary +// 知识库查询 +POST /ai/knowledge-query +// 实时字幕服务 +WebSocket /ai/realtime-subtitle +``` + +### 5. 推送通知API +```kotlin +// 发送会议提醒 +POST /notification/meeting-reminder +// 发送呼叫通知 +POST /notification/call-invitation +// 获取通知历史 +GET /notification/history +``` + +### 6. 统计和分析API +```kotlin +// 会议时长统计 +GET /analytics/meeting-duration +// 用户活跃度统计 +GET /analytics/user-activity +// 功能使用统计 +GET /analytics/feature-usage +``` + +## 🚀 开发建议 + +1. **按阶段顺序开发**：严格遵循8个阶段，确保基础稳固 +2. **测试驱动开发**：先写测试，特别是MQTT消息处理测试 +3. **持续集成**：每阶段完成后运行全部测试套件 +4. **监控和日志**：集成完整的监控和日志系统 +5. **性能基准**：建立性能基准并持续监控 + +这个增强版计划确保了所有关键集成点的完整性，特别是GXRTC API、MQTT消息系统和AR标注同步机制，使AI能够准确高效地完成开发任务。 \ No newline at end of file