Files

chaoq 3bdc40df94 Phase 1 complete. build Ok, apk OK

Signed-off-by: chaoq <chaoq@gxtech.ltd>

2025-09-17 22:50:29 +08:00

44 KiB

Raw Permalink Blame History

增强版AI开发计划 - XSynergy AR远程协作App

🎯 项目概览

项目类型: Android AR协作应用
技术栈: Kotlin + Jetpack Compose + MVVM + LiveKit + ARCore + MQTT
目标: 完整的AI可执行开发计划，包含所有关键集成点

📋 阶段1：基础架构和认证系统（增强版）

🎯 目标

搭建完整的项目架构，集成GXRTC服务器认证
实现MQTT客户端基础框架

🔧 核心实现增强

1.1 正确的API配置

// data/remote/RetrofitClient.kt - 修正
@Module
@InstallIn(SingletonComponent::class)
object NetworkModule {

    @Provides
    @Singleton
    fun provideGxrtcApi(retrofit: Retrofit): GxrtcApi = retrofit.create(GxrtcApi::class.java)

    @Provides
    @Singleton
    fun provideArAnnotationApi(retrofit: Retrofit): ArAnnotationApi = retrofit.create(ArAnnotationApi::class.java)

    @Provides
    @Singleton
    fun provideFeaturePointApi(retrofit: Retrofit): FeaturePointApi = retrofit.create(FeaturePointApi::class.java)
}

// data/remote/GxrtcApi.kt - 新增
interface GxrtcApi {
    @POST("room/token")
    @FormUrlEncoded
    suspend fun getAccessToken(
        @Field("uid") userId: String,
        @Field("room") roomId: String
    ): Response<GxrtcResponse<AccessTokenResponse>>

    @POST("room/")
    @FormUrlEncoded
    suspend fun createRoom(@Field("room") roomName: String): Response<GxrtcResponse<RoomResponse>>

    @GET("room/")
    suspend fun getRooms(): Response<GxrtcResponse<List<RoomResponse>>>
}

// data/remote/ArAnnotationApi.kt - 新增
interface ArAnnotationApi {
    @POST("ar/anchor/save")
    suspend fun saveAnchor(@Body request: SaveAnchorRequest): Response<GxrtcResponse<Unit>>

    @GET("ar/room/{roomId}/anchors")
    suspend fun getRoomAnchors(@Path("roomId") roomId: String): Response<GxrtcResponse<AnchorListResponse>>

    @POST("ar/annotation/save")
    suspend fun saveAnnotation(@Body annotation: ArAnnotation): Response<GxrtcResponse<Unit>>

    @GET("ar/room/{roomId}/annotations")
    suspend fun getRoomAnnotations(@Path("roomId") roomId: String): Response<GxrtcResponse<AnnotationListResponse>>

    @DELETE("ar/annotation/{annotationId}")
    suspend fun deleteAnnotation(@Path("annotationId") annotationId: String): Response<GxrtcResponse<Unit>>

    @POST("ar/spatial/validate")
    suspend fun validateSpatialConsistency(@Body request: SpatialValidationRequest): Response<GxrtcResponse<SpatialValidationResult>>

    @POST("ar/annotations/version")
    suspend fun createAnnotationVersion(@Body request: CreateVersionRequest): Response<GxrtcResponse<VersionResponse>>

    @GET("ar/annotations/{roomId}/versions")
    suspend fun getAnnotationVersions(@Path("roomId") roomId: String): Response<GxrtcResponse<List<VersionResponse>>>
}

// data/remote/FeaturePointApi.kt - 新增
interface FeaturePointApi {
    @POST("ar/feature-points/save")
    suspend fun saveFeaturePoints(@Body request: SaveFeaturePointsRequest): Response<GxrtcResponse<Unit>>

    @GET("ar/feature-points/{anchorId}")
    suspend fun getAnchorFeatures(@Path("anchorId") anchorId: String): Response<GxrtcResponse<FeaturePointListResponse>>

    @POST("ar/feature-points/match")
    suspend fun matchFeaturePoints(@Body request: FeatureMatchRequest): Response<GxrtcResponse<FeatureMatchResponse>>

    @POST("ar/feature-points/quality-assessment")
    suspend fun assessFeatureQuality(@Body request: FeatureQualityRequest): Response<GxrtcResponse<FeatureQualityResponse>>
}

// 请求和响应数据模型
data class SaveAnchorRequest(
    val roomId: String,
    val anchorId: String,
    val position: List<Float>,
    val rotation: List<Float>,
    val createdBy: String
)

data class AnchorListResponse(val anchors: List<AnchorData>)

data class AnnotationListResponse(val annotations: List<ArAnnotation>)

data class SaveFeaturePointsRequest(
    val anchorId: String,
    val roomId: String,
    val featurePoints: List<FeaturePoint>,
    val deviceInfo: DeviceInfo
)

data class FeaturePointListResponse(val featurePoints: List<FeaturePoint>)

data class SpatialValidationRequest(
    val anchorId: String,
    val currentPosition: List<Float>,
    val currentRotation: List<Float>,
    val deviceInfo: DeviceInfo
)

data class SpatialValidationResult(
    val isValid: Boolean,
    val confidence: Float,
    val suggestedCorrection: SpatialTransform?,
    val errorMessage: String?
)

data class FeatureMatchRequest(
    val anchorId: String,
    val queryFeatures: List<FeaturePoint>,
    val maxResults: Int = 10
)

data class FeatureMatchResponse(
    val matches: List<FeatureMatch>,
    val confidence: Float
)

data class FeatureMatch(
    val featurePoint: FeaturePoint,
    val similarity: Float,
    val spatialDistance: Float
)

data class FeatureQualityRequest(
    val anchorId: String,
    val featurePoints: List<FeaturePoint>
)

data class FeatureQualityResponse(
    val overallQuality: Float,
    val individualScores: Map<String, Float>,
    val recommendations: List<String>
)

data class CreateVersionRequest(
    val roomId: String,
    val versionName: String,
    val description: String,
    val createdBy: String
)

data class VersionResponse(
    val versionId: String,
    val versionName: String,
    val description: String,
    val createdAt: Long,
    val createdBy: String,
    val annotationCount: Int
)

// 响应数据结构
data class GxrtcResponse<T>(
    @SerializedName("meta") val meta: Meta,
    @SerializedName("data") val data: T
)

data class Meta(
    @SerializedName("code") val code: Int,
    @SerializedName("message") val message: String? = null
)

data class AccessTokenResponse(@SerializedName("access_token") val accessToken: String)

// data/model/ChatMessage.kt - 新增
data class ChatMessage(
    val id: String,
    val senderId: String,
    val message: String,
    val timestamp: Long,
    val type: String = "text", // text, image, file
    val isPrivate: Boolean = false
)

// data/model/PrivateMessage.kt - 新增
data class PrivateMessage(
    val id: String,
    val senderId: String,
    val recipientId: String,
    val message: String,
    val timestamp: Long,
    val isRead: Boolean = false
)

// data/model/LaserPointerData.kt - 新增
data class LaserPointerData(
    val start: List<Double>, // [x, y] 相对坐标
    val end: List<Double>,   // [x, y] 相对坐标
    val color: List<Int>,    // [r, g, b]
    val thickness: Int,      // 像素粗细
    val timestamp: Long      // 时间戳
)

// data/model/WhiteboardModels.kt - 新增
sealed class WhiteboardState {
    object Inactive : WhiteboardState()
    data class Active(
        val tools: List<WhiteboardTool>,
        val currentTool: WhiteboardTool,
        val strokes: List<WhiteboardStroke>
    ) : WhiteboardState()
}

sealed class WhiteboardTool {
    data class Brush(val color: Color, val size: Float) : WhiteboardTool()
    data class Eraser(val size: Float) : WhiteboardTool()
    data class Shape(val shapeType: ShapeType) : WhiteboardTool()
    data class Text(val fontSize: Float = 16f) : WhiteboardTool()
}

enum class ShapeType { Rectangle, Circle, Triangle, Arrow }

enum class WhiteboardToolType { BRUSH, ERASER, SHAPE, TEXT }

data class WhiteboardStroke(
    val points: List<PointF>,
    val color: Color,
    val size: Float,
    val toolType: WhiteboardToolType
)

1.2 MQTT客户端集成

// data/remote/MqttManager.kt - 新增
class MqttManager @Inject constructor(
    @ApplicationContext private val context: Context,
    private val userRepository: UserRepository
) {
    private var mqttClient: MqttAndroidClient? = null
    private val _mqttState = MutableStateFlow<MqttConnectionState>(MqttConnectionState.Disconnected)
    val mqttState: StateFlow<MqttConnectionState> = _mqttState.asStateFlow()

    suspend fun connect() {
        val serverUri = "tcp://mqtt.cnsdt.com:1883"
        val clientId = MqttClient.generateClientId()

        mqttClient = MqttAndroidClient(context, serverUri, clientId).apply {
            setCallback(object : MqttCallback {
                override fun connectionLost(cause: Throwable?) {
                    _mqttState.value = MqttConnectionState.Disconnected
                }

                override fun messageArrived(topic: String, message: MqttMessage) {
                    handleIncomingMessage(topic, message)
                }

                override fun deliveryComplete(token: IMqttDeliveryToken?) {}
            })
        }

        val options = MqttConnectOptions().apply {
            isCleanSession = true
            connectionTimeout = 30
            keepAliveInterval = 60
        }

        try {
            mqttClient?.connect(options)
            _mqttState.value = MqttConnectionState.Connected
            subscribeToUserTopics()
        } catch (e: Exception) {
            _mqttState.value = MqttConnectionState.Error(e.message ?: "Connection failed")
        }
    }

    private fun subscribeToUserTopics() {
        val user = userRepository.getCurrentUser()
        user?.let {
            val userTopic = "/zrsk/remote-assistant/user/${it.id}"
            mqttClient?.subscribe(userTopic, 1)
        }
    }

    private fun handleIncomingMessage(topic: String, message: MqttMessage) {
        val payload = String(message.payload, Charsets.UTF_8)
        val messageData = Json.decodeFromString<MqttMessageData>(payload)

        when (messageData.event) {
            "call" -> handleCallMessage(messageData.data)
            "call-cancel" -> handleCallCancelMessage(messageData.data)
            "laser-pointer" -> handleLaserPointerMessage(messageData.data)
            "chat-message" -> handleChatMessage(messageData.data)
            "private-message" -> handlePrivateMessage(messageData.data)
            "whiteboard-state" -> handleWhiteboardMessage(messageData.data)
            "whiteboard-stroke" -> handleWhiteboardMessage(messageData.data)
            // 其他消息类型...
        }
    }
}

✅ 增强验收标准

GXRTC API集成正确，能成功获取access token
MQTT客户端能正常连接和接收消息，连接成功率>99.9%
消息解析和处理框架完整，支持所有消息类型
用户认证状态持久化，应用重启后保持登录状态
网络错误处理完善，有明确的用户提示
单元测试覆盖率>85%，关键业务逻辑100%覆盖

📋 阶段2：实时通信基础（增强版）

🎯 目标

集成LiveKit with正确的token获取流程
实现完整的音视频通话功能

🔧 核心实现增强

2.1 正确的LiveKit连接流程

// domain/manager/LiveKitManager.kt - 修正
class LiveKitManager @Inject constructor(
    @ApplicationContext private val context: Context,
    private val gxrtcApi: GxrtcApi,
    private val userRepository: UserRepository
) {
    suspend fun connectToRoom(roomName: String): Result<Room> {
        return try {
            // 1. 获取access token
            val user = userRepository.getCurrentUser()
            val tokenResponse = gxrtcApi.getAccessToken(user.id, roomName)

            if (!tokenResponse.isSuccessful || tokenResponse.body()?.meta?.code != 200) {
                return Result.failure(Exception("Failed to get access token"))
            }

            val accessToken = tokenResponse.body()!!.data.accessToken

            // 2. 连接LiveKit
            val newRoom = LiveKit.create(
                appContext = context,
                options = RoomOptions(
                    adaptiveStream = true,
                    dynacast = true,
                    e2eeOptions = E2EEOptions(
                        keyProvider = BaseKeyProvider("encryption-key")
                    )
                )
            )

            newRoom.connect("wss://meeting.cnsdt.com", accessToken)
            Result.success(newRoom)
        } catch (e: Exception) {
            Result.failure(e)
        }
    }
}

2.2 MQTT房间消息订阅

// 在进入房间时订阅相关主题
fun subscribeToRoomTopics(roomId: String) {
    val roomTopic = "/zrsk/remote-assistant/room/$roomId"
    val userRoomTopic = "/zrsk/remote-assistant/room/$roomId/user/${currentUser.id}"

    mqttClient?.subscribe(roomTopic, 1)
    mqttClient?.subscribe(userRoomTopic, 1)
}

✅ 增强验收标准

使用GXRTC token成功连接LiveKit，连接时间<3s
房间相关的MQTT消息能正确接收和处理，消息延迟<200ms
断线重连机制完善，网络恢复后30s内自动重连
音视频权限处理正确，用户拒绝权限时有友好提示
本地视频预览正常，画面清晰无卡顿
远程视频渲染正常，支持多种分辨率和帧率

📋 阶段3：协作会话管理（增强版）

🎯 目标

完整的会议生命周期管理
MQTT消息驱动的呼叫系统

🔧 核心实现增强

3.1 呼叫系统实现

// domain/manager/CallManager.kt - 新增
class CallManager @Inject constructor(
    private val mqttManager: MqttManager,
    private val userRepository: UserRepository
) {
    fun sendCallInvitation(roomId: String, roomTitle: String, targetUserId: String) {
        val callMessage = MqttMessageData(
            event = "call",
            data = mapOf(
                "room_id" to roomId,
                "title" to roomTitle,
                "caller" to userRepository.getCurrentUser()?.id ?: ""
            )
        )

        val topic = "/zrsk/remote-assistant/user/$targetUserId"
        mqttManager.publish(topic, callMessage)
    }

    fun handleCallMessage(data: Map<String, Any>) {
        val roomId = data["room_id"] as String
        val title = data["title"] as String
        val callerId = data["caller"] as String

        // 显示呼叫UI，用户可以选择接听或拒绝
        _incomingCall.value = IncomingCall(roomId, title, callerId)
    }
}

3.2 会议状态管理

// 处理会议结束消息
private fun handleEndMeetingMessage(data: Map<String, Any>) {
    val reason = data["reason"] as? String
    val code = data["code"] as Int

    when (code) {
        1 -> showToast("用户挂断了会议")
        2 -> showToast("管理员结束了会议")
        else -> showToast("会议已结束")
    }

    // 退出会议界面
    navigateToHome()
}

✅ 增强验收标准

呼叫邀请能正确发送和接收，送达率100%
会议状态变化能正确处理，状态同步延迟<1s
用户加入/离开通知完善，实时更新参与者列表
会议录制功能正常，录制文件可完整回放
会议时长统计准确，支持暂停/继续计时
多语言支持完整，中英文界面无缝切换

📋 阶段4：AR环境集成（增强版）

🎯 目标

ARCore环境集成与平面检测
后置摄像头默认配置与空间锚点同步
服务器端AR标注持久化存储

🔧 核心实现增强

4.1 AR场景管理与后置摄像头配置

// ui/ar/ArSceneManager.kt - 增强
class ArSceneManager @Inject constructor(
    private val mqttManager: MqttManager,
    private val arAnnotationService: ArAnnotationService
) {
    private var arSession: Session? = null
    private val cameraConfig = CameraConfig().apply {
        facingDirection = CameraConfig.FacingDirection.BACK // 默认后置摄像头
        depthSensorUsage = CameraConfig.DepthSensorUsage.AUTOMATIC
    }

    suspend fun initializeArSession(context: Context): Result<Session> {
        return try {
            val session = Session(context)
            session.cameraConfig = cameraConfig
            session.configure(session.config)

            arSession = session
            Result.success(session)
        } catch (e: Exception) {
            Result.failure(e)
        }
    }

    fun createAnchorAtPosition(position: Vector3): Anchor {
        val anchor = arSession!!.createAnchor(Pose(position, Quaternion.identity()))

        // 同步到服务器和其他用户
        syncAnchorToParticipants(anchor)

        return anchor
    }

    private suspend fun syncAnchorToParticipants(anchor: Anchor) {
        // 保存到服务器
        arAnnotationService.saveAnchor(
            roomId = currentRoomId,
            anchorId = anchor.trackableId,
            position = anchor.pose
        )

        // 同步到其他用户
        val anchorData = mapOf(
            "anchor_id" to anchor.trackableId,
            "position" to listOf(anchor.pose.tx(), anchor.pose.ty(), anchor.pose.tz()),
            "rotation" to listOf(anchor.pose.qx(), anchor.pose.qy(), anchor.pose.qz(), anchor.pose.qw())
        )

        val message = MqttMessageData(
            event = "ar-anchor-create",
            data = anchorData
        )
        mqttManager.publishToRoom(roomId, message)
    }

    suspend fun restoreAnchors(roomId: String): List<Anchor> {
        val savedAnchors = arAnnotationService.loadRoomAnchors(roomId)
        return savedAnchors.map { anchorData ->
            val pose = Pose(
                floatArrayOf(anchorData.position[0], anchorData.position[1], anchorData.position[2]),
                floatArrayOf(anchorData.rotation[0], anchorData.rotation[1], anchorData.rotation[2], anchorData.rotation[3])
            )
            arSession!!.createAnchor(pose)
        }
    }
}

// data/model/AnchorData.kt - 新增
data class AnchorData(
    val anchorId: String,
    val position: List<Float>,
    val rotation: List<Float>,
    val roomId: String,
    val createdAt: Long = System.currentTimeMillis(),
    val createdBy: String
)

4.2 AR标注持久化服务（增强版）

// domain/service/ArAnnotationService.kt - 新增
class ArAnnotationService @Inject constructor(
    private val arAnnotationApi: ArAnnotationApi,
    private val featurePointApi: FeaturePointApi,
    private val userRepository: UserRepository
) {
    suspend fun saveAnchorWithFeatures(
        roomId: String,
        anchorId: String,
        position: Pose,
        featurePoints: List<FeaturePoint>,
        deviceInfo: DeviceInfo
    ): Result<Unit> {
        return try {
            // 保存锚点基本信息
            val anchorRequest = SaveAnchorRequest(
                roomId = roomId,
                anchorId = anchorId,
                position = listOf(position.tx(), position.ty(), position.tz()),
                rotation = listOf(position.qx(), position.qy(), position.qz(), position.qw()),
                createdBy = userRepository.getCurrentUser()?.id ?: ""
            )
            arAnnotationApi.saveAnchor(anchorRequest)

            // 保存特征点数据
            val featureRequest = SaveFeaturePointsRequest(
                anchorId = anchorId,
                roomId = roomId,
                featurePoints = featurePoints,
                deviceInfo = deviceInfo
            )
            featurePointApi.saveFeaturePoints(featureRequest)

            Result.success(Unit)
        } catch (e: Exception) {
            Result.failure(e)
        }
    }

    suspend fun loadRoomAnchorsWithFeatures(roomId: String): List<AnchorWithFeatures> {
        val anchors = arAnnotationApi.getRoomAnchors(roomId).anchors
        return anchors.map { anchor ->
            val features = featurePointApi.getAnchorFeatures(anchor.anchorId).featurePoints
            AnchorWithFeatures(anchor, features)
        }
    }

    suspend fun saveAnnotation(annotation: ArAnnotation): Result<Unit> {
        return try {
            arAnnotationApi.saveAnnotation(annotation)
            Result.success(Unit)
        } catch (e: Exception) {
            Result.failure(e)
        }
    }

    suspend fun loadRoomAnnotations(roomId: String): List<ArAnnotation> {
        return arAnnotationApi.getRoomAnnotations(roomId).annotations
    }

    suspend fun validateSpatialConsistency(
        anchorId: String,
        currentPose: Pose,
        deviceInfo: DeviceInfo
    ): Result<SpatialValidationResult> {
        return try {
            val request = SpatialValidationRequest(
                anchorId = anchorId,
                currentPosition = listOf(currentPose.tx(), currentPose.ty(), currentPose.tz()),
                currentRotation = listOf(currentPose.qx(), currentPose.qy(), currentPose.qz(), currentPose.qw()),
                deviceInfo = deviceInfo
            )
            val result = arAnnotationApi.validateSpatialConsistency(request)
            Result.success(result)
        } catch (e: Exception) {
            Result.failure(e)
        }
    }
}

// 数据模型增强
data class AnchorWithFeatures(
    val anchor: AnchorData,
    val featurePoints: List<FeaturePoint>
)

data class FeaturePoint(
    val position: List<Float>,
    val descriptor: String, // base64编码的特征描述符
    val qualityScore: Float,
    val timestamp: Long = System.currentTimeMillis()
)

data class DeviceInfo(
    val model: String,
    val arCoreVersion: String,
    val cameraCalibration: CameraCalibration?,
    val imuCalibration: ImuCalibration?
)

data class CameraCalibration(
    val focalLength: List<Float>,
    val principalPoint: List<Float>,
    val distortionCoefficients: List<Float>?
)

data class ImuCalibration(
    val gyroBias: List<Float>?,
    val accelBias: List<Float>?,
    val magnetometerBias: List<Float>?
)

data class ArAnnotation(
    val id: String,
    val anchorId: String,
    val type: AnnotationType,
    val content: String,
    val color: List<Int>,
    val size: Float,
    val roomId: String,
    val createdBy: String,
    val spatialTransform: SpatialTransform? = null,
    val createdAt: Long = System.currentTimeMillis(),
    val updatedAt: Long = System.currentTimeMillis(),
    val version: Int = 1
)

data class SpatialTransform(
    val position: List<Float>,
    val rotation: List<Float>,
    val scale: List<Float>,
    val confidence: Float
)

enum class AnnotationType {
    ARROW, CIRCLE, RECTANGLE, TEXT, HIGHLIGHT, MEASUREMENT, NOTE
}

✅ 增强验收标准

AR场景能正确初始化和渲染，初始化时间<5s
默认使用后置摄像头，支持自动切换
空间锚点能跨设备同步，位置误差<5cm
服务器端锚点存储可靠，存储成功率>99.9%
锚点恢复准确，重定位误差<10cm
支持会话间AR标注持久化恢复
平面检测准确稳定，检测成功率>95%
环境光估计准确，虚拟物体光照一致

📋 阶段5：AR标注工具（增强版）

🎯 目标

完整的AR标注系统
实时同步机制

🔧 核心实现增强

5.1 激光笔工具实现

// ui/ar/tools/LaserPointerTool.kt - 新增
class LaserPointerTool @Inject constructor(
    private val mqttManager: MqttManager
) : ArTool {

    fun onTouchStart(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) {
        val startX = screenX / screenWidth
        val startY = screenY / screenHeight

        currentLaserData = LaserData(
            start = listOf(startX, startY),
            end = listOf(startX, startY),
            color = listOf(255, 0, 0), // 红色
            thickness = 3
        )
    }

    fun onTouchMove(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) {
        val endX = screenX / screenWidth
        val endY = screenY / screenHeight

        currentLaserData = currentLaserData?.copy(end = listOf(endX, endY))

        // 实时发送激光笔位置
        sendLaserPointerData()
    }

    fun onTouchEnd() {
        // 发送最终的激光笔数据
        sendLaserPointerData()
        currentLaserData = null
    }

    private fun sendLaserPointerData() {
        currentLaserData?.let { laserData ->
            val message = MqttMessageData(
                event = "laser-pointer",
                data = mapOf(
                    "start" to laserData.start,
                    "end" to laserData.end,
                    "color" to laserData.color,
                    "thickness" to laserData.thickness
                )
            )

            mqttManager.publishToRoom(roomId, message)
        }
    }

    private fun handleLaserPointerMessage(data: Map<String, Any>) {
        val start = data["start"] as List<Double>
        val end = data["end"] as List<Double>
        val color = data["color"] as List<Int>
        val thickness = data["thickness"] as Int

        // 更新激光笔显示
        _laserPointerData.value = LaserPointerData(
            start = start,
            end = end,
            color = color,
            thickness = thickness,
            timestamp = System.currentTimeMillis()
        )
    }

    private fun handleChatMessage(data: Map<String, Any>) {
        val senderId = data["sender_id"] as String
        val message = data["message"] as String
        val timestamp = data["timestamp"] as Long
        val messageType = data["type"] as? String ?: "text"

        // 更新聊天界面
        _chatMessages.value = _chatMessages.value + ChatMessage(
            id = UUID.randomUUID().toString(),
            senderId = senderId,
            message = message,
            timestamp = timestamp,
            type = messageType,
            isPrivate = false
        )
    }

    private fun handlePrivateMessage(data: Map<String, Any>) {
        val senderId = data["sender_id"] as String
        val message = data["message"] as String
        val timestamp = data["timestamp"] as Long
        val recipientId = data["recipient_id"] as String

        // 更新私聊界面
        _privateMessages.value = _privateMessages.value + PrivateMessage(
            id = UUID.randomUUID().toString(),
            senderId = senderId,
            recipientId = recipientId,
            message = message,
            timestamp = timestamp
        )
    }

    fun sendChatMessage(message: String, isPrivate: Boolean = false, recipientId: String? = null) {
        val messageData = if (isPrivate && recipientId != null) {
            MqttMessageData(
                event = "private-message",
                data = mapOf(
                    "sender_id" to currentUser.id,
                    "recipient_id" to recipientId,
                    "message" to message,
                    "timestamp" to System.currentTimeMillis()
                )
            )
        } else {
            MqttMessageData(
                event = "chat-message",
                data = mapOf(
                    "sender_id" to currentUser.id,
                    "message" to message,
                    "timestamp" to System.currentTimeMillis(),
                    "type" to "text"
                )
            )
        }

        val topic = if (isPrivate) {
            "/zrsk/remote-assistant/user/$recipientId"
        } else {
            "/zrsk/remote-assistant/room/$currentRoomId"
        }

        mqttManager.publish(topic, messageData)
    }
}

3.3 文字聊天界面实现

// ui/chat/ChatViewModel.kt - 新增
@HiltViewModel
class ChatViewModel @Inject constructor(
    private val mqttManager: MqttManager
) : ViewModel() {

    private val _chatMessages = MutableStateFlow<List<ChatMessage>>(emptyList())
    val chatMessages: StateFlow<List<ChatMessage>> = _chatMessages.asStateFlow()

    private val _privateMessages = MutableStateFlow<List<PrivateMessage>>(emptyList())
    val privateMessages: StateFlow<List<PrivateMessage>> = _privateMessages.asStateFlow()

    fun loadChatHistory(roomId: String) {
        // 从服务器加载聊天历史
    }

    fun clearChatHistory() {
        _chatMessages.value = emptyList()
    }
}

// ui/chat/ChatScreen.kt - 新增
@Composable
fun ChatScreen(
    viewModel: ChatViewModel = hiltViewModel(),
    onBackPressed: () -> Unit
) {
    val messages by viewModel.chatMessages.collectAsState()
    var newMessage by remember { mutableStateOf("") }

    Column(modifier = Modifier.fillMaxSize()) {
        // 消息列表
        LazyColumn(modifier = Modifier.weight(1f)) {
            items(messages) { message ->
                ChatMessageItem(message = message)
            }
        }

        // 输入框
        Row(
            modifier = Modifier
                .fillMaxWidth()
                .padding(8.dp),
            verticalAlignment = Alignment.CenterVertically
        ) {
            TextField(
                value = newMessage,
                onValueChange = { newMessage = it },
                modifier = Modifier.weight(1f),
                placeholder = { Text("输入消息...") }
            )

            Button(
                onClick = {
                    if (newMessage.isNotBlank()) {
                        viewModel.sendMessage(newMessage)
                        newMessage = ""
                    }
                },
                modifier = Modifier.padding(start = 8.dp)
            ) {
                Text("发送")
            }
        }
    }
}

✅ 增强验收标准

激光笔工具实时同步，端到端延迟<100ms
AR标注空间位置准确，位置误差<2cm
多用户同时标注无冲突，支持并发标注
标注持久化存储，支持会话间恢复
标注工具丰富，支持箭头、圆圈、文字等多种类型
标注颜色和大小可自定义，视觉效果清晰

📋 阶段6：协作增强功能（增强版）

🎯 目标

白板、屏幕共享、文件传输
实时字幕和会议纪要

🔧 核心实现增强

6.1 白板协作实现

// ui/whiteboard/WhiteboardManager.kt - 新增
class WhiteboardManager @Inject constructor(
    private val mqttManager: MqttManager
) {
    private val _whiteboardState = MutableStateFlow<WhiteboardState>(WhiteboardState.Inactive)
    val whiteboardState: StateFlow<WhiteboardState> = _whiteboardState.asStateFlow()

    fun enableWhiteboard() {
        _whiteboardState.value = WhiteboardState.Active(
            tools = listOf(
                WhiteboardTool.Brush(color = Color.Red, size = 3f),
                WhiteboardTool.Eraser(size = 10f),
                WhiteboardTool.Shape(shapeType = ShapeType.Rectangle),
                WhiteboardTool.Text()
            ),
            currentTool = WhiteboardTool.Brush(color = Color.Red, size = 3f),
            strokes = emptyList()
        )

        // 通知其他用户白板已开启
        sendWhiteboardState(true)
    }

    fun disableWhiteboard() {
        _whiteboardState.value = WhiteboardState.Inactive
        sendWhiteboardState(false)
    }

    fun addStroke(stroke: WhiteboardStroke) {
        val currentState = _whiteboardState.value
        if (currentState is WhiteboardState.Active) {
            _whiteboardState.value = currentState.copy(
                strokes = currentState.strokes + stroke
            )

            // 同步到其他用户
            sendStroke(stroke)
        }
    }

    private fun sendWhiteboardState(enabled: Boolean) {
        val message = MqttMessageData(
            event = "whiteboard-state",
            data = mapOf("enabled" to enabled)
        )
        mqttManager.publishToRoom(roomId, message)
    }

    private fun sendStroke(stroke: WhiteboardStroke) {
        val message = MqttMessageData(
            event = "whiteboard-stroke",
            data = mapOf(
                "points" to stroke.points.map { listOf(it.x, it.y) },
                "color" to listOf(stroke.color.red, stroke.color.green, stroke.color.blue),
                "size" to stroke.size,
                "tool_type" to stroke.toolType.name
            )
        )
        mqttManager.publishToRoom(roomId, message)
    }

    fun handleWhiteboardMessage(data: Map<String, Any>) {
        when (data["event"] as String) {
            "whiteboard-state" -> {
                val enabled = data["enabled"] as Boolean
                if (enabled) {
                    enableWhiteboard()
                } else {
                    disableWhiteboard()
                }
            }
            "whiteboard-stroke" -> {
                val points = (data["points"] as List<List<Float>>).map { PointF(it[0], it[1]) }
                val color = Color(
                    red = (data["color"] as List<Int>)[0],
                    green = (data["color"] as List<Int>)[1],
                    blue = (data["color"] as List<Int>)[2]
                )
                val size = data["size"] as Float
                val toolType = WhiteboardToolType.valueOf(data["tool_type"] as String)

                val stroke = WhiteboardStroke(points, color, size, toolType)
                addStroke(stroke)
            }
        }
    }
}

6.2 实时字幕集成

// 处理ASR字幕消息
private fun handleAsrSubtitleMessage(data: Map<String, Any>) {
    val uid = data["uid"] as String
    val segment = data["seg"] as String
    val status = data["status"] as String

    when (status) {
        "streaming" -> updateLiveSubtitle(uid, segment)
        "final" -> finalizeSubtitle(uid, segment)
    }
}

6.2 文件预览功能

// 处理文件预览消息
private fun handleFilePreviewMessage(data: Map<String, Any>) {
    val fileName = data["name"] as String
    val filePath = data["path"] as String
    val mimeType = data["mime_type"] as String

    val fileUrl = "https://meeting.cnsdt.com$filePath"

    // 显示文件预览界面
    showFilePreview(fileName, fileUrl, mimeType)
}

✅ 增强验收标准

实时字幕准确率>90%，行业术语识别准确
文件预览功能完整，支持PDF、Word、Excel、图片等格式
屏幕共享流畅，帧率>15fps，延迟<300ms
白板协作实时同步，操作延迟<200ms
文件传输可靠，支持断点续传，100MB文件传输成功率>99%
共同浏览功能正常，页面同步准确

📋 阶段7：AI能力集成（增强版）

🎯 目标

语音识别、会议纪要、知识库
智能助手集成

🔧 核心实现增强

7.1 知识库查询集成

// domain/usecase/QueryKnowledgeBaseUseCase.kt - 新增
class QueryKnowledgeBaseUseCase @Inject constructor(
    private val aiService: AiService
) {
    suspend operator fun invoke(query: String): Result<String> {
        return try {
            val response = aiService.queryKnowledgeBase(query)
            if (response.isSuccessful) {
                Result.success(response.body()?.answer ?: "未找到相关信息")
            } else {
                Result.failure(Exception("查询失败"))
            }
        } catch (e: Exception) {
            Result.failure(e)
        }
    }
}

✅ 增强验收标准

知识库查询响应时间<2秒，相关度>85%
会议纪要生成准确，关键信息提取完整
AI助手交互流畅，自然语言理解准确
语音助手响应迅速，语音识别准确率>92%
智能推荐相关，推荐准确率>80%
多轮对话支持，上下文理解准确

🔌 缺失的服务端API接口（增强版）

根据AR标注和持久化需求，需要以下额外的服务端API接口：

1. AR标注存储API

// 保存AR锚点
POST /ar/anchor/save
Content-Type: application/json
{
  "room_id": "string",
  "anchor_id": "string",
  "position": [0.0, 0.0, 0.0],
  "rotation": [0.0, 0.0, 0.0, 1.0],
  "created_by": "user123"
}

// 获取房间所有锚点
GET /ar/room/{roomId}/anchors

// 保存AR标注内容
POST /ar/annotation/save
Content-Type: application/json
{
  "anchor_id": "string",
  "type": "ARROW",
  "content": "检查这里",
  "color": [255, 0, 0],
  "size": 2.0,
  "room_id": "string",
  "created_by": "user123"
}

// 获取房间所有标注
GET /ar/room/{roomId}/annotations

// 删除标注
DELETE /ar/annotation/{annotationId}

2. AR会话管理API

// 创建AR会话
POST /ar/session/create
{
  "room_id": "string",
  "session_name": "设备维护会话",
  "created_by": "user123"
}

// 获取会话历史
GET /ar/user/{userId}/sessions

// 恢复特定会话
GET /ar/session/{sessionId}

// 导出会话数据
GET /ar/session/{sessionId}/export?format=json|pdf

3. 空间数据验证API

// 验证锚点空间一致性
POST /ar/anchor/validate
{
  "anchor_id": "string",
  "position": [0.0, 0.0, 0.0],
  "rotation": [0.0, 0.0, 0.0, 1.0],
  "device_info": {
    "model": "Pixel 6",
    "ar_core_version": "1.50.0"
  }
}

// 批量验证多个锚点
POST /ar/anchors/validate-batch

4. 特征点备份API（可选）

// 备份ARCore特征点数据
POST /ar/features/backup
Content-Type: application/octet-stream
[二进制特征点数据]

// 恢复特征点数据
GET /ar/features/restore?anchor_id={anchorId}

// 特征点相似度匹配
POST /ar/features/match
{
  "anchor_id": "string",
  "feature_descriptor": "base64_encoded_data"
}

5. 设备校准API

// 上报设备校准数据
POST /ar/device/calibration
{
  "device_id": "string",
  "calibration_data": {
    "camera_params": {...},
    "imu_calibration": {...},
    "accuracy_score": 0.95
  },
  "ar_core_version": "1.50.0"
}

// 获取设备推荐校准参数
GET /ar/device/{deviceModel}/calibration

6. 统计分析API

// AR使用统计
GET /ar/analytics/usage?start_date=2025-01-01&end_date=2025-01-31

// 标注准确率统计
GET /ar/analytics/accuracy?room_id={roomId}

// 设备兼容性报告
GET /ar/analytics/compatibility

7. 特征点管理API（关键补充）

// 保存ARCore特征点数据
POST /ar/feature-points/save
Content-Type: application/json
{
  "anchor_id": "string",
  "room_id": "string",
  "feature_points": [
    {
      "position": [0.0, 0.0, 0.0],
      "descriptor": "base64_encoded_descriptor",
      "quality_score": 0.95
    }
  ],
  "device_info": {
    "model": "Pixel 6",
    "ar_core_version": "1.50.0",
    "camera_calibration": {
      "focal_length": [1000.0, 1000.0],
      "principal_point": [500.0, 500.0]
    }
  }
}

// 获取特征点数据用于重定位
GET /ar/feature-points/{anchorId}

// 批量特征点匹配（跨设备空间对齐）
POST /ar/feature-points/match
{
  "anchor_id": "string",
  "query_features": [
    {
      "position": [0.0, 0.0, 0.0],
      "descriptor": "base64_encoded_descriptor"
    }
  ],
  "max_results": 10
}

// 特征点质量评估
POST /ar/feature-points/quality-assessment
{
  "anchor_id": "string",
  "feature_points": [
    {
      "position": [0.0, 0.0, 0.0],
      "descriptor": "base64_encoded_descriptor"
    }
  ]
}

8. 空间一致性验证API

// 空间坐标转换验证
POST /ar/spatial/validate-transform
{
  "source_anchor": "anchor_123",
  "target_anchor": "anchor_456",
  "transform_matrix": [
    [1.0, 0.0, 0.0, 0.0],
    [0.0, 1.0, 0.0, 0.0],
    [0.0, 0.0, 1.0, 0.0],
    [1.5, 2.0, 0.5, 1.0]
  ],
  "confidence": 0.92
}

// 多设备空间对齐状态
GET /ar/spatial/alignment-status?room_id={roomId}

// 强制空间重新对齐
POST /ar/spatial/realign
{
  "room_id": "string",
  "anchor_ids": ["anchor1", "anchor2", "anchor3"]
}

9. 标注版本管理API

// 创建标注版本
POST /ar/annotations/version
{
  "room_id": "string",
  "version_name": "v1.0",
  "description": "初始标注版本",
  "created_by": "user123"
}

// 获取标注版本历史
GET /ar/annotations/{roomId}/versions

// 恢复到特定版本
POST /ar/annotations/{roomId}/restore/{versionId}

// 标注差异比较
GET /ar/annotations/{roomId}/diff?version1={v1}&version2={v2}

10. 设备性能优化API

// 设备性能基准上报
POST /ar/device/performance
{
  "device_model": "Pixel 6",
  "ar_core_version": "1.50.0",
  "performance_metrics": {
    "frame_rate": 60.0,
    "tracking_quality": 0.95,
    "memory_usage": 256,
    "battery_drain": 15.0
  },
  "recommended_settings": {
    "max_annotations": 50,
    "feature_point_density": 0.8,
    "texture_quality": "medium"
  }
}

// 获取设备优化配置
GET /ar/device/{deviceModel}/optimization

// 性能问题诊断
POST /ar/device/diagnose
{
  "device_info": {...},
  "performance_issues": ["low_framerate", "high_memory"]
}

📋 阶段8：优化和兼容性（增强版）

🎯 目标

性能优化、设备兼容
完整的错误处理和日志系统

🔧 核心实现增强

8.1 完整的错误处理

// utils/ErrorHandler.kt - 新增
class ErrorHandler @Inject constructor(
    @ApplicationContext private val context: Context
) {
    fun handleNetworkError(error: Throwable) {
        when (error) {
            is SocketTimeoutException -> showToast("网络连接超时")
            is ConnectException -> showToast("无法连接到服务器")
            is SSLHandshakeException -> showToast("安全连接失败")
            else -> showToast("网络错误: ${error.message}")
        }
    }

    fun handleMqttError(error: Throwable) {
        showToast("实时通信连接失败")
        // 尝试重连
        mqttManager.reconnect()
    }
}

✅ 增强验收标准

性能指标达标：冷启动<3s，热启动<1s，标注延迟<100ms
兼容Android 8.0+和ARCore设备，覆盖主流设备型号
错误处理完整，用户体验良好，错误提示友好明确
内存使用优化，1小时会议内存增长<50MB
电池消耗合理，1小时会议耗电<15%
网络带宽优化，720p视频通话<800Kbps

🧪 增强测试策略

单元测试增强

MQTT消息解析测试：验证所有消息类型的正确解析
GXRTC API响应处理测试：测试各种HTTP状态码和错误处理
AR标注坐标转换测试：验证屏幕坐标到世界坐标的准确转换
权限管理测试：摄像头、麦克风、存储权限处理
数据模型测试：所有数据类的序列化/反序列化
错误处理测试：网络错误、解析错误、权限错误处理

集成测试增强

多设备同步测试：3+设备同时协作，验证标注同步
网络条件模拟测试：2G/3G/4G/WiFi切换，丢包率20%测试
断线重连测试：网络中断30秒后自动重连
MQTT消息完整性测试：消息不丢失、不重复、有序到达
LiveKit连接测试：Token过期处理、房间满员处理
文件传输测试：大文件(100MB+)传输完整性验证
AR锚点同步测试：多设备间空间锚点一致性

UI测试增强

AR场景交互测试：手势识别、平面检测、标注绘制
实时协作流程测试：完整会议生命周期UI测试
错误状态UI测试：网络错误、权限拒绝、服务器错误的UI反馈
性能监控测试：帧率监控、内存使用、电池消耗
无障碍测试：屏幕阅读器支持、大字體模式
多语言测试：中英文界面切换

性能测试

启动时间测试：冷启动<3s，热启动<1s
标注延迟测试：端到端延迟<100ms
内存泄漏测试：长时间会议无内存泄漏
电池消耗测试：1小时会议耗电<15%
网络带宽测试：720p视频通话带宽<1Mbps

安全测试

数据加密测试：音视频流端到端加密验证
Token安全测试：Token过期和刷新机制
权限提升测试：防止未授权功能访问
输入验证测试：防止注入攻击
安全存储测试：敏感数据加密存储

🔌 缺失的服务端API接口

根据PRD需求，需要以下额外的服务端API接口：

1. 用户管理API

// 获取用户基本信息
GET /user/{userId}/info
// 搜索用户
GET /user/search?keyword={name}&department={dept}
// 获取组织架构
GET /organization/structure

2. 文件服务API

// 文件上传
POST /file/upload
// 文件下载
GET /file/{fileId}/download
// 文件信息获取
GET /file/{fileId}/info
// 文件预览生成
POST /file/{fileId}/generate-preview

3. 会议管理API

// 创建预约会议
POST /meeting/scheduled
// 获取会议列表
GET /meeting/history
// 获取会议详情
GET /meeting/{meetingId}
// 删除会议记录
DELETE /meeting/{meetingId}

4. AI服务API

// 语音转文字
POST /ai/speech-to-text
// 会议纪要生成
POST /ai/meeting-summary
// 知识库查询
POST /ai/knowledge-query
// 实时字幕服务
WebSocket /ai/realtime-subtitle

5. 推送通知API

// 发送会议提醒
POST /notification/meeting-reminder
// 发送呼叫通知
POST /notification/call-invitation
// 获取通知历史
GET /notification/history

6. 统计和分析API

// 会议时长统计
GET /analytics/meeting-duration
// 用户活跃度统计
GET /analytics/user-activity
// 功能使用统计
GET /analytics/feature-usage

🚀 开发建议

按阶段顺序开发：严格遵循8个阶段，确保基础稳固
测试驱动开发：先写测试，特别是MQTT消息处理测试
持续集成：每阶段完成后运行全部测试套件
监控和日志：集成完整的监控和日志系统
性能基准：建立性能基准并持续监控

这个增强版计划确保了所有关键集成点的完整性，特别是GXRTC API、MQTT消息系统和AR标注同步机制，使AI能够准确高效地完成开发任务。

44 KiB Raw Permalink Blame History Unescape Escape

增强版AI开发计划 - XSynergy AR远程协作App

🎯 项目概览

📋 阶段1：基础架构和认证系统（增强版）

🎯 目标

🔧 核心实现增强

1.1 正确的API配置

1.2 MQTT客户端集成

✅ 增强验收标准

📋 阶段2：实时通信基础（增强版）

🎯 目标

🔧 核心实现增强

2.1 正确的LiveKit连接流程

2.2 MQTT房间消息订阅

✅ 增强验收标准

📋 阶段3：协作会话管理（增强版）

🎯 目标

🔧 核心实现增强

3.1 呼叫系统实现

3.2 会议状态管理

✅ 增强验收标准

📋 阶段4：AR环境集成（增强版）

🎯 目标

🔧 核心实现增强

4.1 AR场景管理与后置摄像头配置

4.2 AR标注持久化服务（增强版）

✅ 增强验收标准

📋 阶段5：AR标注工具（增强版）

🎯 目标

🔧 核心实现增强

5.1 激光笔工具实现

3.3 文字聊天界面实现

✅ 增强验收标准

📋 阶段6：协作增强功能（增强版）

🎯 目标

🔧 核心实现增强

6.1 白板协作实现

6.2 实时字幕集成

6.2 文件预览功能

✅ 增强验收标准

📋 阶段7：AI能力集成（增强版）

🎯 目标

🔧 核心实现增强

7.1 知识库查询集成

✅ 增强验收标准

🔌 缺失的服务端API接口（增强版）

1. AR标注存储API

2. AR会话管理API

3. 空间数据验证API

4. 特征点备份API（可选）

5. 设备校准API

6. 统计分析API

7. 特征点管理API（关键补充）

8. 空间一致性验证API

9. 标注版本管理API

10. 设备性能优化API

📋 阶段8：优化和兼容性（增强版）

🎯 目标

🔧 核心实现增强

8.1 完整的错误处理

✅ 增强验收标准

🧪 增强测试策略

单元测试增强

集成测试增强

UI测试增强

性能测试

安全测试

🔌 缺失的服务端API接口

1. 用户管理API

2. 文件服务API

3. 会议管理API

4. AI服务API

5. 推送通知API

6. 统计和分析API

🚀 开发建议

44 KiB

Raw Permalink Blame History