1530 lines
44 KiB
Markdown
1530 lines
44 KiB
Markdown
# 增强版AI开发计划 - XSynergy AR远程协作App
|
||
|
||
## 🎯 项目概览
|
||
- **项目类型**: Android AR协作应用
|
||
- **技术栈**: Kotlin + Jetpack Compose + MVVM + LiveKit + ARCore + MQTT
|
||
- **目标**: 完整的AI可执行开发计划，包含所有关键集成点
|
||
|
||
---
|
||
|
||
## 📋 阶段1：基础架构和认证系统（增强版）
|
||
|
||
### 🎯 目标
|
||
- 搭建完整的项目架构，集成GXRTC服务器认证
|
||
- 实现MQTT客户端基础框架
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 1.1 正确的API配置
|
||
```kotlin
|
||
// data/remote/RetrofitClient.kt - 修正
|
||
@Module
|
||
@InstallIn(SingletonComponent::class)
|
||
object NetworkModule {
|
||
|
||
@Provides
|
||
@Singleton
|
||
fun provideGxrtcApi(retrofit: Retrofit): GxrtcApi = retrofit.create(GxrtcApi::class.java)
|
||
|
||
@Provides
|
||
@Singleton
|
||
fun provideArAnnotationApi(retrofit: Retrofit): ArAnnotationApi = retrofit.create(ArAnnotationApi::class.java)
|
||
|
||
@Provides
|
||
@Singleton
|
||
fun provideFeaturePointApi(retrofit: Retrofit): FeaturePointApi = retrofit.create(FeaturePointApi::class.java)
|
||
}
|
||
|
||
// data/remote/GxrtcApi.kt - 新增
|
||
interface GxrtcApi {
|
||
@POST("room/token")
|
||
@FormUrlEncoded
|
||
suspend fun getAccessToken(
|
||
@Field("uid") userId: String,
|
||
@Field("room") roomId: String
|
||
): Response<GxrtcResponse<AccessTokenResponse>>
|
||
|
||
@POST("room/")
|
||
@FormUrlEncoded
|
||
suspend fun createRoom(@Field("room") roomName: String): Response<GxrtcResponse<RoomResponse>>
|
||
|
||
@GET("room/")
|
||
suspend fun getRooms(): Response<GxrtcResponse<List<RoomResponse>>>
|
||
}
|
||
|
||
// data/remote/ArAnnotationApi.kt - 新增
|
||
interface ArAnnotationApi {
|
||
@POST("ar/anchor/save")
|
||
suspend fun saveAnchor(@Body request: SaveAnchorRequest): Response<GxrtcResponse<Unit>>
|
||
|
||
@GET("ar/room/{roomId}/anchors")
|
||
suspend fun getRoomAnchors(@Path("roomId") roomId: String): Response<GxrtcResponse<AnchorListResponse>>
|
||
|
||
@POST("ar/annotation/save")
|
||
suspend fun saveAnnotation(@Body annotation: ArAnnotation): Response<GxrtcResponse<Unit>>
|
||
|
||
@GET("ar/room/{roomId}/annotations")
|
||
suspend fun getRoomAnnotations(@Path("roomId") roomId: String): Response<GxrtcResponse<AnnotationListResponse>>
|
||
|
||
@DELETE("ar/annotation/{annotationId}")
|
||
suspend fun deleteAnnotation(@Path("annotationId") annotationId: String): Response<GxrtcResponse<Unit>>
|
||
|
||
@POST("ar/spatial/validate")
|
||
suspend fun validateSpatialConsistency(@Body request: SpatialValidationRequest): Response<GxrtcResponse<SpatialValidationResult>>
|
||
|
||
@POST("ar/annotations/version")
|
||
suspend fun createAnnotationVersion(@Body request: CreateVersionRequest): Response<GxrtcResponse<VersionResponse>>
|
||
|
||
@GET("ar/annotations/{roomId}/versions")
|
||
suspend fun getAnnotationVersions(@Path("roomId") roomId: String): Response<GxrtcResponse<List<VersionResponse>>>
|
||
}
|
||
|
||
// data/remote/FeaturePointApi.kt - 新增
|
||
interface FeaturePointApi {
|
||
@POST("ar/feature-points/save")
|
||
suspend fun saveFeaturePoints(@Body request: SaveFeaturePointsRequest): Response<GxrtcResponse<Unit>>
|
||
|
||
@GET("ar/feature-points/{anchorId}")
|
||
suspend fun getAnchorFeatures(@Path("anchorId") anchorId: String): Response<GxrtcResponse<FeaturePointListResponse>>
|
||
|
||
@POST("ar/feature-points/match")
|
||
suspend fun matchFeaturePoints(@Body request: FeatureMatchRequest): Response<GxrtcResponse<FeatureMatchResponse>>
|
||
|
||
@POST("ar/feature-points/quality-assessment")
|
||
suspend fun assessFeatureQuality(@Body request: FeatureQualityRequest): Response<GxrtcResponse<FeatureQualityResponse>>
|
||
}
|
||
|
||
// 请求和响应数据模型
|
||
data class SaveAnchorRequest(
|
||
val roomId: String,
|
||
val anchorId: String,
|
||
val position: List<Float>,
|
||
val rotation: List<Float>,
|
||
val createdBy: String
|
||
)
|
||
|
||
data class AnchorListResponse(val anchors: List<AnchorData>)
|
||
|
||
data class AnnotationListResponse(val annotations: List<ArAnnotation>)
|
||
|
||
data class SaveFeaturePointsRequest(
|
||
val anchorId: String,
|
||
val roomId: String,
|
||
val featurePoints: List<FeaturePoint>,
|
||
val deviceInfo: DeviceInfo
|
||
)
|
||
|
||
data class FeaturePointListResponse(val featurePoints: List<FeaturePoint>)
|
||
|
||
data class SpatialValidationRequest(
|
||
val anchorId: String,
|
||
val currentPosition: List<Float>,
|
||
val currentRotation: List<Float>,
|
||
val deviceInfo: DeviceInfo
|
||
)
|
||
|
||
data class SpatialValidationResult(
|
||
val isValid: Boolean,
|
||
val confidence: Float,
|
||
val suggestedCorrection: SpatialTransform?,
|
||
val errorMessage: String?
|
||
)
|
||
|
||
data class FeatureMatchRequest(
|
||
val anchorId: String,
|
||
val queryFeatures: List<FeaturePoint>,
|
||
val maxResults: Int = 10
|
||
)
|
||
|
||
data class FeatureMatchResponse(
|
||
val matches: List<FeatureMatch>,
|
||
val confidence: Float
|
||
)
|
||
|
||
data class FeatureMatch(
|
||
val featurePoint: FeaturePoint,
|
||
val similarity: Float,
|
||
val spatialDistance: Float
|
||
)
|
||
|
||
data class FeatureQualityRequest(
|
||
val anchorId: String,
|
||
val featurePoints: List<FeaturePoint>
|
||
)
|
||
|
||
data class FeatureQualityResponse(
|
||
val overallQuality: Float,
|
||
val individualScores: Map<String, Float>,
|
||
val recommendations: List<String>
|
||
)
|
||
|
||
data class CreateVersionRequest(
|
||
val roomId: String,
|
||
val versionName: String,
|
||
val description: String,
|
||
val createdBy: String
|
||
)
|
||
|
||
data class VersionResponse(
|
||
val versionId: String,
|
||
val versionName: String,
|
||
val description: String,
|
||
val createdAt: Long,
|
||
val createdBy: String,
|
||
val annotationCount: Int
|
||
)
|
||
|
||
// 响应数据结构
|
||
data class GxrtcResponse<T>(
|
||
@SerializedName("meta") val meta: Meta,
|
||
@SerializedName("data") val data: T
|
||
)
|
||
|
||
data class Meta(
|
||
@SerializedName("code") val code: Int,
|
||
@SerializedName("message") val message: String? = null
|
||
)
|
||
|
||
data class AccessTokenResponse(@SerializedName("access_token") val accessToken: String)
|
||
|
||
// data/model/ChatMessage.kt - 新增
|
||
data class ChatMessage(
|
||
val id: String,
|
||
val senderId: String,
|
||
val message: String,
|
||
val timestamp: Long,
|
||
val type: String = "text", // text, image, file
|
||
val isPrivate: Boolean = false
|
||
)
|
||
|
||
// data/model/PrivateMessage.kt - 新增
|
||
data class PrivateMessage(
|
||
val id: String,
|
||
val senderId: String,
|
||
val recipientId: String,
|
||
val message: String,
|
||
val timestamp: Long,
|
||
val isRead: Boolean = false
|
||
)
|
||
|
||
// data/model/LaserPointerData.kt - 新增
|
||
data class LaserPointerData(
|
||
val start: List<Double>, // [x, y] 相对坐标
|
||
val end: List<Double>, // [x, y] 相对坐标
|
||
val color: List<Int>, // [r, g, b]
|
||
val thickness: Int, // 像素粗细
|
||
val timestamp: Long // 时间戳
|
||
)
|
||
|
||
// data/model/WhiteboardModels.kt - 新增
|
||
sealed class WhiteboardState {
|
||
object Inactive : WhiteboardState()
|
||
data class Active(
|
||
val tools: List<WhiteboardTool>,
|
||
val currentTool: WhiteboardTool,
|
||
val strokes: List<WhiteboardStroke>
|
||
) : WhiteboardState()
|
||
}
|
||
|
||
sealed class WhiteboardTool {
|
||
data class Brush(val color: Color, val size: Float) : WhiteboardTool()
|
||
data class Eraser(val size: Float) : WhiteboardTool()
|
||
data class Shape(val shapeType: ShapeType) : WhiteboardTool()
|
||
data class Text(val fontSize: Float = 16f) : WhiteboardTool()
|
||
}
|
||
|
||
enum class ShapeType { Rectangle, Circle, Triangle, Arrow }
|
||
|
||
enum class WhiteboardToolType { BRUSH, ERASER, SHAPE, TEXT }
|
||
|
||
data class WhiteboardStroke(
|
||
val points: List<PointF>,
|
||
val color: Color,
|
||
val size: Float,
|
||
val toolType: WhiteboardToolType
|
||
)
|
||
```
|
||
|
||
#### 1.2 MQTT客户端集成
|
||
```kotlin
|
||
// data/remote/MqttManager.kt - 新增
|
||
class MqttManager @Inject constructor(
|
||
@ApplicationContext private val context: Context,
|
||
private val userRepository: UserRepository
|
||
) {
|
||
private var mqttClient: MqttAndroidClient? = null
|
||
private val _mqttState = MutableStateFlow<MqttConnectionState>(MqttConnectionState.Disconnected)
|
||
val mqttState: StateFlow<MqttConnectionState> = _mqttState.asStateFlow()
|
||
|
||
suspend fun connect() {
|
||
val serverUri = "tcp://mqtt.cnsdt.com:1883"
|
||
val clientId = MqttClient.generateClientId()
|
||
|
||
mqttClient = MqttAndroidClient(context, serverUri, clientId).apply {
|
||
setCallback(object : MqttCallback {
|
||
override fun connectionLost(cause: Throwable?) {
|
||
_mqttState.value = MqttConnectionState.Disconnected
|
||
}
|
||
|
||
override fun messageArrived(topic: String, message: MqttMessage) {
|
||
handleIncomingMessage(topic, message)
|
||
}
|
||
|
||
override fun deliveryComplete(token: IMqttDeliveryToken?) {}
|
||
})
|
||
}
|
||
|
||
val options = MqttConnectOptions().apply {
|
||
isCleanSession = true
|
||
connectionTimeout = 30
|
||
keepAliveInterval = 60
|
||
}
|
||
|
||
try {
|
||
mqttClient?.connect(options)
|
||
_mqttState.value = MqttConnectionState.Connected
|
||
subscribeToUserTopics()
|
||
} catch (e: Exception) {
|
||
_mqttState.value = MqttConnectionState.Error(e.message ?: "Connection failed")
|
||
}
|
||
}
|
||
|
||
private fun subscribeToUserTopics() {
|
||
val user = userRepository.getCurrentUser()
|
||
user?.let {
|
||
val userTopic = "/zrsk/remote-assistant/user/${it.id}"
|
||
mqttClient?.subscribe(userTopic, 1)
|
||
}
|
||
}
|
||
|
||
private fun handleIncomingMessage(topic: String, message: MqttMessage) {
|
||
val payload = String(message.payload, Charsets.UTF_8)
|
||
val messageData = Json.decodeFromString<MqttMessageData>(payload)
|
||
|
||
when (messageData.event) {
|
||
"call" -> handleCallMessage(messageData.data)
|
||
"call-cancel" -> handleCallCancelMessage(messageData.data)
|
||
"laser-pointer" -> handleLaserPointerMessage(messageData.data)
|
||
"chat-message" -> handleChatMessage(messageData.data)
|
||
"private-message" -> handlePrivateMessage(messageData.data)
|
||
"whiteboard-state" -> handleWhiteboardMessage(messageData.data)
|
||
"whiteboard-stroke" -> handleWhiteboardMessage(messageData.data)
|
||
// 其他消息类型...
|
||
}
|
||
}
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] GXRTC API集成正确，能成功获取access token
|
||
- [ ] MQTT客户端能正常连接和接收消息，连接成功率>99.9%
|
||
- [ ] 消息解析和处理框架完整，支持所有消息类型
|
||
- [ ] 用户认证状态持久化，应用重启后保持登录状态
|
||
- [ ] 网络错误处理完善，有明确的用户提示
|
||
- [ ] 单元测试覆盖率>85%，关键业务逻辑100%覆盖
|
||
|
||
---
|
||
|
||
## 📋 阶段2：实时通信基础（增强版）
|
||
|
||
### 🎯 目标
|
||
- 集成LiveKit with正确的token获取流程
|
||
- 实现完整的音视频通话功能
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 2.1 正确的LiveKit连接流程
|
||
```kotlin
|
||
// domain/manager/LiveKitManager.kt - 修正
|
||
class LiveKitManager @Inject constructor(
|
||
@ApplicationContext private val context: Context,
|
||
private val gxrtcApi: GxrtcApi,
|
||
private val userRepository: UserRepository
|
||
) {
|
||
suspend fun connectToRoom(roomName: String): Result<Room> {
|
||
return try {
|
||
// 1. 获取access token
|
||
val user = userRepository.getCurrentUser()
|
||
val tokenResponse = gxrtcApi.getAccessToken(user.id, roomName)
|
||
|
||
if (!tokenResponse.isSuccessful || tokenResponse.body()?.meta?.code != 200) {
|
||
return Result.failure(Exception("Failed to get access token"))
|
||
}
|
||
|
||
val accessToken = tokenResponse.body()!!.data.accessToken
|
||
|
||
// 2. 连接LiveKit
|
||
val newRoom = LiveKit.create(
|
||
appContext = context,
|
||
options = RoomOptions(
|
||
adaptiveStream = true,
|
||
dynacast = true,
|
||
e2eeOptions = E2EEOptions(
|
||
keyProvider = BaseKeyProvider("encryption-key")
|
||
)
|
||
)
|
||
)
|
||
|
||
newRoom.connect("wss://meeting.cnsdt.com", accessToken)
|
||
Result.success(newRoom)
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
}
|
||
```
|
||
|
||
#### 2.2 MQTT房间消息订阅
|
||
```kotlin
|
||
// 在进入房间时订阅相关主题
|
||
fun subscribeToRoomTopics(roomId: String) {
|
||
val roomTopic = "/zrsk/remote-assistant/room/$roomId"
|
||
val userRoomTopic = "/zrsk/remote-assistant/room/$roomId/user/${currentUser.id}"
|
||
|
||
mqttClient?.subscribe(roomTopic, 1)
|
||
mqttClient?.subscribe(userRoomTopic, 1)
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 使用GXRTC token成功连接LiveKit，连接时间<3s
|
||
- [ ] 房间相关的MQTT消息能正确接收和处理，消息延迟<200ms
|
||
- [ ] 断线重连机制完善，网络恢复后30s内自动重连
|
||
- [ ] 音视频权限处理正确，用户拒绝权限时有友好提示
|
||
- [ ] 本地视频预览正常，画面清晰无卡顿
|
||
- [ ] 远程视频渲染正常，支持多种分辨率和帧率
|
||
|
||
---
|
||
|
||
## 📋 阶段3：协作会话管理（增强版）
|
||
|
||
### 🎯 目标
|
||
- 完整的会议生命周期管理
|
||
- MQTT消息驱动的呼叫系统
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 3.1 呼叫系统实现
|
||
```kotlin
|
||
// domain/manager/CallManager.kt - 新增
|
||
class CallManager @Inject constructor(
|
||
private val mqttManager: MqttManager,
|
||
private val userRepository: UserRepository
|
||
) {
|
||
fun sendCallInvitation(roomId: String, roomTitle: String, targetUserId: String) {
|
||
val callMessage = MqttMessageData(
|
||
event = "call",
|
||
data = mapOf(
|
||
"room_id" to roomId,
|
||
"title" to roomTitle,
|
||
"caller" to userRepository.getCurrentUser()?.id ?: ""
|
||
)
|
||
)
|
||
|
||
val topic = "/zrsk/remote-assistant/user/$targetUserId"
|
||
mqttManager.publish(topic, callMessage)
|
||
}
|
||
|
||
fun handleCallMessage(data: Map<String, Any>) {
|
||
val roomId = data["room_id"] as String
|
||
val title = data["title"] as String
|
||
val callerId = data["caller"] as String
|
||
|
||
// 显示呼叫UI，用户可以选择接听或拒绝
|
||
_incomingCall.value = IncomingCall(roomId, title, callerId)
|
||
}
|
||
}
|
||
```
|
||
|
||
#### 3.2 会议状态管理
|
||
```kotlin
|
||
// 处理会议结束消息
|
||
private fun handleEndMeetingMessage(data: Map<String, Any>) {
|
||
val reason = data["reason"] as? String
|
||
val code = data["code"] as Int
|
||
|
||
when (code) {
|
||
1 -> showToast("用户挂断了会议")
|
||
2 -> showToast("管理员结束了会议")
|
||
else -> showToast("会议已结束")
|
||
}
|
||
|
||
// 退出会议界面
|
||
navigateToHome()
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 呼叫邀请能正确发送和接收，送达率100%
|
||
- [ ] 会议状态变化能正确处理，状态同步延迟<1s
|
||
- [ ] 用户加入/离开通知完善，实时更新参与者列表
|
||
- [ ] 会议录制功能正常，录制文件可完整回放
|
||
- [ ] 会议时长统计准确，支持暂停/继续计时
|
||
- [ ] 多语言支持完整，中英文界面无缝切换
|
||
|
||
---
|
||
|
||
## 📋 阶段4：AR环境集成（增强版）
|
||
|
||
### 🎯 目标
|
||
- ARCore环境集成与平面检测
|
||
- 后置摄像头默认配置与空间锚点同步
|
||
- 服务器端AR标注持久化存储
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 4.1 AR场景管理与后置摄像头配置
|
||
```kotlin
|
||
// ui/ar/ArSceneManager.kt - 增强
|
||
class ArSceneManager @Inject constructor(
|
||
private val mqttManager: MqttManager,
|
||
private val arAnnotationService: ArAnnotationService
|
||
) {
|
||
private var arSession: Session? = null
|
||
private val cameraConfig = CameraConfig().apply {
|
||
facingDirection = CameraConfig.FacingDirection.BACK // 默认后置摄像头
|
||
depthSensorUsage = CameraConfig.DepthSensorUsage.AUTOMATIC
|
||
}
|
||
|
||
suspend fun initializeArSession(context: Context): Result<Session> {
|
||
return try {
|
||
val session = Session(context)
|
||
session.cameraConfig = cameraConfig
|
||
session.configure(session.config)
|
||
|
||
arSession = session
|
||
Result.success(session)
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
|
||
fun createAnchorAtPosition(position: Vector3): Anchor {
|
||
val anchor = arSession!!.createAnchor(Pose(position, Quaternion.identity()))
|
||
|
||
// 同步到服务器和其他用户
|
||
syncAnchorToParticipants(anchor)
|
||
|
||
return anchor
|
||
}
|
||
|
||
private suspend fun syncAnchorToParticipants(anchor: Anchor) {
|
||
// 保存到服务器
|
||
arAnnotationService.saveAnchor(
|
||
roomId = currentRoomId,
|
||
anchorId = anchor.trackableId,
|
||
position = anchor.pose
|
||
)
|
||
|
||
// 同步到其他用户
|
||
val anchorData = mapOf(
|
||
"anchor_id" to anchor.trackableId,
|
||
"position" to listOf(anchor.pose.tx(), anchor.pose.ty(), anchor.pose.tz()),
|
||
"rotation" to listOf(anchor.pose.qx(), anchor.pose.qy(), anchor.pose.qz(), anchor.pose.qw())
|
||
)
|
||
|
||
val message = MqttMessageData(
|
||
event = "ar-anchor-create",
|
||
data = anchorData
|
||
)
|
||
mqttManager.publishToRoom(roomId, message)
|
||
}
|
||
|
||
suspend fun restoreAnchors(roomId: String): List<Anchor> {
|
||
val savedAnchors = arAnnotationService.loadRoomAnchors(roomId)
|
||
return savedAnchors.map { anchorData ->
|
||
val pose = Pose(
|
||
floatArrayOf(anchorData.position[0], anchorData.position[1], anchorData.position[2]),
|
||
floatArrayOf(anchorData.rotation[0], anchorData.rotation[1], anchorData.rotation[2], anchorData.rotation[3])
|
||
)
|
||
arSession!!.createAnchor(pose)
|
||
}
|
||
}
|
||
}
|
||
|
||
// data/model/AnchorData.kt - 新增
|
||
data class AnchorData(
|
||
val anchorId: String,
|
||
val position: List<Float>,
|
||
val rotation: List<Float>,
|
||
val roomId: String,
|
||
val createdAt: Long = System.currentTimeMillis(),
|
||
val createdBy: String
|
||
)
|
||
```
|
||
|
||
#### 4.2 AR标注持久化服务（增强版）
|
||
```kotlin
|
||
// domain/service/ArAnnotationService.kt - 新增
|
||
class ArAnnotationService @Inject constructor(
|
||
private val arAnnotationApi: ArAnnotationApi,
|
||
private val featurePointApi: FeaturePointApi,
|
||
private val userRepository: UserRepository
|
||
) {
|
||
suspend fun saveAnchorWithFeatures(
|
||
roomId: String,
|
||
anchorId: String,
|
||
position: Pose,
|
||
featurePoints: List<FeaturePoint>,
|
||
deviceInfo: DeviceInfo
|
||
): Result<Unit> {
|
||
return try {
|
||
// 保存锚点基本信息
|
||
val anchorRequest = SaveAnchorRequest(
|
||
roomId = roomId,
|
||
anchorId = anchorId,
|
||
position = listOf(position.tx(), position.ty(), position.tz()),
|
||
rotation = listOf(position.qx(), position.qy(), position.qz(), position.qw()),
|
||
createdBy = userRepository.getCurrentUser()?.id ?: ""
|
||
)
|
||
arAnnotationApi.saveAnchor(anchorRequest)
|
||
|
||
// 保存特征点数据
|
||
val featureRequest = SaveFeaturePointsRequest(
|
||
anchorId = anchorId,
|
||
roomId = roomId,
|
||
featurePoints = featurePoints,
|
||
deviceInfo = deviceInfo
|
||
)
|
||
featurePointApi.saveFeaturePoints(featureRequest)
|
||
|
||
Result.success(Unit)
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
|
||
suspend fun loadRoomAnchorsWithFeatures(roomId: String): List<AnchorWithFeatures> {
|
||
val anchors = arAnnotationApi.getRoomAnchors(roomId).anchors
|
||
return anchors.map { anchor ->
|
||
val features = featurePointApi.getAnchorFeatures(anchor.anchorId).featurePoints
|
||
AnchorWithFeatures(anchor, features)
|
||
}
|
||
}
|
||
|
||
suspend fun saveAnnotation(annotation: ArAnnotation): Result<Unit> {
|
||
return try {
|
||
arAnnotationApi.saveAnnotation(annotation)
|
||
Result.success(Unit)
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
|
||
suspend fun loadRoomAnnotations(roomId: String): List<ArAnnotation> {
|
||
return arAnnotationApi.getRoomAnnotations(roomId).annotations
|
||
}
|
||
|
||
suspend fun validateSpatialConsistency(
|
||
anchorId: String,
|
||
currentPose: Pose,
|
||
deviceInfo: DeviceInfo
|
||
): Result<SpatialValidationResult> {
|
||
return try {
|
||
val request = SpatialValidationRequest(
|
||
anchorId = anchorId,
|
||
currentPosition = listOf(currentPose.tx(), currentPose.ty(), currentPose.tz()),
|
||
currentRotation = listOf(currentPose.qx(), currentPose.qy(), currentPose.qz(), currentPose.qw()),
|
||
deviceInfo = deviceInfo
|
||
)
|
||
val result = arAnnotationApi.validateSpatialConsistency(request)
|
||
Result.success(result)
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 数据模型增强
|
||
data class AnchorWithFeatures(
|
||
val anchor: AnchorData,
|
||
val featurePoints: List<FeaturePoint>
|
||
)
|
||
|
||
data class FeaturePoint(
|
||
val position: List<Float>,
|
||
val descriptor: String, // base64编码的特征描述符
|
||
val qualityScore: Float,
|
||
val timestamp: Long = System.currentTimeMillis()
|
||
)
|
||
|
||
data class DeviceInfo(
|
||
val model: String,
|
||
val arCoreVersion: String,
|
||
val cameraCalibration: CameraCalibration?,
|
||
val imuCalibration: ImuCalibration?
|
||
)
|
||
|
||
data class CameraCalibration(
|
||
val focalLength: List<Float>,
|
||
val principalPoint: List<Float>,
|
||
val distortionCoefficients: List<Float>?
|
||
)
|
||
|
||
data class ImuCalibration(
|
||
val gyroBias: List<Float>?,
|
||
val accelBias: List<Float>?,
|
||
val magnetometerBias: List<Float>?
|
||
)
|
||
|
||
data class ArAnnotation(
|
||
val id: String,
|
||
val anchorId: String,
|
||
val type: AnnotationType,
|
||
val content: String,
|
||
val color: List<Int>,
|
||
val size: Float,
|
||
val roomId: String,
|
||
val createdBy: String,
|
||
val spatialTransform: SpatialTransform? = null,
|
||
val createdAt: Long = System.currentTimeMillis(),
|
||
val updatedAt: Long = System.currentTimeMillis(),
|
||
val version: Int = 1
|
||
)
|
||
|
||
data class SpatialTransform(
|
||
val position: List<Float>,
|
||
val rotation: List<Float>,
|
||
val scale: List<Float>,
|
||
val confidence: Float
|
||
)
|
||
|
||
enum class AnnotationType {
|
||
ARROW, CIRCLE, RECTANGLE, TEXT, HIGHLIGHT, MEASUREMENT, NOTE
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] AR场景能正确初始化和渲染，初始化时间<5s
|
||
- [ ] 默认使用后置摄像头，支持自动切换
|
||
- [ ] 空间锚点能跨设备同步，位置误差<5cm
|
||
- [ ] 服务器端锚点存储可靠，存储成功率>99.9%
|
||
- [ ] 锚点恢复准确，重定位误差<10cm
|
||
- [ ] 支持会话间AR标注持久化恢复
|
||
- [ ] 平面检测准确稳定，检测成功率>95%
|
||
- [ ] 环境光估计准确，虚拟物体光照一致
|
||
|
||
---
|
||
|
||
## 📋 阶段5：AR标注工具（增强版）
|
||
|
||
### 🎯 目标
|
||
- 完整的AR标注系统
|
||
- 实时同步机制
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 5.1 激光笔工具实现
|
||
```kotlin
|
||
// ui/ar/tools/LaserPointerTool.kt - 新增
|
||
class LaserPointerTool @Inject constructor(
|
||
private val mqttManager: MqttManager
|
||
) : ArTool {
|
||
|
||
fun onTouchStart(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) {
|
||
val startX = screenX / screenWidth
|
||
val startY = screenY / screenHeight
|
||
|
||
currentLaserData = LaserData(
|
||
start = listOf(startX, startY),
|
||
end = listOf(startX, startY),
|
||
color = listOf(255, 0, 0), // 红色
|
||
thickness = 3
|
||
)
|
||
}
|
||
|
||
fun onTouchMove(screenX: Float, screenY: Float, screenWidth: Int, screenHeight: Int) {
|
||
val endX = screenX / screenWidth
|
||
val endY = screenY / screenHeight
|
||
|
||
currentLaserData = currentLaserData?.copy(end = listOf(endX, endY))
|
||
|
||
// 实时发送激光笔位置
|
||
sendLaserPointerData()
|
||
}
|
||
|
||
fun onTouchEnd() {
|
||
// 发送最终的激光笔数据
|
||
sendLaserPointerData()
|
||
currentLaserData = null
|
||
}
|
||
|
||
private fun sendLaserPointerData() {
|
||
currentLaserData?.let { laserData ->
|
||
val message = MqttMessageData(
|
||
event = "laser-pointer",
|
||
data = mapOf(
|
||
"start" to laserData.start,
|
||
"end" to laserData.end,
|
||
"color" to laserData.color,
|
||
"thickness" to laserData.thickness
|
||
)
|
||
)
|
||
|
||
mqttManager.publishToRoom(roomId, message)
|
||
}
|
||
}
|
||
|
||
private fun handleLaserPointerMessage(data: Map<String, Any>) {
|
||
val start = data["start"] as List<Double>
|
||
val end = data["end"] as List<Double>
|
||
val color = data["color"] as List<Int>
|
||
val thickness = data["thickness"] as Int
|
||
|
||
// 更新激光笔显示
|
||
_laserPointerData.value = LaserPointerData(
|
||
start = start,
|
||
end = end,
|
||
color = color,
|
||
thickness = thickness,
|
||
timestamp = System.currentTimeMillis()
|
||
)
|
||
}
|
||
|
||
private fun handleChatMessage(data: Map<String, Any>) {
|
||
val senderId = data["sender_id"] as String
|
||
val message = data["message"] as String
|
||
val timestamp = data["timestamp"] as Long
|
||
val messageType = data["type"] as? String ?: "text"
|
||
|
||
// 更新聊天界面
|
||
_chatMessages.value = _chatMessages.value + ChatMessage(
|
||
id = UUID.randomUUID().toString(),
|
||
senderId = senderId,
|
||
message = message,
|
||
timestamp = timestamp,
|
||
type = messageType,
|
||
isPrivate = false
|
||
)
|
||
}
|
||
|
||
private fun handlePrivateMessage(data: Map<String, Any>) {
|
||
val senderId = data["sender_id"] as String
|
||
val message = data["message"] as String
|
||
val timestamp = data["timestamp"] as Long
|
||
val recipientId = data["recipient_id"] as String
|
||
|
||
// 更新私聊界面
|
||
_privateMessages.value = _privateMessages.value + PrivateMessage(
|
||
id = UUID.randomUUID().toString(),
|
||
senderId = senderId,
|
||
recipientId = recipientId,
|
||
message = message,
|
||
timestamp = timestamp
|
||
)
|
||
}
|
||
|
||
fun sendChatMessage(message: String, isPrivate: Boolean = false, recipientId: String? = null) {
|
||
val messageData = if (isPrivate && recipientId != null) {
|
||
MqttMessageData(
|
||
event = "private-message",
|
||
data = mapOf(
|
||
"sender_id" to currentUser.id,
|
||
"recipient_id" to recipientId,
|
||
"message" to message,
|
||
"timestamp" to System.currentTimeMillis()
|
||
)
|
||
)
|
||
} else {
|
||
MqttMessageData(
|
||
event = "chat-message",
|
||
data = mapOf(
|
||
"sender_id" to currentUser.id,
|
||
"message" to message,
|
||
"timestamp" to System.currentTimeMillis(),
|
||
"type" to "text"
|
||
)
|
||
)
|
||
}
|
||
|
||
val topic = if (isPrivate) {
|
||
"/zrsk/remote-assistant/user/$recipientId"
|
||
} else {
|
||
"/zrsk/remote-assistant/room/$currentRoomId"
|
||
}
|
||
|
||
mqttManager.publish(topic, messageData)
|
||
}
|
||
}
|
||
```
|
||
|
||
#### 3.3 文字聊天界面实现
|
||
```kotlin
|
||
// ui/chat/ChatViewModel.kt - 新增
|
||
@HiltViewModel
|
||
class ChatViewModel @Inject constructor(
|
||
private val mqttManager: MqttManager
|
||
) : ViewModel() {
|
||
|
||
private val _chatMessages = MutableStateFlow<List<ChatMessage>>(emptyList())
|
||
val chatMessages: StateFlow<List<ChatMessage>> = _chatMessages.asStateFlow()
|
||
|
||
private val _privateMessages = MutableStateFlow<List<PrivateMessage>>(emptyList())
|
||
val privateMessages: StateFlow<List<PrivateMessage>> = _privateMessages.asStateFlow()
|
||
|
||
fun loadChatHistory(roomId: String) {
|
||
// 从服务器加载聊天历史
|
||
}
|
||
|
||
fun clearChatHistory() {
|
||
_chatMessages.value = emptyList()
|
||
}
|
||
}
|
||
|
||
// ui/chat/ChatScreen.kt - 新增
|
||
@Composable
|
||
fun ChatScreen(
|
||
viewModel: ChatViewModel = hiltViewModel(),
|
||
onBackPressed: () -> Unit
|
||
) {
|
||
val messages by viewModel.chatMessages.collectAsState()
|
||
var newMessage by remember { mutableStateOf("") }
|
||
|
||
Column(modifier = Modifier.fillMaxSize()) {
|
||
// 消息列表
|
||
LazyColumn(modifier = Modifier.weight(1f)) {
|
||
items(messages) { message ->
|
||
ChatMessageItem(message = message)
|
||
}
|
||
}
|
||
|
||
// 输入框
|
||
Row(
|
||
modifier = Modifier
|
||
.fillMaxWidth()
|
||
.padding(8.dp),
|
||
verticalAlignment = Alignment.CenterVertically
|
||
) {
|
||
TextField(
|
||
value = newMessage,
|
||
onValueChange = { newMessage = it },
|
||
modifier = Modifier.weight(1f),
|
||
placeholder = { Text("输入消息...") }
|
||
)
|
||
|
||
Button(
|
||
onClick = {
|
||
if (newMessage.isNotBlank()) {
|
||
viewModel.sendMessage(newMessage)
|
||
newMessage = ""
|
||
}
|
||
},
|
||
modifier = Modifier.padding(start = 8.dp)
|
||
) {
|
||
Text("发送")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 激光笔工具实时同步，端到端延迟<100ms
|
||
- [ ] AR标注空间位置准确，位置误差<2cm
|
||
- [ ] 多用户同时标注无冲突，支持并发标注
|
||
- [ ] 标注持久化存储，支持会话间恢复
|
||
- [ ] 标注工具丰富，支持箭头、圆圈、文字等多种类型
|
||
- [ ] 标注颜色和大小可自定义，视觉效果清晰
|
||
|
||
---
|
||
|
||
## 📋 阶段6：协作增强功能（增强版）
|
||
|
||
### 🎯 目标
|
||
- 白板、屏幕共享、文件传输
|
||
- 实时字幕和会议纪要
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 6.1 白板协作实现
|
||
```kotlin
|
||
// ui/whiteboard/WhiteboardManager.kt - 新增
|
||
class WhiteboardManager @Inject constructor(
|
||
private val mqttManager: MqttManager
|
||
) {
|
||
private val _whiteboardState = MutableStateFlow<WhiteboardState>(WhiteboardState.Inactive)
|
||
val whiteboardState: StateFlow<WhiteboardState> = _whiteboardState.asStateFlow()
|
||
|
||
fun enableWhiteboard() {
|
||
_whiteboardState.value = WhiteboardState.Active(
|
||
tools = listOf(
|
||
WhiteboardTool.Brush(color = Color.Red, size = 3f),
|
||
WhiteboardTool.Eraser(size = 10f),
|
||
WhiteboardTool.Shape(shapeType = ShapeType.Rectangle),
|
||
WhiteboardTool.Text()
|
||
),
|
||
currentTool = WhiteboardTool.Brush(color = Color.Red, size = 3f),
|
||
strokes = emptyList()
|
||
)
|
||
|
||
// 通知其他用户白板已开启
|
||
sendWhiteboardState(true)
|
||
}
|
||
|
||
fun disableWhiteboard() {
|
||
_whiteboardState.value = WhiteboardState.Inactive
|
||
sendWhiteboardState(false)
|
||
}
|
||
|
||
fun addStroke(stroke: WhiteboardStroke) {
|
||
val currentState = _whiteboardState.value
|
||
if (currentState is WhiteboardState.Active) {
|
||
_whiteboardState.value = currentState.copy(
|
||
strokes = currentState.strokes + stroke
|
||
)
|
||
|
||
// 同步到其他用户
|
||
sendStroke(stroke)
|
||
}
|
||
}
|
||
|
||
private fun sendWhiteboardState(enabled: Boolean) {
|
||
val message = MqttMessageData(
|
||
event = "whiteboard-state",
|
||
data = mapOf("enabled" to enabled)
|
||
)
|
||
mqttManager.publishToRoom(roomId, message)
|
||
}
|
||
|
||
private fun sendStroke(stroke: WhiteboardStroke) {
|
||
val message = MqttMessageData(
|
||
event = "whiteboard-stroke",
|
||
data = mapOf(
|
||
"points" to stroke.points.map { listOf(it.x, it.y) },
|
||
"color" to listOf(stroke.color.red, stroke.color.green, stroke.color.blue),
|
||
"size" to stroke.size,
|
||
"tool_type" to stroke.toolType.name
|
||
)
|
||
)
|
||
mqttManager.publishToRoom(roomId, message)
|
||
}
|
||
|
||
fun handleWhiteboardMessage(data: Map<String, Any>) {
|
||
when (data["event"] as String) {
|
||
"whiteboard-state" -> {
|
||
val enabled = data["enabled"] as Boolean
|
||
if (enabled) {
|
||
enableWhiteboard()
|
||
} else {
|
||
disableWhiteboard()
|
||
}
|
||
}
|
||
"whiteboard-stroke" -> {
|
||
val points = (data["points"] as List<List<Float>>).map { PointF(it[0], it[1]) }
|
||
val color = Color(
|
||
red = (data["color"] as List<Int>)[0],
|
||
green = (data["color"] as List<Int>)[1],
|
||
blue = (data["color"] as List<Int>)[2]
|
||
)
|
||
val size = data["size"] as Float
|
||
val toolType = WhiteboardToolType.valueOf(data["tool_type"] as String)
|
||
|
||
val stroke = WhiteboardStroke(points, color, size, toolType)
|
||
addStroke(stroke)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
```
|
||
|
||
#### 6.2 实时字幕集成
|
||
```kotlin
|
||
// 处理ASR字幕消息
|
||
private fun handleAsrSubtitleMessage(data: Map<String, Any>) {
|
||
val uid = data["uid"] as String
|
||
val segment = data["seg"] as String
|
||
val status = data["status"] as String
|
||
|
||
when (status) {
|
||
"streaming" -> updateLiveSubtitle(uid, segment)
|
||
"final" -> finalizeSubtitle(uid, segment)
|
||
}
|
||
}
|
||
```
|
||
|
||
#### 6.2 文件预览功能
|
||
```kotlin
|
||
// 处理文件预览消息
|
||
private fun handleFilePreviewMessage(data: Map<String, Any>) {
|
||
val fileName = data["name"] as String
|
||
val filePath = data["path"] as String
|
||
val mimeType = data["mime_type"] as String
|
||
|
||
val fileUrl = "https://meeting.cnsdt.com$filePath"
|
||
|
||
// 显示文件预览界面
|
||
showFilePreview(fileName, fileUrl, mimeType)
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 实时字幕准确率>90%，行业术语识别准确
|
||
- [ ] 文件预览功能完整，支持PDF、Word、Excel、图片等格式
|
||
- [ ] 屏幕共享流畅，帧率>15fps，延迟<300ms
|
||
- [ ] 白板协作实时同步，操作延迟<200ms
|
||
- [ ] 文件传输可靠，支持断点续传，100MB文件传输成功率>99%
|
||
- [ ] 共同浏览功能正常，页面同步准确
|
||
|
||
---
|
||
|
||
## 📋 阶段7：AI能力集成（增强版）
|
||
|
||
### 🎯 目标
|
||
- 语音识别、会议纪要、知识库
|
||
- 智能助手集成
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 7.1 知识库查询集成
|
||
```kotlin
|
||
// domain/usecase/QueryKnowledgeBaseUseCase.kt - 新增
|
||
class QueryKnowledgeBaseUseCase @Inject constructor(
|
||
private val aiService: AiService
|
||
) {
|
||
suspend operator fun invoke(query: String): Result<String> {
|
||
return try {
|
||
val response = aiService.queryKnowledgeBase(query)
|
||
if (response.isSuccessful) {
|
||
Result.success(response.body()?.answer ?: "未找到相关信息")
|
||
} else {
|
||
Result.failure(Exception("查询失败"))
|
||
}
|
||
} catch (e: Exception) {
|
||
Result.failure(e)
|
||
}
|
||
}
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 知识库查询响应时间<2秒，相关度>85%
|
||
- [ ] 会议纪要生成准确，关键信息提取完整
|
||
- [ ] AI助手交互流畅，自然语言理解准确
|
||
- [ ] 语音助手响应迅速，语音识别准确率>92%
|
||
- [ ] 智能推荐相关，推荐准确率>80%
|
||
- [ ] 多轮对话支持，上下文理解准确
|
||
|
||
---
|
||
|
||
## 🔌 缺失的服务端API接口（增强版）
|
||
|
||
根据AR标注和持久化需求，需要以下额外的服务端API接口：
|
||
|
||
### 1. AR标注存储API
|
||
```kotlin
|
||
// 保存AR锚点
|
||
POST /ar/anchor/save
|
||
Content-Type: application/json
|
||
{
|
||
"room_id": "string",
|
||
"anchor_id": "string",
|
||
"position": [0.0, 0.0, 0.0],
|
||
"rotation": [0.0, 0.0, 0.0, 1.0],
|
||
"created_by": "user123"
|
||
}
|
||
|
||
// 获取房间所有锚点
|
||
GET /ar/room/{roomId}/anchors
|
||
|
||
// 保存AR标注内容
|
||
POST /ar/annotation/save
|
||
Content-Type: application/json
|
||
{
|
||
"anchor_id": "string",
|
||
"type": "ARROW",
|
||
"content": "检查这里",
|
||
"color": [255, 0, 0],
|
||
"size": 2.0,
|
||
"room_id": "string",
|
||
"created_by": "user123"
|
||
}
|
||
|
||
// 获取房间所有标注
|
||
GET /ar/room/{roomId}/annotations
|
||
|
||
// 删除标注
|
||
DELETE /ar/annotation/{annotationId}
|
||
```
|
||
|
||
### 2. AR会话管理API
|
||
```kotlin
|
||
// 创建AR会话
|
||
POST /ar/session/create
|
||
{
|
||
"room_id": "string",
|
||
"session_name": "设备维护会话",
|
||
"created_by": "user123"
|
||
}
|
||
|
||
// 获取会话历史
|
||
GET /ar/user/{userId}/sessions
|
||
|
||
// 恢复特定会话
|
||
GET /ar/session/{sessionId}
|
||
|
||
// 导出会话数据
|
||
GET /ar/session/{sessionId}/export?format=json|pdf
|
||
```
|
||
|
||
### 3. 空间数据验证API
|
||
```kotlin
|
||
// 验证锚点空间一致性
|
||
POST /ar/anchor/validate
|
||
{
|
||
"anchor_id": "string",
|
||
"position": [0.0, 0.0, 0.0],
|
||
"rotation": [0.0, 0.0, 0.0, 1.0],
|
||
"device_info": {
|
||
"model": "Pixel 6",
|
||
"ar_core_version": "1.50.0"
|
||
}
|
||
}
|
||
|
||
// 批量验证多个锚点
|
||
POST /ar/anchors/validate-batch
|
||
```
|
||
|
||
### 4. 特征点备份API（可选）
|
||
```kotlin
|
||
// 备份ARCore特征点数据
|
||
POST /ar/features/backup
|
||
Content-Type: application/octet-stream
|
||
[二进制特征点数据]
|
||
|
||
// 恢复特征点数据
|
||
GET /ar/features/restore?anchor_id={anchorId}
|
||
|
||
// 特征点相似度匹配
|
||
POST /ar/features/match
|
||
{
|
||
"anchor_id": "string",
|
||
"feature_descriptor": "base64_encoded_data"
|
||
}
|
||
```
|
||
|
||
### 5. 设备校准API
|
||
```kotlin
|
||
// 上报设备校准数据
|
||
POST /ar/device/calibration
|
||
{
|
||
"device_id": "string",
|
||
"calibration_data": {
|
||
"camera_params": {...},
|
||
"imu_calibration": {...},
|
||
"accuracy_score": 0.95
|
||
},
|
||
"ar_core_version": "1.50.0"
|
||
}
|
||
|
||
// 获取设备推荐校准参数
|
||
GET /ar/device/{deviceModel}/calibration
|
||
```
|
||
|
||
### 6. 统计分析API
|
||
```kotlin
|
||
// AR使用统计
|
||
GET /ar/analytics/usage?start_date=2025-01-01&end_date=2025-01-31
|
||
|
||
// 标注准确率统计
|
||
GET /ar/analytics/accuracy?room_id={roomId}
|
||
|
||
// 设备兼容性报告
|
||
GET /ar/analytics/compatibility
|
||
```
|
||
|
||
### 7. 特征点管理API（关键补充）
|
||
```kotlin
|
||
// 保存ARCore特征点数据
|
||
POST /ar/feature-points/save
|
||
Content-Type: application/json
|
||
{
|
||
"anchor_id": "string",
|
||
"room_id": "string",
|
||
"feature_points": [
|
||
{
|
||
"position": [0.0, 0.0, 0.0],
|
||
"descriptor": "base64_encoded_descriptor",
|
||
"quality_score": 0.95
|
||
}
|
||
],
|
||
"device_info": {
|
||
"model": "Pixel 6",
|
||
"ar_core_version": "1.50.0",
|
||
"camera_calibration": {
|
||
"focal_length": [1000.0, 1000.0],
|
||
"principal_point": [500.0, 500.0]
|
||
}
|
||
}
|
||
}
|
||
|
||
// 获取特征点数据用于重定位
|
||
GET /ar/feature-points/{anchorId}
|
||
|
||
// 批量特征点匹配（跨设备空间对齐）
|
||
POST /ar/feature-points/match
|
||
{
|
||
"anchor_id": "string",
|
||
"query_features": [
|
||
{
|
||
"position": [0.0, 0.0, 0.0],
|
||
"descriptor": "base64_encoded_descriptor"
|
||
}
|
||
],
|
||
"max_results": 10
|
||
}
|
||
|
||
// 特征点质量评估
|
||
POST /ar/feature-points/quality-assessment
|
||
{
|
||
"anchor_id": "string",
|
||
"feature_points": [
|
||
{
|
||
"position": [0.0, 0.0, 0.0],
|
||
"descriptor": "base64_encoded_descriptor"
|
||
}
|
||
]
|
||
}
|
||
```
|
||
|
||
### 8. 空间一致性验证API
|
||
```kotlin
|
||
// 空间坐标转换验证
|
||
POST /ar/spatial/validate-transform
|
||
{
|
||
"source_anchor": "anchor_123",
|
||
"target_anchor": "anchor_456",
|
||
"transform_matrix": [
|
||
[1.0, 0.0, 0.0, 0.0],
|
||
[0.0, 1.0, 0.0, 0.0],
|
||
[0.0, 0.0, 1.0, 0.0],
|
||
[1.5, 2.0, 0.5, 1.0]
|
||
],
|
||
"confidence": 0.92
|
||
}
|
||
|
||
// 多设备空间对齐状态
|
||
GET /ar/spatial/alignment-status?room_id={roomId}
|
||
|
||
// 强制空间重新对齐
|
||
POST /ar/spatial/realign
|
||
{
|
||
"room_id": "string",
|
||
"anchor_ids": ["anchor1", "anchor2", "anchor3"]
|
||
}
|
||
```
|
||
|
||
### 9. 标注版本管理API
|
||
```kotlin
|
||
// 创建标注版本
|
||
POST /ar/annotations/version
|
||
{
|
||
"room_id": "string",
|
||
"version_name": "v1.0",
|
||
"description": "初始标注版本",
|
||
"created_by": "user123"
|
||
}
|
||
|
||
// 获取标注版本历史
|
||
GET /ar/annotations/{roomId}/versions
|
||
|
||
// 恢复到特定版本
|
||
POST /ar/annotations/{roomId}/restore/{versionId}
|
||
|
||
// 标注差异比较
|
||
GET /ar/annotations/{roomId}/diff?version1={v1}&version2={v2}
|
||
```
|
||
|
||
### 10. 设备性能优化API
|
||
```kotlin
|
||
// 设备性能基准上报
|
||
POST /ar/device/performance
|
||
{
|
||
"device_model": "Pixel 6",
|
||
"ar_core_version": "1.50.0",
|
||
"performance_metrics": {
|
||
"frame_rate": 60.0,
|
||
"tracking_quality": 0.95,
|
||
"memory_usage": 256,
|
||
"battery_drain": 15.0
|
||
},
|
||
"recommended_settings": {
|
||
"max_annotations": 50,
|
||
"feature_point_density": 0.8,
|
||
"texture_quality": "medium"
|
||
}
|
||
}
|
||
|
||
// 获取设备优化配置
|
||
GET /ar/device/{deviceModel}/optimization
|
||
|
||
// 性能问题诊断
|
||
POST /ar/device/diagnose
|
||
{
|
||
"device_info": {...},
|
||
"performance_issues": ["low_framerate", "high_memory"]
|
||
}
|
||
```
|
||
|
||
---
|
||
|
||
## 📋 阶段8：优化和兼容性（增强版）
|
||
|
||
### 🎯 目标
|
||
- 性能优化、设备兼容
|
||
- 完整的错误处理和日志系统
|
||
|
||
### 🔧 核心实现增强
|
||
|
||
#### 8.1 完整的错误处理
|
||
```kotlin
|
||
// utils/ErrorHandler.kt - 新增
|
||
class ErrorHandler @Inject constructor(
|
||
@ApplicationContext private val context: Context
|
||
) {
|
||
fun handleNetworkError(error: Throwable) {
|
||
when (error) {
|
||
is SocketTimeoutException -> showToast("网络连接超时")
|
||
is ConnectException -> showToast("无法连接到服务器")
|
||
is SSLHandshakeException -> showToast("安全连接失败")
|
||
else -> showToast("网络错误: ${error.message}")
|
||
}
|
||
}
|
||
|
||
fun handleMqttError(error: Throwable) {
|
||
showToast("实时通信连接失败")
|
||
// 尝试重连
|
||
mqttManager.reconnect()
|
||
}
|
||
}
|
||
```
|
||
|
||
### ✅ 增强验收标准
|
||
- [ ] 性能指标达标：冷启动<3s，热启动<1s，标注延迟<100ms
|
||
- [ ] 兼容Android 8.0+和ARCore设备，覆盖主流设备型号
|
||
- [ ] 错误处理完整，用户体验良好，错误提示友好明确
|
||
- [ ] 内存使用优化，1小时会议内存增长<50MB
|
||
- [ ] 电池消耗合理，1小时会议耗电<15%
|
||
- [ ] 网络带宽优化，720p视频通话<800Kbps
|
||
|
||
---
|
||
|
||
## 🧪 增强测试策略
|
||
|
||
### 单元测试增强
|
||
- **MQTT消息解析测试**：验证所有消息类型的正确解析
|
||
- **GXRTC API响应处理测试**：测试各种HTTP状态码和错误处理
|
||
- **AR标注坐标转换测试**：验证屏幕坐标到世界坐标的准确转换
|
||
- **权限管理测试**：摄像头、麦克风、存储权限处理
|
||
- **数据模型测试**：所有数据类的序列化/反序列化
|
||
- **错误处理测试**：网络错误、解析错误、权限错误处理
|
||
|
||
### 集成测试增强
|
||
- **多设备同步测试**：3+设备同时协作，验证标注同步
|
||
- **网络条件模拟测试**：2G/3G/4G/WiFi切换，丢包率20%测试
|
||
- **断线重连测试**：网络中断30秒后自动重连
|
||
- **MQTT消息完整性测试**：消息不丢失、不重复、有序到达
|
||
- **LiveKit连接测试**：Token过期处理、房间满员处理
|
||
- **文件传输测试**：大文件(100MB+)传输完整性验证
|
||
- **AR锚点同步测试**：多设备间空间锚点一致性
|
||
|
||
### UI测试增强
|
||
- **AR场景交互测试**：手势识别、平面检测、标注绘制
|
||
- **实时协作流程测试**：完整会议生命周期UI测试
|
||
- **错误状态UI测试**：网络错误、权限拒绝、服务器错误的UI反馈
|
||
- **性能监控测试**：帧率监控、内存使用、电池消耗
|
||
- **无障碍测试**：屏幕阅读器支持、大字體模式
|
||
- **多语言测试**：中英文界面切换
|
||
|
||
### 性能测试
|
||
- **启动时间测试**：冷启动<3s，热启动<1s
|
||
- **标注延迟测试**：端到端延迟<100ms
|
||
- **内存泄漏测试**：长时间会议无内存泄漏
|
||
- **电池消耗测试**：1小时会议耗电<15%
|
||
- **网络带宽测试**：720p视频通话带宽<1Mbps
|
||
|
||
### 安全测试
|
||
- **数据加密测试**：音视频流端到端加密验证
|
||
- **Token安全测试**：Token过期和刷新机制
|
||
- **权限提升测试**：防止未授权功能访问
|
||
- **输入验证测试**：防止注入攻击
|
||
- **安全存储测试**：敏感数据加密存储
|
||
|
||
## 🔌 缺失的服务端API接口
|
||
|
||
根据PRD需求，需要以下额外的服务端API接口：
|
||
|
||
### 1. 用户管理API
|
||
```kotlin
|
||
// 获取用户基本信息
|
||
GET /user/{userId}/info
|
||
// 搜索用户
|
||
GET /user/search?keyword={name}&department={dept}
|
||
// 获取组织架构
|
||
GET /organization/structure
|
||
```
|
||
|
||
### 2. 文件服务API
|
||
```kotlin
|
||
// 文件上传
|
||
POST /file/upload
|
||
// 文件下载
|
||
GET /file/{fileId}/download
|
||
// 文件信息获取
|
||
GET /file/{fileId}/info
|
||
// 文件预览生成
|
||
POST /file/{fileId}/generate-preview
|
||
```
|
||
|
||
### 3. 会议管理API
|
||
```kotlin
|
||
// 创建预约会议
|
||
POST /meeting/scheduled
|
||
// 获取会议列表
|
||
GET /meeting/history
|
||
// 获取会议详情
|
||
GET /meeting/{meetingId}
|
||
// 删除会议记录
|
||
DELETE /meeting/{meetingId}
|
||
```
|
||
|
||
### 4. AI服务API
|
||
```kotlin
|
||
// 语音转文字
|
||
POST /ai/speech-to-text
|
||
// 会议纪要生成
|
||
POST /ai/meeting-summary
|
||
// 知识库查询
|
||
POST /ai/knowledge-query
|
||
// 实时字幕服务
|
||
WebSocket /ai/realtime-subtitle
|
||
```
|
||
|
||
### 5. 推送通知API
|
||
```kotlin
|
||
// 发送会议提醒
|
||
POST /notification/meeting-reminder
|
||
// 发送呼叫通知
|
||
POST /notification/call-invitation
|
||
// 获取通知历史
|
||
GET /notification/history
|
||
```
|
||
|
||
### 6. 统计和分析API
|
||
```kotlin
|
||
// 会议时长统计
|
||
GET /analytics/meeting-duration
|
||
// 用户活跃度统计
|
||
GET /analytics/user-activity
|
||
// 功能使用统计
|
||
GET /analytics/feature-usage
|
||
```
|
||
|
||
## 🚀 开发建议
|
||
|
||
1. **按阶段顺序开发**：严格遵循8个阶段，确保基础稳固
|
||
2. **测试驱动开发**：先写测试，特别是MQTT消息处理测试
|
||
3. **持续集成**：每阶段完成后运行全部测试套件
|
||
4. **监控和日志**：集成完整的监控和日志系统
|
||
5. **性能基准**：建立性能基准并持续监控
|
||
|
||
这个增强版计划确保了所有关键集成点的完整性，特别是GXRTC API、MQTT消息系统和AR标注同步机制，使AI能够准确高效地完成开发任务。 |