From f7a0b0934dd9a2f3920a22ba24d3fc0a5e96902a Mon Sep 17 00:00:00 2001 From: Shadow Date: Tue, 27 Jan 2026 14:46:27 -0600 Subject: [PATCH] Branding: update bot.molt bundle IDs + launchd labels --- CHANGELOG.md | 1 + apps/android/app/build.gradle.kts | 4 +- .../java/bot/molt/android/CameraHudState.kt | 14 + .../main/java/bot/molt/android/DeviceNames.kt | 26 + .../java/bot/molt/android/LocationMode.kt | 15 + .../java/bot/molt/android/MainActivity.kt | 130 ++ .../java/bot/molt/android/MainViewModel.kt | 174 +++ .../src/main/java/bot/molt/android/NodeApp.kt | 26 + .../bot/molt/android/NodeForegroundService.kt | 180 +++ .../main/java/bot/molt/android/NodeRuntime.kt | 1268 +++++++++++++++++ .../bot/molt/android/PermissionRequester.kt | 133 ++ .../molt/android/ScreenCaptureRequester.kt | 65 + .../main/java/bot/molt/android/SecurePrefs.kt | 308 ++++ .../main/java/bot/molt/android/SessionKey.kt | 13 + .../java/bot/molt/android/VoiceWakeMode.kt | 14 + .../main/java/bot/molt/android/WakeWords.kt | 21 + .../bot/molt/android/chat/ChatController.kt | 524 +++++++ .../java/bot/molt/android/chat/ChatModels.kt | 44 + .../molt/android/gateway/BonjourEscapes.kt | 35 + .../molt/android/gateway/DeviceAuthStore.kt | 26 + .../android/gateway/DeviceIdentityStore.kt | 146 ++ .../molt/android/gateway/GatewayDiscovery.kt | 519 +++++++ .../molt/android/gateway/GatewayEndpoint.kt | 26 + .../molt/android/gateway/GatewayProtocol.kt | 3 + .../molt/android/gateway/GatewaySession.kt | 683 +++++++++ .../bot/molt/android/gateway/GatewayTls.kt | 90 ++ .../molt/android/node/CameraCaptureManager.kt | 316 ++++ .../bot/molt/android/node/CanvasController.kt | 264 ++++ .../bot/molt/android/node/JpegSizeLimiter.kt | 61 + .../android/node/LocationCaptureManager.kt | 117 ++ .../molt/android/node/ScreenRecordManager.kt | 199 +++ .../java/bot/molt/android/node/SmsManager.kt | 230 +++ .../protocol/ClawdbotCanvasA2UIAction.kt | 66 + .../protocol/ClawdbotProtocolConstants.kt | 71 + .../bot/molt/android/tools/ToolDisplay.kt | 222 +++ .../bot/molt/android/ui/CameraHudOverlay.kt | 44 + .../java/bot/molt/android/ui/ChatSheet.kt | 10 + .../java/bot/molt/android/ui/ClawdbotTheme.kt | 32 + .../java/bot/molt/android/ui/RootScreen.kt | 449 ++++++ .../java/bot/molt/android/ui/SettingsSheet.kt | 686 +++++++++ .../java/bot/molt/android/ui/StatusPill.kt | 114 ++ .../bot/molt/android/ui/TalkOrbOverlay.kt | 134 ++ .../bot/molt/android/ui/chat/ChatComposer.kt | 285 ++++ .../bot/molt/android/ui/chat/ChatMarkdown.kt | 215 +++ .../android/ui/chat/ChatMessageListCard.kt | 111 ++ .../molt/android/ui/chat/ChatMessageViews.kt | 252 ++++ .../android/ui/chat/ChatSessionsDialog.kt | 92 ++ .../molt/android/ui/chat/ChatSheetContent.kt | 147 ++ .../molt/android/ui/chat/SessionFilters.kt | 49 + .../android/voice/StreamingMediaDataSource.kt | 98 ++ .../molt/android/voice/TalkDirectiveParser.kt | 191 +++ .../bot/molt/android/voice/TalkModeManager.kt | 1257 ++++++++++++++++ .../voice/VoiceWakeCommandExtractor.kt | 40 + .../molt/android/voice/VoiceWakeManager.kt | 173 +++ .../molt/android/NodeForegroundServiceTest.kt | 43 + .../java/bot/molt/android/WakeWordsTest.kt | 50 + .../android/gateway/BonjourEscapesTest.kt | 19 + .../CanvasControllerSnapshotParamsTest.kt | 43 + .../molt/android/node/JpegSizeLimiterTest.kt | 47 + .../bot/molt/android/node/SmsManagerTest.kt | 91 ++ .../protocol/ClawdbotCanvasA2UIActionTest.kt | 49 + .../protocol/ClawdbotProtocolConstantsTest.kt | 35 + .../android/ui/chat/SessionFiltersTest.kt | 35 + .../android/voice/TalkDirectiveParserTest.kt | 55 + .../voice/VoiceWakeCommandExtractorTest.kt | 25 + .../Gateway/GatewayDiscoveryModel.swift | 2 +- .../Gateway/GatewaySettingsStore.swift | 74 +- .../Sources/Screen/ScreenRecordService.swift | 2 +- apps/ios/Sources/Voice/TalkModeManager.swift | 2 +- .../ios/Tests/GatewaySettingsStoreTests.swift | 4 +- apps/ios/Tests/KeychainStoreTests.swift | 2 +- apps/ios/fastlane/Appfile | 2 +- apps/ios/project.yml | 8 +- .../Sources/MoltbotChatUI/ChatViewModel.swift | 2 +- .../Sources/MoltbotKit/GatewayChannel.swift | 2 +- .../MoltbotKit/GatewayNodeSession.swift | 2 +- .../MoltbotKit/GatewayTLSPinning.swift | 17 +- .../Sources/MoltbotKit/InstanceIdentity.swift | 15 +- docs/gateway/index.md | 14 +- docs/gateway/remote-gateway-readme.md | 12 +- docs/gateway/troubleshooting.md | 4 +- docs/help/faq.md | 2 +- docs/install/nix.md | 2 +- docs/install/uninstall.md | 8 +- docs/install/updating.md | 2 +- docs/platforms/index.md | 2 +- docs/platforms/mac/bundled-gateway.md | 6 +- docs/platforms/mac/child-process.md | 10 +- docs/platforms/mac/dev-setup.md | 2 +- docs/platforms/mac/logging.md | 8 +- docs/platforms/mac/permissions.md | 4 +- docs/platforms/mac/release.md | 4 +- docs/platforms/mac/signing.md | 2 +- docs/platforms/mac/voice-overlay.md | 4 +- docs/platforms/mac/webchat.md | 2 +- docs/platforms/macos.md | 10 +- package.json | 4 +- scripts/clawlog.sh | 4 +- scripts/package-mac-app.sh | 2 +- scripts/restart-mac.sh | 6 +- src/cli/daemon-cli.coverage.test.ts | 2 +- src/commands/status.test.ts | 4 +- src/daemon/constants.test.ts | 8 +- src/daemon/constants.ts | 19 +- src/daemon/inspect.ts | 7 +- src/daemon/launchd.test.ts | 20 +- src/daemon/launchd.ts | 6 +- src/daemon/service-env.test.ts | 4 +- 108 files changed, 11111 insertions(+), 112 deletions(-) create mode 100644 apps/android/app/src/main/java/bot/molt/android/CameraHudState.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/DeviceNames.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/LocationMode.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/MainActivity.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/NodeApp.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/NodeForegroundService.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/NodeRuntime.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/PermissionRequester.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ScreenCaptureRequester.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/SecurePrefs.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/SessionKey.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/VoiceWakeMode.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/WakeWords.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/chat/ChatModels.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/BonjourEscapes.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/DeviceAuthStore.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/DeviceIdentityStore.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/GatewayDiscovery.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/GatewayEndpoint.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/GatewayProtocol.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/gateway/GatewayTls.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/CameraCaptureManager.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/CanvasController.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/JpegSizeLimiter.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/LocationCaptureManager.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/ScreenRecordManager.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/node/SmsManager.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotCanvasA2UIAction.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotProtocolConstants.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/tools/ToolDisplay.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/CameraHudOverlay.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/ChatSheet.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/ClawdbotTheme.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/StatusPill.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/TalkOrbOverlay.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatComposer.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMarkdown.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageListCard.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageViews.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSessionsDialog.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSheetContent.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/ui/chat/SessionFilters.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/voice/StreamingMediaDataSource.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/voice/TalkDirectiveParser.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeCommandExtractor.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeManager.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/NodeForegroundServiceTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/WakeWordsTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/gateway/BonjourEscapesTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/node/CanvasControllerSnapshotParamsTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/node/JpegSizeLimiterTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/node/SmsManagerTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotCanvasA2UIActionTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotProtocolConstantsTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/ui/chat/SessionFiltersTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/voice/TalkDirectiveParserTest.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/voice/VoiceWakeCommandExtractorTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index cc26b079676..e1efce47511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Status: unreleased. - Commands: group /help and /commands output with Telegram paging. (#2504) Thanks @hougangdev. - macOS: limit project-local `node_modules/.bin` PATH preference to debug builds (reduce PATH hijacking risk). - macOS: finish Moltbot app rename for macOS sources, bundle identifiers, and shared kit paths. (#2844) Thanks @fal3. +- Branding: update launchd labels, mobile bundle IDs, and logging subsystems to bot.molt (legacy com.clawdbot migrations). Thanks @thewilloftheshadow. - Tools: add per-sender group tool policies and fix precedence. (#1757) Thanks @adam91holt. - Agents: summarize dropped messages during compaction safeguard pruning. (#2509) Thanks @jogi47. - Skills: add multi-image input support to Nano Banana Pro skill. (#1958) Thanks @tyler6204. diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts index ef2fb8dd2e8..6d3fa60452b 100644 --- a/apps/android/app/build.gradle.kts +++ b/apps/android/app/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } android { - namespace = "com.clawdbot.android" + namespace = "bot.molt.android" compileSdk = 36 sourceSets { @@ -18,7 +18,7 @@ android { } defaultConfig { - applicationId = "com.clawdbot.android" + applicationId = "bot.molt.android" minSdk = 31 targetSdk = 36 versionCode = 202601260 diff --git a/apps/android/app/src/main/java/bot/molt/android/CameraHudState.kt b/apps/android/app/src/main/java/bot/molt/android/CameraHudState.kt new file mode 100644 index 00000000000..91531bb5e70 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/CameraHudState.kt @@ -0,0 +1,14 @@ +package bot.molt.android + +enum class CameraHudKind { + Photo, + Recording, + Success, + Error, +} + +data class CameraHudState( + val token: Long, + val kind: CameraHudKind, + val message: String, +) diff --git a/apps/android/app/src/main/java/bot/molt/android/DeviceNames.kt b/apps/android/app/src/main/java/bot/molt/android/DeviceNames.kt new file mode 100644 index 00000000000..36cc3e8e273 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/DeviceNames.kt @@ -0,0 +1,26 @@ +package bot.molt.android + +import android.content.Context +import android.os.Build +import android.provider.Settings + +object DeviceNames { + fun bestDefaultNodeName(context: Context): String { + val deviceName = + runCatching { + Settings.Global.getString(context.contentResolver, "device_name") + } + .getOrNull() + ?.trim() + .orEmpty() + + if (deviceName.isNotEmpty()) return deviceName + + val model = + listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() }) + .joinToString(" ") + .trim() + + return model.ifEmpty { "Android Node" } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/LocationMode.kt b/apps/android/app/src/main/java/bot/molt/android/LocationMode.kt new file mode 100644 index 00000000000..40005c324a9 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/LocationMode.kt @@ -0,0 +1,15 @@ +package bot.molt.android + +enum class LocationMode(val rawValue: String) { + Off("off"), + WhileUsing("whileUsing"), + Always("always"), + ; + + companion object { + fun fromRawValue(raw: String?): LocationMode { + val normalized = raw?.trim()?.lowercase() + return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt b/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt new file mode 100644 index 00000000000..1e8707e7254 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt @@ -0,0 +1,130 @@ +package bot.molt.android + +import android.Manifest +import android.content.pm.ApplicationInfo +import android.os.Bundle +import android.os.Build +import android.view.WindowManager +import android.webkit.WebView +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.viewModels +import androidx.compose.material3.Surface +import androidx.compose.ui.Modifier +import androidx.core.content.ContextCompat +import androidx.core.view.WindowCompat +import androidx.core.view.WindowInsetsCompat +import androidx.core.view.WindowInsetsControllerCompat +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.lifecycleScope +import androidx.lifecycle.repeatOnLifecycle +import bot.molt.android.ui.RootScreen +import bot.molt.android.ui.MoltbotTheme +import kotlinx.coroutines.launch + +class MainActivity : ComponentActivity() { + private val viewModel: MainViewModel by viewModels() + private lateinit var permissionRequester: PermissionRequester + private lateinit var screenCaptureRequester: ScreenCaptureRequester + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0 + WebView.setWebContentsDebuggingEnabled(isDebuggable) + applyImmersiveMode() + requestDiscoveryPermissionsIfNeeded() + requestNotificationPermissionIfNeeded() + NodeForegroundService.start(this) + permissionRequester = PermissionRequester(this) + screenCaptureRequester = ScreenCaptureRequester(this) + viewModel.camera.attachLifecycleOwner(this) + viewModel.camera.attachPermissionRequester(permissionRequester) + viewModel.sms.attachPermissionRequester(permissionRequester) + viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester) + viewModel.screenRecorder.attachPermissionRequester(permissionRequester) + + lifecycleScope.launch { + repeatOnLifecycle(Lifecycle.State.STARTED) { + viewModel.preventSleep.collect { enabled -> + if (enabled) { + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } else { + window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + } + } + } + } + + setContent { + MoltbotTheme { + Surface(modifier = Modifier) { + RootScreen(viewModel = viewModel) + } + } + } + } + + override fun onResume() { + super.onResume() + applyImmersiveMode() + } + + override fun onWindowFocusChanged(hasFocus: Boolean) { + super.onWindowFocusChanged(hasFocus) + if (hasFocus) { + applyImmersiveMode() + } + } + + override fun onStart() { + super.onStart() + viewModel.setForeground(true) + } + + override fun onStop() { + viewModel.setForeground(false) + super.onStop() + } + + private fun applyImmersiveMode() { + WindowCompat.setDecorFitsSystemWindows(window, false) + val controller = WindowInsetsControllerCompat(window, window.decorView) + controller.systemBarsBehavior = + WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE + controller.hide(WindowInsetsCompat.Type.systemBars()) + } + + private fun requestDiscoveryPermissionsIfNeeded() { + if (Build.VERSION.SDK_INT >= 33) { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.NEARBY_WIFI_DEVICES, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100) + } + } else { + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101) + } + } + } + + private fun requestNotificationPermissionIfNeeded() { + if (Build.VERSION.SDK_INT < 33) return + val ok = + ContextCompat.checkSelfPermission( + this, + Manifest.permission.POST_NOTIFICATIONS, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (!ok) { + requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 102) + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt b/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt new file mode 100644 index 00000000000..a4f5ee23ec7 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt @@ -0,0 +1,174 @@ +package bot.molt.android + +import android.app.Application +import androidx.lifecycle.AndroidViewModel +import bot.molt.android.gateway.GatewayEndpoint +import bot.molt.android.chat.OutgoingAttachment +import bot.molt.android.node.CameraCaptureManager +import bot.molt.android.node.CanvasController +import bot.molt.android.node.ScreenRecordManager +import bot.molt.android.node.SmsManager +import kotlinx.coroutines.flow.StateFlow + +class MainViewModel(app: Application) : AndroidViewModel(app) { + private val runtime: NodeRuntime = (app as NodeApp).runtime + + val canvas: CanvasController = runtime.canvas + val camera: CameraCaptureManager = runtime.camera + val screenRecorder: ScreenRecordManager = runtime.screenRecorder + val sms: SmsManager = runtime.sms + + val gateways: StateFlow> = runtime.gateways + val discoveryStatusText: StateFlow = runtime.discoveryStatusText + + val isConnected: StateFlow = runtime.isConnected + val statusText: StateFlow = runtime.statusText + val serverName: StateFlow = runtime.serverName + val remoteAddress: StateFlow = runtime.remoteAddress + val isForeground: StateFlow = runtime.isForeground + val seamColorArgb: StateFlow = runtime.seamColorArgb + val mainSessionKey: StateFlow = runtime.mainSessionKey + + val cameraHud: StateFlow = runtime.cameraHud + val cameraFlashToken: StateFlow = runtime.cameraFlashToken + val screenRecordActive: StateFlow = runtime.screenRecordActive + + val instanceId: StateFlow = runtime.instanceId + val displayName: StateFlow = runtime.displayName + val cameraEnabled: StateFlow = runtime.cameraEnabled + val locationMode: StateFlow = runtime.locationMode + val locationPreciseEnabled: StateFlow = runtime.locationPreciseEnabled + val preventSleep: StateFlow = runtime.preventSleep + val wakeWords: StateFlow> = runtime.wakeWords + val voiceWakeMode: StateFlow = runtime.voiceWakeMode + val voiceWakeStatusText: StateFlow = runtime.voiceWakeStatusText + val voiceWakeIsListening: StateFlow = runtime.voiceWakeIsListening + val talkEnabled: StateFlow = runtime.talkEnabled + val talkStatusText: StateFlow = runtime.talkStatusText + val talkIsListening: StateFlow = runtime.talkIsListening + val talkIsSpeaking: StateFlow = runtime.talkIsSpeaking + val manualEnabled: StateFlow = runtime.manualEnabled + val manualHost: StateFlow = runtime.manualHost + val manualPort: StateFlow = runtime.manualPort + val manualTls: StateFlow = runtime.manualTls + val canvasDebugStatusEnabled: StateFlow = runtime.canvasDebugStatusEnabled + + val chatSessionKey: StateFlow = runtime.chatSessionKey + val chatSessionId: StateFlow = runtime.chatSessionId + val chatMessages = runtime.chatMessages + val chatError: StateFlow = runtime.chatError + val chatHealthOk: StateFlow = runtime.chatHealthOk + val chatThinkingLevel: StateFlow = runtime.chatThinkingLevel + val chatStreamingAssistantText: StateFlow = runtime.chatStreamingAssistantText + val chatPendingToolCalls = runtime.chatPendingToolCalls + val chatSessions = runtime.chatSessions + val pendingRunCount: StateFlow = runtime.pendingRunCount + + fun setForeground(value: Boolean) { + runtime.setForeground(value) + } + + fun setDisplayName(value: String) { + runtime.setDisplayName(value) + } + + fun setCameraEnabled(value: Boolean) { + runtime.setCameraEnabled(value) + } + + fun setLocationMode(mode: LocationMode) { + runtime.setLocationMode(mode) + } + + fun setLocationPreciseEnabled(value: Boolean) { + runtime.setLocationPreciseEnabled(value) + } + + fun setPreventSleep(value: Boolean) { + runtime.setPreventSleep(value) + } + + fun setManualEnabled(value: Boolean) { + runtime.setManualEnabled(value) + } + + fun setManualHost(value: String) { + runtime.setManualHost(value) + } + + fun setManualPort(value: Int) { + runtime.setManualPort(value) + } + + fun setManualTls(value: Boolean) { + runtime.setManualTls(value) + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + runtime.setCanvasDebugStatusEnabled(value) + } + + fun setWakeWords(words: List) { + runtime.setWakeWords(words) + } + + fun resetWakeWordsDefaults() { + runtime.resetWakeWordsDefaults() + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + runtime.setVoiceWakeMode(mode) + } + + fun setTalkEnabled(enabled: Boolean) { + runtime.setTalkEnabled(enabled) + } + + fun refreshGatewayConnection() { + runtime.refreshGatewayConnection() + } + + fun connect(endpoint: GatewayEndpoint) { + runtime.connect(endpoint) + } + + fun connectManual() { + runtime.connectManual() + } + + fun disconnect() { + runtime.disconnect() + } + + fun handleCanvasA2UIActionFromWebView(payloadJson: String) { + runtime.handleCanvasA2UIActionFromWebView(payloadJson) + } + + fun loadChat(sessionKey: String) { + runtime.loadChat(sessionKey) + } + + fun refreshChat() { + runtime.refreshChat() + } + + fun refreshChatSessions(limit: Int? = null) { + runtime.refreshChatSessions(limit = limit) + } + + fun setChatThinkingLevel(level: String) { + runtime.setChatThinkingLevel(level) + } + + fun switchChatSession(sessionKey: String) { + runtime.switchChatSession(sessionKey) + } + + fun abortChat() { + runtime.abortChat() + } + + fun sendChat(message: String, thinking: String, attachments: List) { + runtime.sendChat(message = message, thinking = thinking, attachments = attachments) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt b/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt new file mode 100644 index 00000000000..53b2c58f581 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt @@ -0,0 +1,26 @@ +package bot.molt.android + +import android.app.Application +import android.os.StrictMode + +class NodeApp : Application() { + val runtime: NodeRuntime by lazy { NodeRuntime(this) } + + override fun onCreate() { + super.onCreate() + if (BuildConfig.DEBUG) { + StrictMode.setThreadPolicy( + StrictMode.ThreadPolicy.Builder() + .detectAll() + .penaltyLog() + .build(), + ) + StrictMode.setVmPolicy( + StrictMode.VmPolicy.Builder() + .detectAll() + .penaltyLog() + .build(), + ) + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeForegroundService.kt b/apps/android/app/src/main/java/bot/molt/android/NodeForegroundService.kt new file mode 100644 index 00000000000..03cc42f2d8a --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/NodeForegroundService.kt @@ -0,0 +1,180 @@ +package bot.molt.android + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.Service +import android.app.PendingIntent +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ServiceInfo +import androidx.core.app.NotificationCompat +import androidx.core.content.ContextCompat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.launch + +class NodeForegroundService : Service() { + private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) + private var notificationJob: Job? = null + private var lastRequiresMic = false + private var didStartForeground = false + + override fun onCreate() { + super.onCreate() + ensureChannel() + val initial = buildNotification(title = "Moltbot Node", text = "Starting…") + startForegroundWithTypes(notification = initial, requiresMic = false) + + val runtime = (application as NodeApp).runtime + notificationJob = + scope.launch { + combine( + runtime.statusText, + runtime.serverName, + runtime.isConnected, + runtime.voiceWakeMode, + runtime.voiceWakeIsListening, + ) { status, server, connected, voiceMode, voiceListening -> + Quint(status, server, connected, voiceMode, voiceListening) + }.collect { (status, server, connected, voiceMode, voiceListening) -> + val title = if (connected) "Moltbot Node · Connected" else "Moltbot Node" + val voiceSuffix = + if (voiceMode == VoiceWakeMode.Always) { + if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused" + } else { + "" + } + val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix + + val requiresMic = + voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission() + startForegroundWithTypes( + notification = buildNotification(title = title, text = text), + requiresMic = requiresMic, + ) + } + } + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + when (intent?.action) { + ACTION_STOP -> { + (application as NodeApp).runtime.disconnect() + stopSelf() + return START_NOT_STICKY + } + } + // Keep running; connection is managed by NodeRuntime (auto-reconnect + manual). + return START_STICKY + } + + override fun onDestroy() { + notificationJob?.cancel() + scope.cancel() + super.onDestroy() + } + + override fun onBind(intent: Intent?) = null + + private fun ensureChannel() { + val mgr = getSystemService(NotificationManager::class.java) + val channel = + NotificationChannel( + CHANNEL_ID, + "Connection", + NotificationManager.IMPORTANCE_LOW, + ).apply { + description = "Moltbot node connection status" + setShowBadge(false) + } + mgr.createNotificationChannel(channel) + } + + private fun buildNotification(title: String, text: String): Notification { + val launchIntent = Intent(this, MainActivity::class.java).apply { + flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP + } + val launchPending = + PendingIntent.getActivity( + this, + 1, + launchIntent, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, + ) + + val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP) + val stopPending = + PendingIntent.getService( + this, + 2, + stopIntent, + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, + ) + + return NotificationCompat.Builder(this, CHANNEL_ID) + .setSmallIcon(R.mipmap.ic_launcher) + .setContentTitle(title) + .setContentText(text) + .setContentIntent(launchPending) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .addAction(0, "Disconnect", stopPending) + .build() + } + + private fun updateNotification(notification: Notification) { + val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + mgr.notify(NOTIFICATION_ID, notification) + } + + private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) { + if (didStartForeground && requiresMic == lastRequiresMic) { + updateNotification(notification) + return + } + + lastRequiresMic = requiresMic + val types = + if (requiresMic) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } else { + ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC + } + startForeground(NOTIFICATION_ID, notification, types) + didStartForeground = true + } + + private fun hasRecordAudioPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + ) + } + + companion object { + private const val CHANNEL_ID = "connection" + private const val NOTIFICATION_ID = 1 + + private const val ACTION_STOP = "bot.molt.android.action.STOP" + + fun start(context: Context) { + val intent = Intent(context, NodeForegroundService::class.java) + context.startForegroundService(intent) + } + + fun stop(context: Context) { + val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP) + context.startService(intent) + } + } +} + +private data class Quint(val first: A, val second: B, val third: C, val fourth: D, val fifth: E) diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeRuntime.kt b/apps/android/app/src/main/java/bot/molt/android/NodeRuntime.kt new file mode 100644 index 00000000000..5fd429e9e01 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/NodeRuntime.kt @@ -0,0 +1,1268 @@ +package bot.molt.android + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.LocationManager +import android.os.Build +import android.os.SystemClock +import androidx.core.content.ContextCompat +import bot.molt.android.chat.ChatController +import bot.molt.android.chat.ChatMessage +import bot.molt.android.chat.ChatPendingToolCall +import bot.molt.android.chat.ChatSessionEntry +import bot.molt.android.chat.OutgoingAttachment +import bot.molt.android.gateway.DeviceAuthStore +import bot.molt.android.gateway.DeviceIdentityStore +import bot.molt.android.gateway.GatewayClientInfo +import bot.molt.android.gateway.GatewayConnectOptions +import bot.molt.android.gateway.GatewayDiscovery +import bot.molt.android.gateway.GatewayEndpoint +import bot.molt.android.gateway.GatewaySession +import bot.molt.android.gateway.GatewayTlsParams +import bot.molt.android.node.CameraCaptureManager +import bot.molt.android.node.LocationCaptureManager +import bot.molt.android.BuildConfig +import bot.molt.android.node.CanvasController +import bot.molt.android.node.ScreenRecordManager +import bot.molt.android.node.SmsManager +import bot.molt.android.protocol.MoltbotCapability +import bot.molt.android.protocol.MoltbotCameraCommand +import bot.molt.android.protocol.MoltbotCanvasA2UIAction +import bot.molt.android.protocol.MoltbotCanvasA2UICommand +import bot.molt.android.protocol.MoltbotCanvasCommand +import bot.molt.android.protocol.MoltbotScreenCommand +import bot.molt.android.protocol.MoltbotLocationCommand +import bot.molt.android.protocol.MoltbotSmsCommand +import bot.molt.android.voice.TalkModeManager +import bot.molt.android.voice.VoiceWakeManager +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import java.util.concurrent.atomic.AtomicLong + +class NodeRuntime(context: Context) { + private val appContext = context.applicationContext + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + + val prefs = SecurePrefs(appContext) + private val deviceAuthStore = DeviceAuthStore(prefs) + val canvas = CanvasController() + val camera = CameraCaptureManager(appContext) + val location = LocationCaptureManager(appContext) + val screenRecorder = ScreenRecordManager(appContext) + val sms = SmsManager(appContext) + private val json = Json { ignoreUnknownKeys = true } + + private val externalAudioCaptureActive = MutableStateFlow(false) + + private val voiceWake: VoiceWakeManager by lazy { + VoiceWakeManager( + context = appContext, + scope = scope, + onCommand = { command -> + nodeSession.sendNodeEvent( + event = "agent.request", + payloadJson = + buildJsonObject { + put("message", JsonPrimitive(command)) + put("sessionKey", JsonPrimitive(resolveMainSessionKey())) + put("thinking", JsonPrimitive(chatThinkingLevel.value)) + put("deliver", JsonPrimitive(false)) + }.toString(), + ) + }, + ) + } + + val voiceWakeIsListening: StateFlow + get() = voiceWake.isListening + + val voiceWakeStatusText: StateFlow + get() = voiceWake.statusText + + val talkStatusText: StateFlow + get() = talkMode.statusText + + val talkIsListening: StateFlow + get() = talkMode.isListening + + val talkIsSpeaking: StateFlow + get() = talkMode.isSpeaking + + private val discovery = GatewayDiscovery(appContext, scope = scope) + val gateways: StateFlow> = discovery.gateways + val discoveryStatusText: StateFlow = discovery.statusText + + private val identityStore = DeviceIdentityStore(appContext) + + private val _isConnected = MutableStateFlow(false) + val isConnected: StateFlow = _isConnected.asStateFlow() + + private val _statusText = MutableStateFlow("Offline") + val statusText: StateFlow = _statusText.asStateFlow() + + private val _mainSessionKey = MutableStateFlow("main") + val mainSessionKey: StateFlow = _mainSessionKey.asStateFlow() + + private val cameraHudSeq = AtomicLong(0) + private val _cameraHud = MutableStateFlow(null) + val cameraHud: StateFlow = _cameraHud.asStateFlow() + + private val _cameraFlashToken = MutableStateFlow(0L) + val cameraFlashToken: StateFlow = _cameraFlashToken.asStateFlow() + + private val _screenRecordActive = MutableStateFlow(false) + val screenRecordActive: StateFlow = _screenRecordActive.asStateFlow() + + private val _serverName = MutableStateFlow(null) + val serverName: StateFlow = _serverName.asStateFlow() + + private val _remoteAddress = MutableStateFlow(null) + val remoteAddress: StateFlow = _remoteAddress.asStateFlow() + + private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB) + val seamColorArgb: StateFlow = _seamColorArgb.asStateFlow() + + private val _isForeground = MutableStateFlow(true) + val isForeground: StateFlow = _isForeground.asStateFlow() + + private var lastAutoA2uiUrl: String? = null + private var operatorConnected = false + private var nodeConnected = false + private var operatorStatusText: String = "Offline" + private var nodeStatusText: String = "Offline" + private var connectedEndpoint: GatewayEndpoint? = null + + private val operatorSession = + GatewaySession( + scope = scope, + identityStore = identityStore, + deviceAuthStore = deviceAuthStore, + onConnected = { name, remote, mainSessionKey -> + operatorConnected = true + operatorStatusText = "Connected" + _serverName.value = name + _remoteAddress.value = remote + _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB + applyMainSessionKey(mainSessionKey) + updateStatus() + scope.launch { refreshBrandingFromGateway() } + scope.launch { refreshWakeWordsFromGateway() } + }, + onDisconnected = { message -> + operatorConnected = false + operatorStatusText = message + _serverName.value = null + _remoteAddress.value = null + _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB + if (!isCanonicalMainSessionKey(_mainSessionKey.value)) { + _mainSessionKey.value = "main" + } + val mainKey = resolveMainSessionKey() + talkMode.setMainSessionKey(mainKey) + chat.applyMainSessionKey(mainKey) + chat.onDisconnected(message) + updateStatus() + }, + onEvent = { event, payloadJson -> + handleGatewayEvent(event, payloadJson) + }, + ) + + private val nodeSession = + GatewaySession( + scope = scope, + identityStore = identityStore, + deviceAuthStore = deviceAuthStore, + onConnected = { _, _, _ -> + nodeConnected = true + nodeStatusText = "Connected" + updateStatus() + maybeNavigateToA2uiOnConnect() + }, + onDisconnected = { message -> + nodeConnected = false + nodeStatusText = message + updateStatus() + showLocalCanvasOnDisconnect() + }, + onEvent = { _, _ -> }, + onInvoke = { req -> + handleInvoke(req.command, req.paramsJson) + }, + onTlsFingerprint = { stableId, fingerprint -> + prefs.saveGatewayTlsFingerprint(stableId, fingerprint) + }, + ) + + private val chat: ChatController = + ChatController( + scope = scope, + session = operatorSession, + json = json, + supportsChatSubscribe = false, + ) + private val talkMode: TalkModeManager by lazy { + TalkModeManager( + context = appContext, + scope = scope, + session = operatorSession, + supportsChatSubscribe = false, + isConnected = { operatorConnected }, + ) + } + + private fun applyMainSessionKey(candidate: String?) { + val trimmed = candidate?.trim().orEmpty() + if (trimmed.isEmpty()) return + if (isCanonicalMainSessionKey(_mainSessionKey.value)) return + if (_mainSessionKey.value == trimmed) return + _mainSessionKey.value = trimmed + talkMode.setMainSessionKey(trimmed) + chat.applyMainSessionKey(trimmed) + } + + private fun updateStatus() { + _isConnected.value = operatorConnected + _statusText.value = + when { + operatorConnected && nodeConnected -> "Connected" + operatorConnected && !nodeConnected -> "Connected (node offline)" + !operatorConnected && nodeConnected -> "Connected (operator offline)" + operatorStatusText.isNotBlank() && operatorStatusText != "Offline" -> operatorStatusText + else -> nodeStatusText + } + } + + private fun resolveMainSessionKey(): String { + val trimmed = _mainSessionKey.value.trim() + return if (trimmed.isEmpty()) "main" else trimmed + } + + private fun maybeNavigateToA2uiOnConnect() { + val a2uiUrl = resolveA2uiHostUrl() ?: return + val current = canvas.currentUrl()?.trim().orEmpty() + if (current.isEmpty() || current == lastAutoA2uiUrl) { + lastAutoA2uiUrl = a2uiUrl + canvas.navigate(a2uiUrl) + } + } + + private fun showLocalCanvasOnDisconnect() { + lastAutoA2uiUrl = null + canvas.navigate("") + } + + val instanceId: StateFlow = prefs.instanceId + val displayName: StateFlow = prefs.displayName + val cameraEnabled: StateFlow = prefs.cameraEnabled + val locationMode: StateFlow = prefs.locationMode + val locationPreciseEnabled: StateFlow = prefs.locationPreciseEnabled + val preventSleep: StateFlow = prefs.preventSleep + val wakeWords: StateFlow> = prefs.wakeWords + val voiceWakeMode: StateFlow = prefs.voiceWakeMode + val talkEnabled: StateFlow = prefs.talkEnabled + val manualEnabled: StateFlow = prefs.manualEnabled + val manualHost: StateFlow = prefs.manualHost + val manualPort: StateFlow = prefs.manualPort + val manualTls: StateFlow = prefs.manualTls + val lastDiscoveredStableId: StateFlow = prefs.lastDiscoveredStableId + val canvasDebugStatusEnabled: StateFlow = prefs.canvasDebugStatusEnabled + + private var didAutoConnect = false + private var suppressWakeWordsSync = false + private var wakeWordsSyncJob: Job? = null + + val chatSessionKey: StateFlow = chat.sessionKey + val chatSessionId: StateFlow = chat.sessionId + val chatMessages: StateFlow> = chat.messages + val chatError: StateFlow = chat.errorText + val chatHealthOk: StateFlow = chat.healthOk + val chatThinkingLevel: StateFlow = chat.thinkingLevel + val chatStreamingAssistantText: StateFlow = chat.streamingAssistantText + val chatPendingToolCalls: StateFlow> = chat.pendingToolCalls + val chatSessions: StateFlow> = chat.sessions + val pendingRunCount: StateFlow = chat.pendingRunCount + + init { + scope.launch { + combine( + voiceWakeMode, + isForeground, + externalAudioCaptureActive, + wakeWords, + ) { mode, foreground, externalAudio, words -> + Quad(mode, foreground, externalAudio, words) + }.distinctUntilChanged() + .collect { (mode, foreground, externalAudio, words) -> + voiceWake.setTriggerWords(words) + + val shouldListen = + when (mode) { + VoiceWakeMode.Off -> false + VoiceWakeMode.Foreground -> foreground + VoiceWakeMode.Always -> true + } && !externalAudio + + if (!shouldListen) { + voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused") + return@collect + } + + if (!hasRecordAudioPermission()) { + voiceWake.stop(statusText = "Microphone permission required") + return@collect + } + + voiceWake.start() + } + } + + scope.launch { + talkEnabled.collect { enabled -> + talkMode.setEnabled(enabled) + externalAudioCaptureActive.value = enabled + } + } + + scope.launch(Dispatchers.Default) { + gateways.collect { list -> + if (list.isNotEmpty()) { + // Persist the last discovered gateway (best-effort UX parity with iOS). + prefs.setLastDiscoveredStableId(list.last().stableId) + } + + if (didAutoConnect) return@collect + if (_isConnected.value) return@collect + + if (manualEnabled.value) { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isNotEmpty() && port in 1..65535) { + didAutoConnect = true + connect(GatewayEndpoint.manual(host = host, port = port)) + } + return@collect + } + + val targetStableId = lastDiscoveredStableId.value.trim() + if (targetStableId.isEmpty()) return@collect + val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect + didAutoConnect = true + connect(target) + } + } + + scope.launch { + combine( + canvasDebugStatusEnabled, + statusText, + serverName, + remoteAddress, + ) { debugEnabled, status, server, remote -> + Quad(debugEnabled, status, server, remote) + }.distinctUntilChanged() + .collect { (debugEnabled, status, server, remote) -> + canvas.setDebugStatusEnabled(debugEnabled) + if (!debugEnabled) return@collect + canvas.setDebugStatus(status, server ?: remote) + } + } + } + + fun setForeground(value: Boolean) { + _isForeground.value = value + } + + fun setDisplayName(value: String) { + prefs.setDisplayName(value) + } + + fun setCameraEnabled(value: Boolean) { + prefs.setCameraEnabled(value) + } + + fun setLocationMode(mode: LocationMode) { + prefs.setLocationMode(mode) + } + + fun setLocationPreciseEnabled(value: Boolean) { + prefs.setLocationPreciseEnabled(value) + } + + fun setPreventSleep(value: Boolean) { + prefs.setPreventSleep(value) + } + + fun setManualEnabled(value: Boolean) { + prefs.setManualEnabled(value) + } + + fun setManualHost(value: String) { + prefs.setManualHost(value) + } + + fun setManualPort(value: Int) { + prefs.setManualPort(value) + } + + fun setManualTls(value: Boolean) { + prefs.setManualTls(value) + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + prefs.setCanvasDebugStatusEnabled(value) + } + + fun setWakeWords(words: List) { + prefs.setWakeWords(words) + scheduleWakeWordsSyncIfNeeded() + } + + fun resetWakeWordsDefaults() { + setWakeWords(SecurePrefs.defaultWakeWords) + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + prefs.setVoiceWakeMode(mode) + } + + fun setTalkEnabled(value: Boolean) { + prefs.setTalkEnabled(value) + } + + private fun buildInvokeCommands(): List = + buildList { + add(MoltbotCanvasCommand.Present.rawValue) + add(MoltbotCanvasCommand.Hide.rawValue) + add(MoltbotCanvasCommand.Navigate.rawValue) + add(MoltbotCanvasCommand.Eval.rawValue) + add(MoltbotCanvasCommand.Snapshot.rawValue) + add(MoltbotCanvasA2UICommand.Push.rawValue) + add(MoltbotCanvasA2UICommand.PushJSONL.rawValue) + add(MoltbotCanvasA2UICommand.Reset.rawValue) + add(MoltbotScreenCommand.Record.rawValue) + if (cameraEnabled.value) { + add(MoltbotCameraCommand.Snap.rawValue) + add(MoltbotCameraCommand.Clip.rawValue) + } + if (locationMode.value != LocationMode.Off) { + add(MoltbotLocationCommand.Get.rawValue) + } + if (sms.canSendSms()) { + add(MoltbotSmsCommand.Send.rawValue) + } + } + + private fun buildCapabilities(): List = + buildList { + add(MoltbotCapability.Canvas.rawValue) + add(MoltbotCapability.Screen.rawValue) + if (cameraEnabled.value) add(MoltbotCapability.Camera.rawValue) + if (sms.canSendSms()) add(MoltbotCapability.Sms.rawValue) + if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) { + add(MoltbotCapability.VoiceWake.rawValue) + } + if (locationMode.value != LocationMode.Off) { + add(MoltbotCapability.Location.rawValue) + } + } + + private fun resolvedVersionName(): String { + val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } + return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { + "$versionName-dev" + } else { + versionName + } + } + + private fun resolveModelIdentifier(): String? { + return listOfNotNull(Build.MANUFACTURER, Build.MODEL) + .joinToString(" ") + .trim() + .ifEmpty { null } + } + + private fun buildUserAgent(): String { + val version = resolvedVersionName() + val release = Build.VERSION.RELEASE?.trim().orEmpty() + val releaseLabel = if (release.isEmpty()) "unknown" else release + return "MoltbotAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})" + } + + private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo { + return GatewayClientInfo( + id = clientId, + displayName = displayName.value, + version = resolvedVersionName(), + platform = "android", + mode = clientMode, + instanceId = instanceId.value, + deviceFamily = "Android", + modelIdentifier = resolveModelIdentifier(), + ) + } + + private fun buildNodeConnectOptions(): GatewayConnectOptions { + return GatewayConnectOptions( + role = "node", + scopes = emptyList(), + caps = buildCapabilities(), + commands = buildInvokeCommands(), + permissions = emptyMap(), + client = buildClientInfo(clientId = "moltbot-android", clientMode = "node"), + userAgent = buildUserAgent(), + ) + } + + private fun buildOperatorConnectOptions(): GatewayConnectOptions { + return GatewayConnectOptions( + role = "operator", + scopes = emptyList(), + caps = emptyList(), + commands = emptyList(), + permissions = emptyMap(), + client = buildClientInfo(clientId = "moltbot-control-ui", clientMode = "ui"), + userAgent = buildUserAgent(), + ) + } + + fun refreshGatewayConnection() { + val endpoint = connectedEndpoint ?: return + val token = prefs.loadGatewayToken() + val password = prefs.loadGatewayPassword() + val tls = resolveTlsParams(endpoint) + operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) + nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) + operatorSession.reconnect() + nodeSession.reconnect() + } + + fun connect(endpoint: GatewayEndpoint) { + connectedEndpoint = endpoint + operatorStatusText = "Connecting…" + nodeStatusText = "Connecting…" + updateStatus() + val token = prefs.loadGatewayToken() + val password = prefs.loadGatewayPassword() + val tls = resolveTlsParams(endpoint) + operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) + nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) + } + + private fun hasRecordAudioPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasFineLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasCoarseLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + private fun hasBackgroundLocationPermission(): Boolean { + return ( + ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + ) + } + + fun connectManual() { + val host = manualHost.value.trim() + val port = manualPort.value + if (host.isEmpty() || port <= 0 || port > 65535) { + _statusText.value = "Failed: invalid manual host/port" + return + } + connect(GatewayEndpoint.manual(host = host, port = port)) + } + + fun disconnect() { + connectedEndpoint = null + operatorSession.disconnect() + nodeSession.disconnect() + } + + private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? { + val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId) + val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank() + val manual = endpoint.stableId.startsWith("manual|") + + if (manual) { + if (!manualTls.value) return null + return GatewayTlsParams( + required = true, + expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, + allowTOFU = stored == null, + stableId = endpoint.stableId, + ) + } + + if (hinted) { + return GatewayTlsParams( + required = true, + expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, + allowTOFU = stored == null, + stableId = endpoint.stableId, + ) + } + + if (!stored.isNullOrBlank()) { + return GatewayTlsParams( + required = true, + expectedFingerprint = stored, + allowTOFU = false, + stableId = endpoint.stableId, + ) + } + + return null + } + + fun handleCanvasA2UIActionFromWebView(payloadJson: String) { + scope.launch { + val trimmed = payloadJson.trim() + if (trimmed.isEmpty()) return@launch + + val root = + try { + json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch + } catch (_: Throwable) { + return@launch + } + + val userActionObj = (root["userAction"] as? JsonObject) ?: root + val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { + java.util.UUID.randomUUID().toString() + } + val name = MoltbotCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch + + val surfaceId = + (userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" } + val sourceComponentId = + (userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" } + val contextJson = (userActionObj["context"] as? JsonObject)?.toString() + + val sessionKey = resolveMainSessionKey() + val message = + MoltbotCanvasA2UIAction.formatAgentMessage( + actionName = name, + sessionKey = sessionKey, + surfaceId = surfaceId, + sourceComponentId = sourceComponentId, + host = displayName.value, + instanceId = instanceId.value.lowercase(), + contextJson = contextJson, + ) + + val connected = nodeConnected + var error: String? = null + if (connected) { + try { + nodeSession.sendNodeEvent( + event = "agent.request", + payloadJson = + buildJsonObject { + put("message", JsonPrimitive(message)) + put("sessionKey", JsonPrimitive(sessionKey)) + put("thinking", JsonPrimitive("low")) + put("deliver", JsonPrimitive(false)) + put("key", JsonPrimitive(actionId)) + }.toString(), + ) + } catch (e: Throwable) { + error = e.message ?: "send failed" + } + } else { + error = "gateway not connected" + } + + try { + canvas.eval( + MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus( + actionId = actionId, + ok = connected && error == null, + error = error, + ), + ) + } catch (_: Throwable) { + // ignore + } + } + } + + fun loadChat(sessionKey: String) { + val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() } + chat.load(key) + } + + fun refreshChat() { + chat.refresh() + } + + fun refreshChatSessions(limit: Int? = null) { + chat.refreshSessions(limit = limit) + } + + fun setChatThinkingLevel(level: String) { + chat.setThinkingLevel(level) + } + + fun switchChatSession(sessionKey: String) { + chat.switchSession(sessionKey) + } + + fun abortChat() { + chat.abort() + } + + fun sendChat(message: String, thinking: String, attachments: List) { + chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments) + } + + private fun handleGatewayEvent(event: String, payloadJson: String?) { + if (event == "voicewake.changed") { + if (payloadJson.isNullOrBlank()) return + try { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (_: Throwable) { + // ignore + } + return + } + + talkMode.handleGatewayEvent(event, payloadJson) + chat.handleGatewayEvent(event, payloadJson) + } + + private fun applyWakeWordsFromGateway(words: List) { + suppressWakeWordsSync = true + prefs.setWakeWords(words) + suppressWakeWordsSync = false + } + + private fun scheduleWakeWordsSyncIfNeeded() { + if (suppressWakeWordsSync) return + if (!_isConnected.value) return + + val snapshot = prefs.wakeWords.value + wakeWordsSyncJob?.cancel() + wakeWordsSyncJob = + scope.launch { + delay(650) + val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() } + val params = """{"triggers":[$jsonList]}""" + try { + operatorSession.request("voicewake.set", params) + } catch (_: Throwable) { + // ignore + } + } + } + + private suspend fun refreshWakeWordsFromGateway() { + if (!_isConnected.value) return + try { + val res = operatorSession.request("voicewake.get", "{}") + val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (_: Throwable) { + // ignore + } + } + + private suspend fun refreshBrandingFromGateway() { + if (!_isConnected.value) return + try { + val res = operatorSession.request("config.get", "{}") + val root = json.parseToJsonElement(res).asObjectOrNull() + val config = root?.get("config").asObjectOrNull() + val ui = config?.get("ui").asObjectOrNull() + val raw = ui?.get("seamColor").asStringOrNull()?.trim() + val sessionCfg = config?.get("session").asObjectOrNull() + val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) + applyMainSessionKey(mainKey) + + val parsed = parseHexColorArgb(raw) + _seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB + } catch (_: Throwable) { + // ignore + } + } + + private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult { + if ( + command.startsWith(MoltbotCanvasCommand.NamespacePrefix) || + command.startsWith(MoltbotCanvasA2UICommand.NamespacePrefix) || + command.startsWith(MoltbotCameraCommand.NamespacePrefix) || + command.startsWith(MoltbotScreenCommand.NamespacePrefix) + ) { + if (!isForeground.value) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground", + ) + } + } + if (command.startsWith(MoltbotCameraCommand.NamespacePrefix) && !cameraEnabled.value) { + return GatewaySession.InvokeResult.error( + code = "CAMERA_DISABLED", + message = "CAMERA_DISABLED: enable Camera in Settings", + ) + } + if (command.startsWith(MoltbotLocationCommand.NamespacePrefix) && + locationMode.value == LocationMode.Off + ) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_DISABLED", + message = "LOCATION_DISABLED: enable Location in Settings", + ) + } + + return when (command) { + MoltbotCanvasCommand.Present.rawValue -> { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } + MoltbotCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null) + MoltbotCanvasCommand.Navigate.rawValue -> { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + GatewaySession.InvokeResult.ok(null) + } + MoltbotCanvasCommand.Eval.rawValue -> { + val js = + CanvasController.parseEvalJs(paramsJson) + ?: return GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: javaScript required", + ) + val result = + try { + canvas.eval(js) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""") + } + MoltbotCanvasCommand.Snapshot.rawValue -> { + val snapshotParams = CanvasController.parseSnapshotParams(paramsJson) + val base64 = + try { + canvas.snapshotBase64( + format = snapshotParams.format, + quality = snapshotParams.quality, + maxWidth = snapshotParams.maxWidth, + ) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""") + } + MoltbotCanvasA2UICommand.Reset.rawValue -> { + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val res = canvas.eval(a2uiResetJS) + GatewaySession.InvokeResult.ok(res) + } + MoltbotCanvasA2UICommand.Push.rawValue, MoltbotCanvasA2UICommand.PushJSONL.rawValue -> { + val messages = + try { + decodeA2uiMessages(command, paramsJson) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload") + } + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val js = a2uiApplyMessagesJS(messages) + val res = canvas.eval(js) + GatewaySession.InvokeResult.ok(res) + } + MoltbotCameraCommand.Snap.rawValue -> { + showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo) + triggerCameraFlash() + val res = + try { + camera.snap(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600) + GatewaySession.InvokeResult.ok(res.payloadJson) + } + MoltbotCameraCommand.Clip.rawValue -> { + val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false + if (includeAudio) externalAudioCaptureActive.value = true + try { + showCameraHud(message = "Recording…", kind = CameraHudKind.Recording) + val res = + try { + camera.clip(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800) + GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + if (includeAudio) externalAudioCaptureActive.value = false + } + } + MoltbotLocationCommand.Get.rawValue -> { + val mode = locationMode.value + if (!isForeground.value && mode != LocationMode.Always) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_BACKGROUND_UNAVAILABLE", + message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always", + ) + } + if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: grant Location permission", + ) + } + if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings", + ) + } + val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson) + val preciseEnabled = locationPreciseEnabled.value + val accuracy = + when (desiredAccuracy) { + "precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + "coarse" -> "coarse" + else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + } + val providers = + when (accuracy) { + "precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER) + "coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + } + try { + val payload = + location.getLocation( + desiredProviders = providers, + maxAgeMs = maxAgeMs, + timeoutMs = timeoutMs, + isPrecise = accuracy == "precise", + ) + GatewaySession.InvokeResult.ok(payload.payloadJson) + } catch (err: TimeoutCancellationException) { + GatewaySession.InvokeResult.error( + code = "LOCATION_TIMEOUT", + message = "LOCATION_TIMEOUT: no fix in time", + ) + } catch (err: Throwable) { + val message = err.message ?: "LOCATION_UNAVAILABLE: no fix" + GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message) + } + } + MoltbotScreenCommand.Record.rawValue -> { + // Status pill mirrors screen recording state so it stays visible without overlay stacking. + _screenRecordActive.value = true + try { + val res = + try { + screenRecorder.record(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + _screenRecordActive.value = false + } + } + MoltbotSmsCommand.Send.rawValue -> { + val res = sms.send(paramsJson) + if (res.ok) { + GatewaySession.InvokeResult.ok(res.payloadJson) + } else { + val error = res.error ?: "SMS_SEND_FAILED" + val idx = error.indexOf(':') + val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED" + GatewaySession.InvokeResult.error(code = code, message = error) + } + } + else -> + GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: unknown command", + ) + } + } + + private fun triggerCameraFlash() { + // Token is used as a pulse trigger; value doesn't matter as long as it changes. + _cameraFlashToken.value = SystemClock.elapsedRealtimeNanos() + } + + private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) { + val token = cameraHudSeq.incrementAndGet() + _cameraHud.value = CameraHudState(token = token, kind = kind, message = message) + + if (autoHideMs != null && autoHideMs > 0) { + scope.launch { + delay(autoHideMs) + if (_cameraHud.value?.token == token) _cameraHud.value = null + } + } + } + + private fun invokeErrorFromThrowable(err: Throwable): Pair { + val raw = (err.message ?: "").trim() + if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error" + + val idx = raw.indexOf(':') + if (idx <= 0) return "UNAVAILABLE" to raw + val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" } + val message = raw.substring(idx + 1).trim().ifEmpty { raw } + // Preserve full string for callers/logging, but keep the returned message human-friendly. + return code to "$code: $message" + } + + private fun parseLocationParams(paramsJson: String?): Triple { + if (paramsJson.isNullOrBlank()) { + return Triple(null, 10_000L, null) + } + val root = + try { + json.parseToJsonElement(paramsJson).asObjectOrNull() + } catch (_: Throwable) { + null + } + val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull() + val timeoutMs = + (root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L) + ?: 10_000L + val desiredAccuracy = + (root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase() + return Triple(maxAgeMs, timeoutMs, desiredAccuracy) + } + + private fun resolveA2uiHostUrl(): String? { + val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty() + val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty() + val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw + if (raw.isBlank()) return null + val base = raw.trimEnd('/') + return "${base}/__moltbot__/a2ui/?platform=android" + } + + private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean { + try { + val already = canvas.eval(a2uiReadyCheckJS) + if (already == "true") return true + } catch (_: Throwable) { + // ignore + } + + canvas.navigate(a2uiUrl) + repeat(50) { + try { + val ready = canvas.eval(a2uiReadyCheckJS) + if (ready == "true") return true + } catch (_: Throwable) { + // ignore + } + delay(120) + } + return false + } + + private fun decodeA2uiMessages(command: String, paramsJson: String?): String { + val raw = paramsJson?.trim().orEmpty() + if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required") + + val obj = + json.parseToJsonElement(raw) as? JsonObject + ?: throw IllegalArgumentException("INVALID_REQUEST: expected object params") + + val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty() + val hasMessagesArray = obj["messages"] is JsonArray + + if (command == MoltbotCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) { + val jsonl = jsonlField + if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required") + val messages = + jsonl + .lineSequence() + .map { it.trim() } + .filter { it.isNotBlank() } + .mapIndexed { idx, line -> + val el = json.parseToJsonElement(line) + val msg = + el as? JsonObject + ?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + .toList() + return JsonArray(messages).toString() + } + + val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required") + val out = + arr.mapIndexed { idx, el -> + val msg = + el as? JsonObject + ?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + return JsonArray(out).toString() + } + + private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) { + if (msg.containsKey("createSurface")) { + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.", + ) + } + val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface") + val matched = msg.keys.filter { allowed.contains(it) } + if (matched.size != 1) { + val found = msg.keys.sorted().joinToString(", ") + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found", + ) + } + } +} + +private data class Quad(val first: A, val second: B, val third: C, val fourth: D) + +private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A + +private const val a2uiReadyCheckJS: String = + """ + (() => { + try { + return !!globalThis.clawdbotA2UI && typeof globalThis.clawdbotA2UI.applyMessages === 'function'; + } catch (_) { + return false; + } + })() + """ + +private const val a2uiResetJS: String = + """ + (() => { + try { + if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing moltbotA2UI" }; + return globalThis.clawdbotA2UI.reset(); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } + })() + """ + +private fun a2uiApplyMessagesJS(messagesJson: String): String { + return """ + (() => { + try { + if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing moltbotA2UI" }; + const messages = $messagesJson; + return globalThis.clawdbotA2UI.applyMessages(messages); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } + })() + """.trimIndent() +} + +private fun String.toJsonString(): String { + val escaped = + this.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + return "\"$escaped\"" +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun parseHexColorArgb(raw: String?): Long? { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed + if (hex.length != 6) return null + val rgb = hex.toLongOrNull(16) ?: return null + return 0xFF000000L or rgb +} diff --git a/apps/android/app/src/main/java/bot/molt/android/PermissionRequester.kt b/apps/android/app/src/main/java/bot/molt/android/PermissionRequester.kt new file mode 100644 index 00000000000..78ae0b62b16 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/PermissionRequester.kt @@ -0,0 +1,133 @@ +package bot.molt.android + +import android.content.pm.PackageManager +import android.content.Intent +import android.Manifest +import android.net.Uri +import android.provider.Settings +import androidx.appcompat.app.AlertDialog +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.core.content.ContextCompat +import androidx.core.app.ActivityCompat +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class PermissionRequester(private val activity: ComponentActivity) { + private val mutex = Mutex() + private var pending: CompletableDeferred>? = null + + private val launcher: ActivityResultLauncher> = + activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> + val p = pending + pending = null + p?.complete(result) + } + + suspend fun requestIfMissing( + permissions: List, + timeoutMs: Long = 20_000, + ): Map = + mutex.withLock { + val missing = + permissions.filter { perm -> + ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED + } + if (missing.isEmpty()) { + return permissions.associateWith { true } + } + + val needsRationale = + missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) } + if (needsRationale) { + val proceed = showRationaleDialog(missing) + if (!proceed) { + return permissions.associateWith { perm -> + ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED + } + } + } + + val deferred = CompletableDeferred>() + pending = deferred + withContext(Dispatchers.Main) { + launcher.launch(missing.toTypedArray()) + } + + val result = + withContext(Dispatchers.Default) { + kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() } + } + + // Merge: if something was already granted, treat it as granted even if launcher omitted it. + val merged = + permissions.associateWith { perm -> + val nowGranted = + ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED + result[perm] == true || nowGranted + } + + val denied = + merged.filterValues { !it }.keys.filter { + !ActivityCompat.shouldShowRequestPermissionRationale(activity, it) + } + if (denied.isNotEmpty()) { + showSettingsDialog(denied) + } + + return merged + } + + private suspend fun showRationaleDialog(permissions: List): Boolean = + withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + AlertDialog.Builder(activity) + .setTitle("Permission required") + .setMessage(buildRationaleMessage(permissions)) + .setPositiveButton("Continue") { _, _ -> cont.resume(true) } + .setNegativeButton("Not now") { _, _ -> cont.resume(false) } + .setOnCancelListener { cont.resume(false) } + .show() + } + } + + private fun showSettingsDialog(permissions: List) { + AlertDialog.Builder(activity) + .setTitle("Enable permission in Settings") + .setMessage(buildSettingsMessage(permissions)) + .setPositiveButton("Open Settings") { _, _ -> + val intent = + Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", activity.packageName, null), + ) + activity.startActivity(intent) + } + .setNegativeButton("Cancel", null) + .show() + } + + private fun buildRationaleMessage(permissions: List): String { + val labels = permissions.map { permissionLabel(it) } + return "Moltbot needs ${labels.joinToString(", ")} permissions to continue." + } + + private fun buildSettingsMessage(permissions: List): String { + val labels = permissions.map { permissionLabel(it) } + return "Please enable ${labels.joinToString(", ")} in Android Settings to continue." + } + + private fun permissionLabel(permission: String): String = + when (permission) { + Manifest.permission.CAMERA -> "Camera" + Manifest.permission.RECORD_AUDIO -> "Microphone" + Manifest.permission.SEND_SMS -> "SMS" + else -> permission + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ScreenCaptureRequester.kt b/apps/android/app/src/main/java/bot/molt/android/ScreenCaptureRequester.kt new file mode 100644 index 00000000000..29d66204418 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ScreenCaptureRequester.kt @@ -0,0 +1,65 @@ +package bot.molt.android + +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.media.projection.MediaProjectionManager +import androidx.activity.ComponentActivity +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.contract.ActivityResultContracts +import androidx.appcompat.app.AlertDialog +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class ScreenCaptureRequester(private val activity: ComponentActivity) { + data class CaptureResult(val resultCode: Int, val data: Intent) + + private val mutex = Mutex() + private var pending: CompletableDeferred? = null + + private val launcher: ActivityResultLauncher = + activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> + val p = pending + pending = null + val data = result.data + if (result.resultCode == Activity.RESULT_OK && data != null) { + p?.complete(CaptureResult(result.resultCode, data)) + } else { + p?.complete(null) + } + } + + suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? = + mutex.withLock { + val proceed = showRationaleDialog() + if (!proceed) return null + + val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + val intent = mgr.createScreenCaptureIntent() + + val deferred = CompletableDeferred() + pending = deferred + withContext(Dispatchers.Main) { launcher.launch(intent) } + + withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } } + } + + private suspend fun showRationaleDialog(): Boolean = + withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + AlertDialog.Builder(activity) + .setTitle("Screen recording required") + .setMessage("Moltbot needs to record the screen for this command.") + .setPositiveButton("Continue") { _, _ -> cont.resume(true) } + .setNegativeButton("Not now") { _, _ -> cont.resume(false) } + .setOnCancelListener { cont.resume(false) } + .show() + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/SecurePrefs.kt b/apps/android/app/src/main/java/bot/molt/android/SecurePrefs.kt new file mode 100644 index 00000000000..7ee3294dc3f --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/SecurePrefs.kt @@ -0,0 +1,308 @@ +@file:Suppress("DEPRECATION") + +package bot.molt.android + +import android.content.Context +import androidx.core.content.edit +import androidx.security.crypto.EncryptedSharedPreferences +import androidx.security.crypto.MasterKey +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonPrimitive +import java.util.UUID + +class SecurePrefs(context: Context) { + companion object { + val defaultWakeWords: List = listOf("clawd", "claude") + private const val displayNameKey = "node.displayName" + private const val voiceWakeModeKey = "voiceWake.mode" + } + + private val json = Json { ignoreUnknownKeys = true } + + private val masterKey = + MasterKey.Builder(context) + .setKeyScheme(MasterKey.KeyScheme.AES256_GCM) + .build() + + private val prefs = + EncryptedSharedPreferences.create( + context, + "moltbot.node.secure", + masterKey, + EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV, + EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM, + ) + + private val _instanceId = MutableStateFlow(loadOrCreateInstanceId()) + val instanceId: StateFlow = _instanceId + + private val _displayName = + MutableStateFlow(loadOrMigrateDisplayName(context = context)) + val displayName: StateFlow = _displayName + + private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true)) + val cameraEnabled: StateFlow = _cameraEnabled + + private val _locationMode = + MutableStateFlow(LocationMode.fromRawValue(prefs.getString("location.enabledMode", "off"))) + val locationMode: StateFlow = _locationMode + + private val _locationPreciseEnabled = + MutableStateFlow(prefs.getBoolean("location.preciseEnabled", true)) + val locationPreciseEnabled: StateFlow = _locationPreciseEnabled + + private val _preventSleep = MutableStateFlow(prefs.getBoolean("screen.preventSleep", true)) + val preventSleep: StateFlow = _preventSleep + + private val _manualEnabled = + MutableStateFlow(readBoolWithMigration("gateway.manual.enabled", "bridge.manual.enabled", false)) + val manualEnabled: StateFlow = _manualEnabled + + private val _manualHost = + MutableStateFlow(readStringWithMigration("gateway.manual.host", "bridge.manual.host", "")) + val manualHost: StateFlow = _manualHost + + private val _manualPort = + MutableStateFlow(readIntWithMigration("gateway.manual.port", "bridge.manual.port", 18789)) + val manualPort: StateFlow = _manualPort + + private val _manualTls = + MutableStateFlow(readBoolWithMigration("gateway.manual.tls", null, true)) + val manualTls: StateFlow = _manualTls + + private val _lastDiscoveredStableId = + MutableStateFlow( + readStringWithMigration( + "gateway.lastDiscoveredStableID", + "bridge.lastDiscoveredStableId", + "", + ), + ) + val lastDiscoveredStableId: StateFlow = _lastDiscoveredStableId + + private val _canvasDebugStatusEnabled = + MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false)) + val canvasDebugStatusEnabled: StateFlow = _canvasDebugStatusEnabled + + private val _wakeWords = MutableStateFlow(loadWakeWords()) + val wakeWords: StateFlow> = _wakeWords + + private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode()) + val voiceWakeMode: StateFlow = _voiceWakeMode + + private val _talkEnabled = MutableStateFlow(prefs.getBoolean("talk.enabled", false)) + val talkEnabled: StateFlow = _talkEnabled + + fun setLastDiscoveredStableId(value: String) { + val trimmed = value.trim() + prefs.edit { putString("gateway.lastDiscoveredStableID", trimmed) } + _lastDiscoveredStableId.value = trimmed + } + + fun setDisplayName(value: String) { + val trimmed = value.trim() + prefs.edit { putString(displayNameKey, trimmed) } + _displayName.value = trimmed + } + + fun setCameraEnabled(value: Boolean) { + prefs.edit { putBoolean("camera.enabled", value) } + _cameraEnabled.value = value + } + + fun setLocationMode(mode: LocationMode) { + prefs.edit { putString("location.enabledMode", mode.rawValue) } + _locationMode.value = mode + } + + fun setLocationPreciseEnabled(value: Boolean) { + prefs.edit { putBoolean("location.preciseEnabled", value) } + _locationPreciseEnabled.value = value + } + + fun setPreventSleep(value: Boolean) { + prefs.edit { putBoolean("screen.preventSleep", value) } + _preventSleep.value = value + } + + fun setManualEnabled(value: Boolean) { + prefs.edit { putBoolean("gateway.manual.enabled", value) } + _manualEnabled.value = value + } + + fun setManualHost(value: String) { + val trimmed = value.trim() + prefs.edit { putString("gateway.manual.host", trimmed) } + _manualHost.value = trimmed + } + + fun setManualPort(value: Int) { + prefs.edit { putInt("gateway.manual.port", value) } + _manualPort.value = value + } + + fun setManualTls(value: Boolean) { + prefs.edit { putBoolean("gateway.manual.tls", value) } + _manualTls.value = value + } + + fun setCanvasDebugStatusEnabled(value: Boolean) { + prefs.edit { putBoolean("canvas.debugStatusEnabled", value) } + _canvasDebugStatusEnabled.value = value + } + + fun loadGatewayToken(): String? { + val key = "gateway.token.${_instanceId.value}" + val stored = prefs.getString(key, null)?.trim() + if (!stored.isNullOrEmpty()) return stored + val legacy = prefs.getString("bridge.token.${_instanceId.value}", null)?.trim() + return legacy?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayToken(token: String) { + val key = "gateway.token.${_instanceId.value}" + prefs.edit { putString(key, token.trim()) } + } + + fun loadGatewayPassword(): String? { + val key = "gateway.password.${_instanceId.value}" + val stored = prefs.getString(key, null)?.trim() + return stored?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayPassword(password: String) { + val key = "gateway.password.${_instanceId.value}" + prefs.edit { putString(key, password.trim()) } + } + + fun loadGatewayTlsFingerprint(stableId: String): String? { + val key = "gateway.tls.$stableId" + return prefs.getString(key, null)?.trim()?.takeIf { it.isNotEmpty() } + } + + fun saveGatewayTlsFingerprint(stableId: String, fingerprint: String) { + val key = "gateway.tls.$stableId" + prefs.edit { putString(key, fingerprint.trim()) } + } + + fun getString(key: String): String? { + return prefs.getString(key, null) + } + + fun putString(key: String, value: String) { + prefs.edit { putString(key, value) } + } + + fun remove(key: String) { + prefs.edit { remove(key) } + } + + private fun loadOrCreateInstanceId(): String { + val existing = prefs.getString("node.instanceId", null)?.trim() + if (!existing.isNullOrBlank()) return existing + val fresh = UUID.randomUUID().toString() + prefs.edit { putString("node.instanceId", fresh) } + return fresh + } + + private fun loadOrMigrateDisplayName(context: Context): String { + val existing = prefs.getString(displayNameKey, null)?.trim().orEmpty() + if (existing.isNotEmpty() && existing != "Android Node") return existing + + val candidate = DeviceNames.bestDefaultNodeName(context).trim() + val resolved = candidate.ifEmpty { "Android Node" } + + prefs.edit { putString(displayNameKey, resolved) } + return resolved + } + + fun setWakeWords(words: List) { + val sanitized = WakeWords.sanitize(words, defaultWakeWords) + val encoded = + JsonArray(sanitized.map { JsonPrimitive(it) }).toString() + prefs.edit { putString("voiceWake.triggerWords", encoded) } + _wakeWords.value = sanitized + } + + fun setVoiceWakeMode(mode: VoiceWakeMode) { + prefs.edit { putString(voiceWakeModeKey, mode.rawValue) } + _voiceWakeMode.value = mode + } + + fun setTalkEnabled(value: Boolean) { + prefs.edit { putBoolean("talk.enabled", value) } + _talkEnabled.value = value + } + + private fun loadVoiceWakeMode(): VoiceWakeMode { + val raw = prefs.getString(voiceWakeModeKey, null) + val resolved = VoiceWakeMode.fromRawValue(raw) + + // Default ON (foreground) when unset. + if (raw.isNullOrBlank()) { + prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) } + } + + return resolved + } + + private fun loadWakeWords(): List { + val raw = prefs.getString("voiceWake.triggerWords", null)?.trim() + if (raw.isNullOrEmpty()) return defaultWakeWords + return try { + val element = json.parseToJsonElement(raw) + val array = element as? JsonArray ?: return defaultWakeWords + val decoded = + array.mapNotNull { item -> + when (item) { + is JsonNull -> null + is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() } + else -> null + } + } + WakeWords.sanitize(decoded, defaultWakeWords) + } catch (_: Throwable) { + defaultWakeWords + } + } + + private fun readBoolWithMigration(newKey: String, oldKey: String?, defaultValue: Boolean): Boolean { + if (prefs.contains(newKey)) { + return prefs.getBoolean(newKey, defaultValue) + } + if (oldKey != null && prefs.contains(oldKey)) { + val value = prefs.getBoolean(oldKey, defaultValue) + prefs.edit { putBoolean(newKey, value) } + return value + } + return defaultValue + } + + private fun readStringWithMigration(newKey: String, oldKey: String?, defaultValue: String): String { + if (prefs.contains(newKey)) { + return prefs.getString(newKey, defaultValue) ?: defaultValue + } + if (oldKey != null && prefs.contains(oldKey)) { + val value = prefs.getString(oldKey, defaultValue) ?: defaultValue + prefs.edit { putString(newKey, value) } + return value + } + return defaultValue + } + + private fun readIntWithMigration(newKey: String, oldKey: String?, defaultValue: Int): Int { + if (prefs.contains(newKey)) { + return prefs.getInt(newKey, defaultValue) + } + if (oldKey != null && prefs.contains(oldKey)) { + val value = prefs.getInt(oldKey, defaultValue) + prefs.edit { putInt(newKey, value) } + return value + } + return defaultValue + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/SessionKey.kt b/apps/android/app/src/main/java/bot/molt/android/SessionKey.kt new file mode 100644 index 00000000000..e640516495b --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/SessionKey.kt @@ -0,0 +1,13 @@ +package bot.molt.android + +internal fun normalizeMainKey(raw: String?): String { + val trimmed = raw?.trim() + return if (!trimmed.isNullOrEmpty()) trimmed else "main" +} + +internal fun isCanonicalMainSessionKey(raw: String?): Boolean { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return false + if (trimmed == "global") return true + return trimmed.startsWith("agent:") +} diff --git a/apps/android/app/src/main/java/bot/molt/android/VoiceWakeMode.kt b/apps/android/app/src/main/java/bot/molt/android/VoiceWakeMode.kt new file mode 100644 index 00000000000..e0862cc25db --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/VoiceWakeMode.kt @@ -0,0 +1,14 @@ +package bot.molt.android + +enum class VoiceWakeMode(val rawValue: String) { + Off("off"), + Foreground("foreground"), + Always("always"), + ; + + companion object { + fun fromRawValue(raw: String?): VoiceWakeMode { + return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/WakeWords.kt b/apps/android/app/src/main/java/bot/molt/android/WakeWords.kt new file mode 100644 index 00000000000..56b85a5df91 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/WakeWords.kt @@ -0,0 +1,21 @@ +package bot.molt.android + +object WakeWords { + const val maxWords: Int = 32 + const val maxWordLength: Int = 64 + + fun parseCommaSeparated(input: String): List { + return input.split(",").map { it.trim() }.filter { it.isNotEmpty() } + } + + fun parseIfChanged(input: String, current: List): List? { + val parsed = parseCommaSeparated(input) + return if (parsed == current) null else parsed + } + + fun sanitize(words: List, defaults: List): List { + val cleaned = + words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) } + return cleaned.ifEmpty { defaults } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt b/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt new file mode 100644 index 00000000000..eef66fece4d --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt @@ -0,0 +1,524 @@ +package bot.molt.android.chat + +import bot.molt.android.gateway.GatewaySession +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject + +class ChatController( + private val scope: CoroutineScope, + private val session: GatewaySession, + private val json: Json, + private val supportsChatSubscribe: Boolean, +) { + private val _sessionKey = MutableStateFlow("main") + val sessionKey: StateFlow = _sessionKey.asStateFlow() + + private val _sessionId = MutableStateFlow(null) + val sessionId: StateFlow = _sessionId.asStateFlow() + + private val _messages = MutableStateFlow>(emptyList()) + val messages: StateFlow> = _messages.asStateFlow() + + private val _errorText = MutableStateFlow(null) + val errorText: StateFlow = _errorText.asStateFlow() + + private val _healthOk = MutableStateFlow(false) + val healthOk: StateFlow = _healthOk.asStateFlow() + + private val _thinkingLevel = MutableStateFlow("off") + val thinkingLevel: StateFlow = _thinkingLevel.asStateFlow() + + private val _pendingRunCount = MutableStateFlow(0) + val pendingRunCount: StateFlow = _pendingRunCount.asStateFlow() + + private val _streamingAssistantText = MutableStateFlow(null) + val streamingAssistantText: StateFlow = _streamingAssistantText.asStateFlow() + + private val pendingToolCallsById = ConcurrentHashMap() + private val _pendingToolCalls = MutableStateFlow>(emptyList()) + val pendingToolCalls: StateFlow> = _pendingToolCalls.asStateFlow() + + private val _sessions = MutableStateFlow>(emptyList()) + val sessions: StateFlow> = _sessions.asStateFlow() + + private val pendingRuns = mutableSetOf() + private val pendingRunTimeoutJobs = ConcurrentHashMap() + private val pendingRunTimeoutMs = 120_000L + + private var lastHealthPollAtMs: Long? = null + + fun onDisconnected(message: String) { + _healthOk.value = false + // Not an error; keep connection status in the UI pill. + _errorText.value = null + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + _sessionId.value = null + } + + fun load(sessionKey: String) { + val key = sessionKey.trim().ifEmpty { "main" } + _sessionKey.value = key + scope.launch { bootstrap(forceHealth = true) } + } + + fun applyMainSessionKey(mainSessionKey: String) { + val trimmed = mainSessionKey.trim() + if (trimmed.isEmpty()) return + if (_sessionKey.value == trimmed) return + if (_sessionKey.value != "main") return + _sessionKey.value = trimmed + scope.launch { bootstrap(forceHealth = true) } + } + + fun refresh() { + scope.launch { bootstrap(forceHealth = true) } + } + + fun refreshSessions(limit: Int? = null) { + scope.launch { fetchSessions(limit = limit) } + } + + fun setThinkingLevel(thinkingLevel: String) { + val normalized = normalizeThinking(thinkingLevel) + if (normalized == _thinkingLevel.value) return + _thinkingLevel.value = normalized + } + + fun switchSession(sessionKey: String) { + val key = sessionKey.trim() + if (key.isEmpty()) return + if (key == _sessionKey.value) return + _sessionKey.value = key + scope.launch { bootstrap(forceHealth = true) } + } + + fun sendMessage( + message: String, + thinkingLevel: String, + attachments: List, + ) { + val trimmed = message.trim() + if (trimmed.isEmpty() && attachments.isEmpty()) return + if (!_healthOk.value) { + _errorText.value = "Gateway health not OK; cannot send" + return + } + + val runId = UUID.randomUUID().toString() + val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed + val sessionKey = _sessionKey.value + val thinking = normalizeThinking(thinkingLevel) + + // Optimistic user message. + val userContent = + buildList { + add(ChatMessageContent(type = "text", text = text)) + for (att in attachments) { + add( + ChatMessageContent( + type = att.type, + mimeType = att.mimeType, + fileName = att.fileName, + base64 = att.base64, + ), + ) + } + } + _messages.value = + _messages.value + + ChatMessage( + id = UUID.randomUUID().toString(), + role = "user", + content = userContent, + timestampMs = System.currentTimeMillis(), + ) + + armPendingRunTimeout(runId) + synchronized(pendingRuns) { + pendingRuns.add(runId) + _pendingRunCount.value = pendingRuns.size + } + + _errorText.value = null + _streamingAssistantText.value = null + pendingToolCallsById.clear() + publishPendingToolCalls() + + scope.launch { + try { + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(sessionKey)) + put("message", JsonPrimitive(text)) + put("thinking", JsonPrimitive(thinking)) + put("timeoutMs", JsonPrimitive(30_000)) + put("idempotencyKey", JsonPrimitive(runId)) + if (attachments.isNotEmpty()) { + put( + "attachments", + JsonArray( + attachments.map { att -> + buildJsonObject { + put("type", JsonPrimitive(att.type)) + put("mimeType", JsonPrimitive(att.mimeType)) + put("fileName", JsonPrimitive(att.fileName)) + put("content", JsonPrimitive(att.base64)) + } + }, + ), + ) + } + } + val res = session.request("chat.send", params.toString()) + val actualRunId = parseRunId(res) ?: runId + if (actualRunId != runId) { + clearPendingRun(runId) + armPendingRunTimeout(actualRunId) + synchronized(pendingRuns) { + pendingRuns.add(actualRunId) + _pendingRunCount.value = pendingRuns.size + } + } + } catch (err: Throwable) { + clearPendingRun(runId) + _errorText.value = err.message + } + } + } + + fun abort() { + val runIds = + synchronized(pendingRuns) { + pendingRuns.toList() + } + if (runIds.isEmpty()) return + scope.launch { + for (runId in runIds) { + try { + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(_sessionKey.value)) + put("runId", JsonPrimitive(runId)) + } + session.request("chat.abort", params.toString()) + } catch (_: Throwable) { + // best-effort + } + } + } + } + + fun handleGatewayEvent(event: String, payloadJson: String?) { + when (event) { + "tick" -> { + scope.launch { pollHealthIfNeeded(force = false) } + } + "health" -> { + // If we receive a health snapshot, the gateway is reachable. + _healthOk.value = true + } + "seqGap" -> { + _errorText.value = "Event stream interrupted; try refreshing." + clearPendingRuns() + } + "chat" -> { + if (payloadJson.isNullOrBlank()) return + handleChatEvent(payloadJson) + } + "agent" -> { + if (payloadJson.isNullOrBlank()) return + handleAgentEvent(payloadJson) + } + } + } + + private suspend fun bootstrap(forceHealth: Boolean) { + _errorText.value = null + _healthOk.value = false + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + _sessionId.value = null + + val key = _sessionKey.value + try { + if (supportsChatSubscribe) { + try { + session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") + } catch (_: Throwable) { + // best-effort + } + } + + val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""") + val history = parseHistory(historyJson, sessionKey = key) + _messages.value = history.messages + _sessionId.value = history.sessionId + history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } + + pollHealthIfNeeded(force = forceHealth) + fetchSessions(limit = 50) + } catch (err: Throwable) { + _errorText.value = err.message + } + } + + private suspend fun fetchSessions(limit: Int?) { + try { + val params = + buildJsonObject { + put("includeGlobal", JsonPrimitive(true)) + put("includeUnknown", JsonPrimitive(false)) + if (limit != null && limit > 0) put("limit", JsonPrimitive(limit)) + } + val res = session.request("sessions.list", params.toString()) + _sessions.value = parseSessions(res) + } catch (_: Throwable) { + // best-effort + } + } + + private suspend fun pollHealthIfNeeded(force: Boolean) { + val now = System.currentTimeMillis() + val last = lastHealthPollAtMs + if (!force && last != null && now - last < 10_000) return + lastHealthPollAtMs = now + try { + session.request("health", null) + _healthOk.value = true + } catch (_: Throwable) { + _healthOk.value = false + } + } + + private fun handleChatEvent(payloadJson: String) { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val sessionKey = payload["sessionKey"].asStringOrNull()?.trim() + if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return + + val runId = payload["runId"].asStringOrNull() + if (runId != null) { + val isPending = + synchronized(pendingRuns) { + pendingRuns.contains(runId) + } + if (!isPending) return + } + + val state = payload["state"].asStringOrNull() + when (state) { + "final", "aborted", "error" -> { + if (state == "error") { + _errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed" + } + if (runId != null) clearPendingRun(runId) else clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + scope.launch { + try { + val historyJson = + session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""") + val history = parseHistory(historyJson, sessionKey = _sessionKey.value) + _messages.value = history.messages + _sessionId.value = history.sessionId + history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } + } catch (_: Throwable) { + // best-effort + } + } + } + } + } + + private fun handleAgentEvent(payloadJson: String) { + val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val runId = payload["runId"].asStringOrNull() + val sessionId = _sessionId.value + if (sessionId != null && runId != sessionId) return + + val stream = payload["stream"].asStringOrNull() + val data = payload["data"].asObjectOrNull() + + when (stream) { + "assistant" -> { + val text = data?.get("text")?.asStringOrNull() + if (!text.isNullOrEmpty()) { + _streamingAssistantText.value = text + } + } + "tool" -> { + val phase = data?.get("phase")?.asStringOrNull() + val name = data?.get("name")?.asStringOrNull() + val toolCallId = data?.get("toolCallId")?.asStringOrNull() + if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return + + val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis() + if (phase == "start") { + val args = data?.get("args").asObjectOrNull() + pendingToolCallsById[toolCallId] = + ChatPendingToolCall( + toolCallId = toolCallId, + name = name, + args = args, + startedAtMs = ts, + isError = null, + ) + publishPendingToolCalls() + } else if (phase == "result") { + pendingToolCallsById.remove(toolCallId) + publishPendingToolCalls() + } + } + "error" -> { + _errorText.value = "Event stream interrupted; try refreshing." + clearPendingRuns() + pendingToolCallsById.clear() + publishPendingToolCalls() + _streamingAssistantText.value = null + } + } + } + + private fun publishPendingToolCalls() { + _pendingToolCalls.value = + pendingToolCallsById.values.sortedBy { it.startedAtMs } + } + + private fun armPendingRunTimeout(runId: String) { + pendingRunTimeoutJobs[runId]?.cancel() + pendingRunTimeoutJobs[runId] = + scope.launch { + delay(pendingRunTimeoutMs) + val stillPending = + synchronized(pendingRuns) { + pendingRuns.contains(runId) + } + if (!stillPending) return@launch + clearPendingRun(runId) + _errorText.value = "Timed out waiting for a reply; try again or refresh." + } + } + + private fun clearPendingRun(runId: String) { + pendingRunTimeoutJobs.remove(runId)?.cancel() + synchronized(pendingRuns) { + pendingRuns.remove(runId) + _pendingRunCount.value = pendingRuns.size + } + } + + private fun clearPendingRuns() { + for ((_, job) in pendingRunTimeoutJobs) { + job.cancel() + } + pendingRunTimeoutJobs.clear() + synchronized(pendingRuns) { + pendingRuns.clear() + _pendingRunCount.value = 0 + } + } + + private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory { + val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList()) + val sid = root["sessionId"].asStringOrNull() + val thinkingLevel = root["thinkingLevel"].asStringOrNull() + val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList()) + + val messages = + array.mapNotNull { item -> + val obj = item.asObjectOrNull() ?: return@mapNotNull null + val role = obj["role"].asStringOrNull() ?: return@mapNotNull null + val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList() + val ts = obj["timestamp"].asLongOrNull() + ChatMessage( + id = UUID.randomUUID().toString(), + role = role, + content = content, + timestampMs = ts, + ) + } + + return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages) + } + + private fun parseMessageContent(el: JsonElement): ChatMessageContent? { + val obj = el.asObjectOrNull() ?: return null + val type = obj["type"].asStringOrNull() ?: "text" + return if (type == "text") { + ChatMessageContent(type = "text", text = obj["text"].asStringOrNull()) + } else { + ChatMessageContent( + type = type, + mimeType = obj["mimeType"].asStringOrNull(), + fileName = obj["fileName"].asStringOrNull(), + base64 = obj["content"].asStringOrNull(), + ) + } + } + + private fun parseSessions(jsonString: String): List { + val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList() + val sessions = root["sessions"].asArrayOrNull() ?: return emptyList() + return sessions.mapNotNull { item -> + val obj = item.asObjectOrNull() ?: return@mapNotNull null + val key = obj["key"].asStringOrNull()?.trim().orEmpty() + if (key.isEmpty()) return@mapNotNull null + val updatedAt = obj["updatedAt"].asLongOrNull() + val displayName = obj["displayName"].asStringOrNull()?.trim() + ChatSessionEntry(key = key, updatedAtMs = updatedAt, displayName = displayName) + } + } + + private fun parseRunId(resJson: String): String? { + return try { + json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull() + } catch (_: Throwable) { + null + } + } + + private fun normalizeThinking(raw: String): String { + return when (raw.trim().lowercase()) { + "low" -> "low" + "medium" -> "medium" + "high" -> "high" + else -> "off" + } + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun JsonElement?.asLongOrNull(): Long? = + when (this) { + is JsonPrimitive -> content.toLongOrNull() + else -> null + } diff --git a/apps/android/app/src/main/java/bot/molt/android/chat/ChatModels.kt b/apps/android/app/src/main/java/bot/molt/android/chat/ChatModels.kt new file mode 100644 index 00000000000..3406244521f --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/chat/ChatModels.kt @@ -0,0 +1,44 @@ +package bot.molt.android.chat + +data class ChatMessage( + val id: String, + val role: String, + val content: List, + val timestampMs: Long?, +) + +data class ChatMessageContent( + val type: String = "text", + val text: String? = null, + val mimeType: String? = null, + val fileName: String? = null, + val base64: String? = null, +) + +data class ChatPendingToolCall( + val toolCallId: String, + val name: String, + val args: kotlinx.serialization.json.JsonObject? = null, + val startedAtMs: Long, + val isError: Boolean? = null, +) + +data class ChatSessionEntry( + val key: String, + val updatedAtMs: Long?, + val displayName: String? = null, +) + +data class ChatHistory( + val sessionKey: String, + val sessionId: String?, + val thinkingLevel: String?, + val messages: List, +) + +data class OutgoingAttachment( + val type: String, + val mimeType: String, + val fileName: String, + val base64: String, +) diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/BonjourEscapes.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/BonjourEscapes.kt new file mode 100644 index 00000000000..2c0c34d68ea --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/BonjourEscapes.kt @@ -0,0 +1,35 @@ +package bot.molt.android.gateway + +object BonjourEscapes { + fun decode(input: String): String { + if (input.isEmpty()) return input + + val bytes = mutableListOf() + var i = 0 + while (i < input.length) { + if (input[i] == '\\' && i + 3 < input.length) { + val d0 = input[i + 1] + val d1 = input[i + 2] + val d2 = input[i + 3] + if (d0.isDigit() && d1.isDigit() && d2.isDigit()) { + val value = + ((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code) + if (value in 0..255) { + bytes.add(value.toByte()) + i += 4 + continue + } + } + } + + val codePoint = Character.codePointAt(input, i) + val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8) + for (b in charBytes) { + bytes.add(b) + } + i += Character.charCount(codePoint) + } + + return String(bytes.toByteArray(), Charsets.UTF_8) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceAuthStore.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceAuthStore.kt new file mode 100644 index 00000000000..6b90b467291 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceAuthStore.kt @@ -0,0 +1,26 @@ +package bot.molt.android.gateway + +import bot.molt.android.SecurePrefs + +class DeviceAuthStore(private val prefs: SecurePrefs) { + fun loadToken(deviceId: String, role: String): String? { + val key = tokenKey(deviceId, role) + return prefs.getString(key)?.trim()?.takeIf { it.isNotEmpty() } + } + + fun saveToken(deviceId: String, role: String, token: String) { + val key = tokenKey(deviceId, role) + prefs.putString(key, token.trim()) + } + + fun clearToken(deviceId: String, role: String) { + val key = tokenKey(deviceId, role) + prefs.remove(key) + } + + private fun tokenKey(deviceId: String, role: String): String { + val normalizedDevice = deviceId.trim().lowercase() + val normalizedRole = role.trim().lowercase() + return "gateway.deviceToken.$normalizedDevice.$normalizedRole" + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceIdentityStore.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceIdentityStore.kt new file mode 100644 index 00000000000..58a0acefffd --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/DeviceIdentityStore.kt @@ -0,0 +1,146 @@ +package bot.molt.android.gateway + +import android.content.Context +import android.util.Base64 +import java.io.File +import java.security.KeyFactory +import java.security.KeyPairGenerator +import java.security.MessageDigest +import java.security.Signature +import java.security.spec.PKCS8EncodedKeySpec +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +@Serializable +data class DeviceIdentity( + val deviceId: String, + val publicKeyRawBase64: String, + val privateKeyPkcs8Base64: String, + val createdAtMs: Long, +) + +class DeviceIdentityStore(context: Context) { + private val json = Json { ignoreUnknownKeys = true } + private val identityFile = File(context.filesDir, "moltbot/identity/device.json") + + @Synchronized + fun loadOrCreate(): DeviceIdentity { + val existing = load() + if (existing != null) { + val derived = deriveDeviceId(existing.publicKeyRawBase64) + if (derived != null && derived != existing.deviceId) { + val updated = existing.copy(deviceId = derived) + save(updated) + return updated + } + return existing + } + val fresh = generate() + save(fresh) + return fresh + } + + fun signPayload(payload: String, identity: DeviceIdentity): String? { + return try { + val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT) + val keySpec = PKCS8EncodedKeySpec(privateKeyBytes) + val keyFactory = KeyFactory.getInstance("Ed25519") + val privateKey = keyFactory.generatePrivate(keySpec) + val signature = Signature.getInstance("Ed25519") + signature.initSign(privateKey) + signature.update(payload.toByteArray(Charsets.UTF_8)) + base64UrlEncode(signature.sign()) + } catch (_: Throwable) { + null + } + } + + fun publicKeyBase64Url(identity: DeviceIdentity): String? { + return try { + val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT) + base64UrlEncode(raw) + } catch (_: Throwable) { + null + } + } + + private fun load(): DeviceIdentity? { + return try { + if (!identityFile.exists()) return null + val raw = identityFile.readText(Charsets.UTF_8) + val decoded = json.decodeFromString(DeviceIdentity.serializer(), raw) + if (decoded.deviceId.isBlank() || + decoded.publicKeyRawBase64.isBlank() || + decoded.privateKeyPkcs8Base64.isBlank() + ) { + null + } else { + decoded + } + } catch (_: Throwable) { + null + } + } + + private fun save(identity: DeviceIdentity) { + try { + identityFile.parentFile?.mkdirs() + val encoded = json.encodeToString(DeviceIdentity.serializer(), identity) + identityFile.writeText(encoded, Charsets.UTF_8) + } catch (_: Throwable) { + // best-effort only + } + } + + private fun generate(): DeviceIdentity { + val keyPair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair() + val spki = keyPair.public.encoded + val rawPublic = stripSpkiPrefix(spki) + val deviceId = sha256Hex(rawPublic) + val privateKey = keyPair.private.encoded + return DeviceIdentity( + deviceId = deviceId, + publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP), + privateKeyPkcs8Base64 = Base64.encodeToString(privateKey, Base64.NO_WRAP), + createdAtMs = System.currentTimeMillis(), + ) + } + + private fun deriveDeviceId(publicKeyRawBase64: String): String? { + return try { + val raw = Base64.decode(publicKeyRawBase64, Base64.DEFAULT) + sha256Hex(raw) + } catch (_: Throwable) { + null + } + } + + private fun stripSpkiPrefix(spki: ByteArray): ByteArray { + if (spki.size == ED25519_SPKI_PREFIX.size + 32 && + spki.copyOfRange(0, ED25519_SPKI_PREFIX.size).contentEquals(ED25519_SPKI_PREFIX) + ) { + return spki.copyOfRange(ED25519_SPKI_PREFIX.size, spki.size) + } + return spki + } + + private fun sha256Hex(data: ByteArray): String { + val digest = MessageDigest.getInstance("SHA-256").digest(data) + val out = StringBuilder(digest.size * 2) + for (byte in digest) { + out.append(String.format("%02x", byte)) + } + return out.toString() + } + + private fun base64UrlEncode(data: ByteArray): String { + return Base64.encodeToString(data, Base64.URL_SAFE or Base64.NO_WRAP or Base64.NO_PADDING) + } + + companion object { + private val ED25519_SPKI_PREFIX = + byteArrayOf( + 0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x03, 0x21, 0x00, + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayDiscovery.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayDiscovery.kt new file mode 100644 index 00000000000..53bdb5588ac --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayDiscovery.kt @@ -0,0 +1,519 @@ +package bot.molt.android.gateway + +import android.content.Context +import android.net.ConnectivityManager +import android.net.DnsResolver +import android.net.NetworkCapabilities +import android.net.nsd.NsdManager +import android.net.nsd.NsdServiceInfo +import android.os.CancellationSignal +import android.util.Log +import java.io.IOException +import java.net.InetSocketAddress +import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Executor +import java.util.concurrent.Executors +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import org.xbill.DNS.AAAARecord +import org.xbill.DNS.ARecord +import org.xbill.DNS.DClass +import org.xbill.DNS.ExtendedResolver +import org.xbill.DNS.Message +import org.xbill.DNS.Name +import org.xbill.DNS.PTRRecord +import org.xbill.DNS.Record +import org.xbill.DNS.Rcode +import org.xbill.DNS.Resolver +import org.xbill.DNS.SRVRecord +import org.xbill.DNS.Section +import org.xbill.DNS.SimpleResolver +import org.xbill.DNS.TextParseException +import org.xbill.DNS.TXTRecord +import org.xbill.DNS.Type +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +@Suppress("DEPRECATION") +class GatewayDiscovery( + context: Context, + private val scope: CoroutineScope, +) { + private val nsd = context.getSystemService(NsdManager::class.java) + private val connectivity = context.getSystemService(ConnectivityManager::class.java) + private val dns = DnsResolver.getInstance() + private val serviceType = "_moltbot-gw._tcp." + private val wideAreaDomain = "moltbot.internal." + private val logTag = "Moltbot/GatewayDiscovery" + + private val localById = ConcurrentHashMap() + private val unicastById = ConcurrentHashMap() + private val _gateways = MutableStateFlow>(emptyList()) + val gateways: StateFlow> = _gateways.asStateFlow() + + private val _statusText = MutableStateFlow("Searching…") + val statusText: StateFlow = _statusText.asStateFlow() + + private var unicastJob: Job? = null + private val dnsExecutor: Executor = Executors.newCachedThreadPool() + + @Volatile private var lastWideAreaRcode: Int? = null + @Volatile private var lastWideAreaCount: Int = 0 + + private val discoveryListener = + object : NsdManager.DiscoveryListener { + override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {} + override fun onDiscoveryStarted(serviceType: String) {} + override fun onDiscoveryStopped(serviceType: String) {} + + override fun onServiceFound(serviceInfo: NsdServiceInfo) { + if (serviceInfo.serviceType != this@GatewayDiscovery.serviceType) return + resolve(serviceInfo) + } + + override fun onServiceLost(serviceInfo: NsdServiceInfo) { + val serviceName = BonjourEscapes.decode(serviceInfo.serviceName) + val id = stableId(serviceName, "local.") + localById.remove(id) + publish() + } + } + + init { + startLocalDiscovery() + startUnicastDiscovery(wideAreaDomain) + } + + private fun startLocalDiscovery() { + try { + nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener) + } catch (_: Throwable) { + // ignore (best-effort) + } + } + + private fun stopLocalDiscovery() { + try { + nsd.stopServiceDiscovery(discoveryListener) + } catch (_: Throwable) { + // ignore (best-effort) + } + } + + private fun startUnicastDiscovery(domain: String) { + unicastJob = + scope.launch(Dispatchers.IO) { + while (true) { + try { + refreshUnicast(domain) + } catch (_: Throwable) { + // ignore (best-effort) + } + delay(5000) + } + } + } + + private fun resolve(serviceInfo: NsdServiceInfo) { + nsd.resolveService( + serviceInfo, + object : NsdManager.ResolveListener { + override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {} + + override fun onServiceResolved(resolved: NsdServiceInfo) { + val host = resolved.host?.hostAddress ?: return + val port = resolved.port + if (port <= 0) return + + val rawServiceName = resolved.serviceName + val serviceName = BonjourEscapes.decode(rawServiceName) + val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName) + val lanHost = txt(resolved, "lanHost") + val tailnetDns = txt(resolved, "tailnetDns") + val gatewayPort = txtInt(resolved, "gatewayPort") + val canvasPort = txtInt(resolved, "canvasPort") + val tlsEnabled = txtBool(resolved, "gatewayTls") + val tlsFingerprint = txt(resolved, "gatewayTlsSha256") + val id = stableId(serviceName, "local.") + localById[id] = + GatewayEndpoint( + stableId = id, + name = displayName, + host = host, + port = port, + lanHost = lanHost, + tailnetDns = tailnetDns, + gatewayPort = gatewayPort, + canvasPort = canvasPort, + tlsEnabled = tlsEnabled, + tlsFingerprintSha256 = tlsFingerprint, + ) + publish() + } + }, + ) + } + + private fun publish() { + _gateways.value = + (localById.values + unicastById.values).sortedBy { it.name.lowercase() } + _statusText.value = buildStatusText() + } + + private fun buildStatusText(): String { + val localCount = localById.size + val wideRcode = lastWideAreaRcode + val wideCount = lastWideAreaCount + + val wide = + when (wideRcode) { + null -> "Wide: ?" + Rcode.NOERROR -> "Wide: $wideCount" + Rcode.NXDOMAIN -> "Wide: NXDOMAIN" + else -> "Wide: ${Rcode.string(wideRcode)}" + } + + return when { + localCount == 0 && wideRcode == null -> "Searching for gateways…" + localCount == 0 -> "$wide" + else -> "Local: $localCount • $wide" + } + } + + private fun stableId(serviceName: String, domain: String): String { + return "${serviceType}|${domain}|${normalizeName(serviceName)}" + } + + private fun normalizeName(raw: String): String { + return raw.trim().split(Regex("\\s+")).joinToString(" ") + } + + private fun txt(info: NsdServiceInfo, key: String): String? { + val bytes = info.attributes[key] ?: return null + return try { + String(bytes, Charsets.UTF_8).trim().ifEmpty { null } + } catch (_: Throwable) { + null + } + } + + private fun txtInt(info: NsdServiceInfo, key: String): Int? { + return txt(info, key)?.toIntOrNull() + } + + private fun txtBool(info: NsdServiceInfo, key: String): Boolean { + val raw = txt(info, key)?.trim()?.lowercase() ?: return false + return raw == "1" || raw == "true" || raw == "yes" + } + + private suspend fun refreshUnicast(domain: String) { + val ptrName = "${serviceType}${domain}" + val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return + val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord } + + val next = LinkedHashMap() + for (ptr in ptrRecords) { + val instanceFqdn = ptr.target.toString() + val srv = + recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord + ?: run { + val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null + recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord + } + ?: continue + val port = srv.port + if (port <= 0) continue + + val targetFqdn = srv.target.toString() + val host = + resolveHostFromMessage(ptrMsg, targetFqdn) + ?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn) + ?: resolveHostUnicast(targetFqdn) + ?: continue + + val txtFromPtr = + recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)] + .orEmpty() + .mapNotNull { it as? TXTRecord } + val txt = + if (txtFromPtr.isNotEmpty()) { + txtFromPtr + } else { + val msg = lookupUnicastMessage(instanceFqdn, Type.TXT) + records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord } + } + val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain)) + val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName) + val lanHost = txtValue(txt, "lanHost") + val tailnetDns = txtValue(txt, "tailnetDns") + val gatewayPort = txtIntValue(txt, "gatewayPort") + val canvasPort = txtIntValue(txt, "canvasPort") + val tlsEnabled = txtBoolValue(txt, "gatewayTls") + val tlsFingerprint = txtValue(txt, "gatewayTlsSha256") + val id = stableId(instanceName, domain) + next[id] = + GatewayEndpoint( + stableId = id, + name = displayName, + host = host, + port = port, + lanHost = lanHost, + tailnetDns = tailnetDns, + gatewayPort = gatewayPort, + canvasPort = canvasPort, + tlsEnabled = tlsEnabled, + tlsFingerprintSha256 = tlsFingerprint, + ) + } + + unicastById.clear() + unicastById.putAll(next) + lastWideAreaRcode = ptrMsg.header.rcode + lastWideAreaCount = next.size + publish() + + if (next.isEmpty()) { + Log.d( + logTag, + "wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})", + ) + } + } + + private fun decodeInstanceName(instanceFqdn: String, domain: String): String { + val suffix = "${serviceType}${domain}" + val withoutSuffix = + if (instanceFqdn.endsWith(suffix)) { + instanceFqdn.removeSuffix(suffix) + } else { + instanceFqdn.substringBefore(serviceType) + } + return normalizeName(stripTrailingDot(withoutSuffix)) + } + + private fun stripTrailingDot(raw: String): String { + return raw.removeSuffix(".") + } + + private suspend fun lookupUnicastMessage(name: String, type: Int): Message? { + val query = + try { + Message.newQuery( + org.xbill.DNS.Record.newRecord( + Name.fromString(name), + type, + DClass.IN, + ), + ) + } catch (_: TextParseException) { + return null + } + + val system = queryViaSystemDns(query) + if (records(system, Section.ANSWER).any { it.type == type }) return system + + val direct = createDirectResolver() ?: return system + return try { + val msg = direct.send(query) + if (records(msg, Section.ANSWER).any { it.type == type }) msg else system + } catch (_: Throwable) { + system + } + } + + private suspend fun queryViaSystemDns(query: Message): Message? { + val network = preferredDnsNetwork() + val bytes = + try { + rawQuery(network, query.toWire()) + } catch (_: Throwable) { + return null + } + + return try { + Message(bytes) + } catch (_: IOException) { + null + } + } + + private fun records(msg: Message?, section: Int): List { + return msg?.getSectionArray(section)?.toList() ?: emptyList() + } + + private fun keyName(raw: String): String { + return raw.trim().lowercase() + } + + private fun recordsByName(msg: Message, section: Int): Map> { + val next = LinkedHashMap>() + for (r in records(msg, section)) { + val name = r.name?.toString() ?: continue + next.getOrPut(keyName(name)) { mutableListOf() }.add(r) + } + return next + } + + private fun recordByName(msg: Message, fqdn: String, type: Int): Record? { + val key = keyName(fqdn) + val byNameAnswer = recordsByName(msg, Section.ANSWER) + val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type } + if (fromAnswer != null) return fromAnswer + + val byNameAdditional = recordsByName(msg, Section.ADDITIONAL) + return byNameAdditional[key].orEmpty().firstOrNull { it.type == type } + } + + private fun resolveHostFromMessage(msg: Message?, hostname: String): String? { + val m = msg ?: return null + val key = keyName(hostname) + val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty() + val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress } + val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress } + return a.firstOrNull() ?: aaaa.firstOrNull() + } + + private fun preferredDnsNetwork(): android.net.Network? { + val cm = connectivity ?: return null + + // Prefer VPN (Tailscale) when present; otherwise use the active network. + cm.allNetworks.firstOrNull { n -> + val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false + caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) + }?.let { return it } + + return cm.activeNetwork + } + + private fun createDirectResolver(): Resolver? { + val cm = connectivity ?: return null + + val candidateNetworks = + buildList { + cm.allNetworks + .firstOrNull { n -> + val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false + caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) + }?.let(::add) + cm.activeNetwork?.let(::add) + }.distinct() + + val servers = + candidateNetworks + .asSequence() + .flatMap { n -> + cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence() + } + .distinctBy { it.hostAddress ?: it.toString() } + .toList() + if (servers.isEmpty()) return null + + return try { + val resolvers = + servers.mapNotNull { addr -> + try { + SimpleResolver().apply { + setAddress(InetSocketAddress(addr, 53)) + setTimeout(3) + } + } catch (_: Throwable) { + null + } + } + if (resolvers.isEmpty()) return null + ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) } + } catch (_: Throwable) { + null + } + } + + private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray = + suspendCancellableCoroutine { cont -> + val signal = CancellationSignal() + cont.invokeOnCancellation { signal.cancel() } + + dns.rawQuery( + network, + wireQuery, + DnsResolver.FLAG_EMPTY, + dnsExecutor, + signal, + object : DnsResolver.Callback { + override fun onAnswer(answer: ByteArray, rcode: Int) { + cont.resume(answer) + } + + override fun onError(error: DnsResolver.DnsException) { + cont.resumeWithException(error) + } + }, + ) + } + + private fun txtValue(records: List, key: String): String? { + val prefix = "$key=" + for (r in records) { + val strings: List = + try { + r.strings.mapNotNull { it as? String } + } catch (_: Throwable) { + emptyList() + } + for (s in strings) { + val trimmed = decodeDnsTxtString(s).trim() + if (trimmed.startsWith(prefix)) { + return trimmed.removePrefix(prefix).trim().ifEmpty { null } + } + } + } + return null + } + + private fun txtIntValue(records: List, key: String): Int? { + return txtValue(records, key)?.toIntOrNull() + } + + private fun txtBoolValue(records: List, key: String): Boolean { + val raw = txtValue(records, key)?.trim()?.lowercase() ?: return false + return raw == "1" || raw == "true" || raw == "yes" + } + + private fun decodeDnsTxtString(raw: String): String { + // dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes. + // Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible. + val bytes = raw.toByteArray(Charsets.ISO_8859_1) + val decoder = + Charsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + return try { + decoder.decode(ByteBuffer.wrap(bytes)).toString() + } catch (_: Throwable) { + raw + } + } + + private suspend fun resolveHostUnicast(hostname: String): String? { + val a = + records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER) + .mapNotNull { it as? ARecord } + .mapNotNull { it.address?.hostAddress } + val aaaa = + records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER) + .mapNotNull { it as? AAAARecord } + .mapNotNull { it.address?.hostAddress } + + return a.firstOrNull() ?: aaaa.firstOrNull() + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayEndpoint.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayEndpoint.kt new file mode 100644 index 00000000000..2c524cc6795 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayEndpoint.kt @@ -0,0 +1,26 @@ +package bot.molt.android.gateway + +data class GatewayEndpoint( + val stableId: String, + val name: String, + val host: String, + val port: Int, + val lanHost: String? = null, + val tailnetDns: String? = null, + val gatewayPort: Int? = null, + val canvasPort: Int? = null, + val tlsEnabled: Boolean = false, + val tlsFingerprintSha256: String? = null, +) { + companion object { + fun manual(host: String, port: Int): GatewayEndpoint = + GatewayEndpoint( + stableId = "manual|${host.lowercase()}|$port", + name = "$host:$port", + host = host, + port = port, + tlsEnabled = false, + tlsFingerprintSha256 = null, + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayProtocol.kt new file mode 100644 index 00000000000..6836331be78 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayProtocol.kt @@ -0,0 +1,3 @@ +package bot.molt.android.gateway + +const val GATEWAY_PROTOCOL_VERSION = 3 diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt new file mode 100644 index 00000000000..13074b91884 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt @@ -0,0 +1,683 @@ +package bot.molt.android.gateway + +import android.util.Log +import java.util.Locale +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import okhttp3.WebSocket +import okhttp3.WebSocketListener + +data class GatewayClientInfo( + val id: String, + val displayName: String?, + val version: String, + val platform: String, + val mode: String, + val instanceId: String?, + val deviceFamily: String?, + val modelIdentifier: String?, +) + +data class GatewayConnectOptions( + val role: String, + val scopes: List, + val caps: List, + val commands: List, + val permissions: Map, + val client: GatewayClientInfo, + val userAgent: String? = null, +) + +class GatewaySession( + private val scope: CoroutineScope, + private val identityStore: DeviceIdentityStore, + private val deviceAuthStore: DeviceAuthStore, + private val onConnected: (serverName: String?, remoteAddress: String?, mainSessionKey: String?) -> Unit, + private val onDisconnected: (message: String) -> Unit, + private val onEvent: (event: String, payloadJson: String?) -> Unit, + private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null, + private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null, +) { + data class InvokeRequest( + val id: String, + val nodeId: String, + val command: String, + val paramsJson: String?, + val timeoutMs: Long?, + ) + + data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) { + companion object { + fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null) + fun error(code: String, message: String) = + InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message)) + } + } + + data class ErrorShape(val code: String, val message: String) + + private val json = Json { ignoreUnknownKeys = true } + private val writeLock = Mutex() + private val pending = ConcurrentHashMap>() + + @Volatile private var canvasHostUrl: String? = null + @Volatile private var mainSessionKey: String? = null + + private data class DesiredConnection( + val endpoint: GatewayEndpoint, + val token: String?, + val password: String?, + val options: GatewayConnectOptions, + val tls: GatewayTlsParams?, + ) + + private var desired: DesiredConnection? = null + private var job: Job? = null + @Volatile private var currentConnection: Connection? = null + + fun connect( + endpoint: GatewayEndpoint, + token: String?, + password: String?, + options: GatewayConnectOptions, + tls: GatewayTlsParams? = null, + ) { + desired = DesiredConnection(endpoint, token, password, options, tls) + if (job == null) { + job = scope.launch(Dispatchers.IO) { runLoop() } + } + } + + fun disconnect() { + desired = null + currentConnection?.closeQuietly() + scope.launch(Dispatchers.IO) { + job?.cancelAndJoin() + job = null + canvasHostUrl = null + mainSessionKey = null + onDisconnected("Offline") + } + } + + fun reconnect() { + currentConnection?.closeQuietly() + } + + fun currentCanvasHostUrl(): String? = canvasHostUrl + fun currentMainSessionKey(): String? = mainSessionKey + + suspend fun sendNodeEvent(event: String, payloadJson: String?) { + val conn = currentConnection ?: return + val parsedPayload = payloadJson?.let { parseJsonOrNull(it) } + val params = + buildJsonObject { + put("event", JsonPrimitive(event)) + if (parsedPayload != null) { + put("payload", parsedPayload) + } else if (payloadJson != null) { + put("payloadJSON", JsonPrimitive(payloadJson)) + } else { + put("payloadJSON", JsonNull) + } + } + try { + conn.request("node.event", params, timeoutMs = 8_000) + } catch (err: Throwable) { + Log.w("MoltbotGateway", "node.event failed: ${err.message ?: err::class.java.simpleName}") + } + } + + suspend fun request(method: String, paramsJson: String?, timeoutMs: Long = 15_000): String { + val conn = currentConnection ?: throw IllegalStateException("not connected") + val params = + if (paramsJson.isNullOrBlank()) { + null + } else { + json.parseToJsonElement(paramsJson) + } + val res = conn.request(method, params, timeoutMs) + if (res.ok) return res.payloadJson ?: "" + val err = res.error + throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}") + } + + private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) + + private inner class Connection( + private val endpoint: GatewayEndpoint, + private val token: String?, + private val password: String?, + private val options: GatewayConnectOptions, + private val tls: GatewayTlsParams?, + ) { + private val connectDeferred = CompletableDeferred() + private val closedDeferred = CompletableDeferred() + private val isClosed = AtomicBoolean(false) + private val connectNonceDeferred = CompletableDeferred() + private val client: OkHttpClient = buildClient() + private var socket: WebSocket? = null + private val loggerTag = "MoltbotGateway" + + val remoteAddress: String = + if (endpoint.host.contains(":")) { + "[${endpoint.host}]:${endpoint.port}" + } else { + "${endpoint.host}:${endpoint.port}" + } + + suspend fun connect() { + val scheme = if (tls != null) "wss" else "ws" + val url = "$scheme://${endpoint.host}:${endpoint.port}" + val request = Request.Builder().url(url).build() + socket = client.newWebSocket(request, Listener()) + try { + connectDeferred.await() + } catch (err: Throwable) { + throw err + } + } + + suspend fun request(method: String, params: JsonElement?, timeoutMs: Long): RpcResponse { + val id = UUID.randomUUID().toString() + val deferred = CompletableDeferred() + pending[id] = deferred + val frame = + buildJsonObject { + put("type", JsonPrimitive("req")) + put("id", JsonPrimitive(id)) + put("method", JsonPrimitive(method)) + if (params != null) put("params", params) + } + sendJson(frame) + return try { + withTimeout(timeoutMs) { deferred.await() } + } catch (err: TimeoutCancellationException) { + pending.remove(id) + throw IllegalStateException("request timeout") + } + } + + suspend fun sendJson(obj: JsonObject) { + val jsonString = obj.toString() + writeLock.withLock { + socket?.send(jsonString) + } + } + + suspend fun awaitClose() = closedDeferred.await() + + fun closeQuietly() { + if (isClosed.compareAndSet(false, true)) { + socket?.close(1000, "bye") + socket = null + closedDeferred.complete(Unit) + } + } + + private fun buildClient(): OkHttpClient { + val builder = OkHttpClient.Builder() + val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint -> + onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint) + } + if (tlsConfig != null) { + builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager) + builder.hostnameVerifier(tlsConfig.hostnameVerifier) + } + return builder.build() + } + + private inner class Listener : WebSocketListener() { + override fun onOpen(webSocket: WebSocket, response: Response) { + scope.launch { + try { + val nonce = awaitConnectNonce() + sendConnect(nonce) + } catch (err: Throwable) { + connectDeferred.completeExceptionally(err) + closeQuietly() + } + } + } + + override fun onMessage(webSocket: WebSocket, text: String) { + scope.launch { handleMessage(text) } + } + + override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { + if (!connectDeferred.isCompleted) { + connectDeferred.completeExceptionally(t) + } + if (isClosed.compareAndSet(false, true)) { + failPending() + closedDeferred.complete(Unit) + onDisconnected("Gateway error: ${t.message ?: t::class.java.simpleName}") + } + } + + override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { + if (!connectDeferred.isCompleted) { + connectDeferred.completeExceptionally(IllegalStateException("Gateway closed: $reason")) + } + if (isClosed.compareAndSet(false, true)) { + failPending() + closedDeferred.complete(Unit) + onDisconnected("Gateway closed: $reason") + } + } + } + + private suspend fun sendConnect(connectNonce: String?) { + val identity = identityStore.loadOrCreate() + val storedToken = deviceAuthStore.loadToken(identity.deviceId, options.role) + val trimmedToken = token?.trim().orEmpty() + val authToken = if (storedToken.isNullOrBlank()) trimmedToken else storedToken + val canFallbackToShared = !storedToken.isNullOrBlank() && trimmedToken.isNotBlank() + val payload = buildConnectParams(identity, connectNonce, authToken, password?.trim()) + val res = request("connect", payload, timeoutMs = 8_000) + if (!res.ok) { + val msg = res.error?.message ?: "connect failed" + if (canFallbackToShared) { + deviceAuthStore.clearToken(identity.deviceId, options.role) + } + throw IllegalStateException(msg) + } + val payloadJson = res.payloadJson ?: throw IllegalStateException("connect failed: missing payload") + val obj = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: throw IllegalStateException("connect failed") + val serverName = obj["server"].asObjectOrNull()?.get("host").asStringOrNull() + val authObj = obj["auth"].asObjectOrNull() + val deviceToken = authObj?.get("deviceToken").asStringOrNull() + val authRole = authObj?.get("role").asStringOrNull() ?: options.role + if (!deviceToken.isNullOrBlank()) { + deviceAuthStore.saveToken(identity.deviceId, authRole, deviceToken) + } + val rawCanvas = obj["canvasHostUrl"].asStringOrNull() + canvasHostUrl = normalizeCanvasHostUrl(rawCanvas, endpoint) + val sessionDefaults = + obj["snapshot"].asObjectOrNull() + ?.get("sessionDefaults").asObjectOrNull() + mainSessionKey = sessionDefaults?.get("mainSessionKey").asStringOrNull() + onConnected(serverName, remoteAddress, mainSessionKey) + connectDeferred.complete(Unit) + } + + private fun buildConnectParams( + identity: DeviceIdentity, + connectNonce: String?, + authToken: String, + authPassword: String?, + ): JsonObject { + val client = options.client + val locale = Locale.getDefault().toLanguageTag() + val clientObj = + buildJsonObject { + put("id", JsonPrimitive(client.id)) + client.displayName?.let { put("displayName", JsonPrimitive(it)) } + put("version", JsonPrimitive(client.version)) + put("platform", JsonPrimitive(client.platform)) + put("mode", JsonPrimitive(client.mode)) + client.instanceId?.let { put("instanceId", JsonPrimitive(it)) } + client.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) } + client.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) } + } + + val password = authPassword?.trim().orEmpty() + val authJson = + when { + authToken.isNotEmpty() -> + buildJsonObject { + put("token", JsonPrimitive(authToken)) + } + password.isNotEmpty() -> + buildJsonObject { + put("password", JsonPrimitive(password)) + } + else -> null + } + + val signedAtMs = System.currentTimeMillis() + val payload = + buildDeviceAuthPayload( + deviceId = identity.deviceId, + clientId = client.id, + clientMode = client.mode, + role = options.role, + scopes = options.scopes, + signedAtMs = signedAtMs, + token = if (authToken.isNotEmpty()) authToken else null, + nonce = connectNonce, + ) + val signature = identityStore.signPayload(payload, identity) + val publicKey = identityStore.publicKeyBase64Url(identity) + val deviceJson = + if (!signature.isNullOrBlank() && !publicKey.isNullOrBlank()) { + buildJsonObject { + put("id", JsonPrimitive(identity.deviceId)) + put("publicKey", JsonPrimitive(publicKey)) + put("signature", JsonPrimitive(signature)) + put("signedAt", JsonPrimitive(signedAtMs)) + if (!connectNonce.isNullOrBlank()) { + put("nonce", JsonPrimitive(connectNonce)) + } + } + } else { + null + } + + return buildJsonObject { + put("minProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) + put("maxProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) + put("client", clientObj) + if (options.caps.isNotEmpty()) put("caps", JsonArray(options.caps.map(::JsonPrimitive))) + if (options.commands.isNotEmpty()) put("commands", JsonArray(options.commands.map(::JsonPrimitive))) + if (options.permissions.isNotEmpty()) { + put( + "permissions", + buildJsonObject { + options.permissions.forEach { (key, value) -> + put(key, JsonPrimitive(value)) + } + }, + ) + } + put("role", JsonPrimitive(options.role)) + if (options.scopes.isNotEmpty()) put("scopes", JsonArray(options.scopes.map(::JsonPrimitive))) + authJson?.let { put("auth", it) } + deviceJson?.let { put("device", it) } + put("locale", JsonPrimitive(locale)) + options.userAgent?.trim()?.takeIf { it.isNotEmpty() }?.let { + put("userAgent", JsonPrimitive(it)) + } + } + } + + private suspend fun handleMessage(text: String) { + val frame = json.parseToJsonElement(text).asObjectOrNull() ?: return + when (frame["type"].asStringOrNull()) { + "res" -> handleResponse(frame) + "event" -> handleEvent(frame) + } + } + + private fun handleResponse(frame: JsonObject) { + val id = frame["id"].asStringOrNull() ?: return + val ok = frame["ok"].asBooleanOrNull() ?: false + val payloadJson = frame["payload"]?.let { payload -> payload.toString() } + val error = + frame["error"]?.asObjectOrNull()?.let { obj -> + val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE" + val msg = obj["message"].asStringOrNull() ?: "request failed" + ErrorShape(code, msg) + } + pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error)) + } + + private fun handleEvent(frame: JsonObject) { + val event = frame["event"].asStringOrNull() ?: return + val payloadJson = + frame["payload"]?.let { it.toString() } ?: frame["payloadJSON"].asStringOrNull() + if (event == "connect.challenge") { + val nonce = extractConnectNonce(payloadJson) + if (!connectNonceDeferred.isCompleted) { + connectNonceDeferred.complete(nonce) + } + return + } + if (event == "node.invoke.request" && payloadJson != null && onInvoke != null) { + handleInvokeEvent(payloadJson) + return + } + onEvent(event, payloadJson) + } + + private suspend fun awaitConnectNonce(): String? { + if (isLoopbackHost(endpoint.host)) return null + return try { + withTimeout(2_000) { connectNonceDeferred.await() } + } catch (_: Throwable) { + null + } + } + + private fun extractConnectNonce(payloadJson: String?): String? { + if (payloadJson.isNullOrBlank()) return null + val obj = parseJsonOrNull(payloadJson)?.asObjectOrNull() ?: return null + return obj["nonce"].asStringOrNull() + } + + private fun handleInvokeEvent(payloadJson: String) { + val payload = + try { + json.parseToJsonElement(payloadJson).asObjectOrNull() + } catch (_: Throwable) { + null + } ?: return + val id = payload["id"].asStringOrNull() ?: return + val nodeId = payload["nodeId"].asStringOrNull() ?: return + val command = payload["command"].asStringOrNull() ?: return + val params = + payload["paramsJSON"].asStringOrNull() + ?: payload["params"]?.let { value -> if (value is JsonNull) null else value.toString() } + val timeoutMs = payload["timeoutMs"].asLongOrNull() + scope.launch { + val result = + try { + onInvoke?.invoke(InvokeRequest(id, nodeId, command, params, timeoutMs)) + ?: InvokeResult.error("UNAVAILABLE", "invoke handler missing") + } catch (err: Throwable) { + invokeErrorFromThrowable(err) + } + sendInvokeResult(id, nodeId, result) + } + } + + private suspend fun sendInvokeResult(id: String, nodeId: String, result: InvokeResult) { + val parsedPayload = result.payloadJson?.let { parseJsonOrNull(it) } + val params = + buildJsonObject { + put("id", JsonPrimitive(id)) + put("nodeId", JsonPrimitive(nodeId)) + put("ok", JsonPrimitive(result.ok)) + if (parsedPayload != null) { + put("payload", parsedPayload) + } else if (result.payloadJson != null) { + put("payloadJSON", JsonPrimitive(result.payloadJson)) + } + result.error?.let { err -> + put( + "error", + buildJsonObject { + put("code", JsonPrimitive(err.code)) + put("message", JsonPrimitive(err.message)) + }, + ) + } + } + try { + request("node.invoke.result", params, timeoutMs = 15_000) + } catch (err: Throwable) { + Log.w(loggerTag, "node.invoke.result failed: ${err.message ?: err::class.java.simpleName}") + } + } + + private fun invokeErrorFromThrowable(err: Throwable): InvokeResult { + val msg = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: err::class.java.simpleName + val parts = msg.split(":", limit = 2) + if (parts.size == 2) { + val code = parts[0].trim() + val rest = parts[1].trim() + if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) { + return InvokeResult.error(code = code, message = rest.ifEmpty { msg }) + } + } + return InvokeResult.error(code = "UNAVAILABLE", message = msg) + } + + private fun failPending() { + for ((_, waiter) in pending) { + waiter.cancel() + } + pending.clear() + } + } + + private suspend fun runLoop() { + var attempt = 0 + while (scope.isActive) { + val target = desired + if (target == null) { + currentConnection?.closeQuietly() + currentConnection = null + delay(250) + continue + } + + try { + onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…") + connectOnce(target) + attempt = 0 + } catch (err: Throwable) { + attempt += 1 + onDisconnected("Gateway error: ${err.message ?: err::class.java.simpleName}") + val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong()) + delay(sleepMs) + } + } + } + + private suspend fun connectOnce(target: DesiredConnection) = withContext(Dispatchers.IO) { + val conn = Connection(target.endpoint, target.token, target.password, target.options, target.tls) + currentConnection = conn + try { + conn.connect() + conn.awaitClose() + } finally { + currentConnection = null + canvasHostUrl = null + mainSessionKey = null + } + } + + private fun buildDeviceAuthPayload( + deviceId: String, + clientId: String, + clientMode: String, + role: String, + scopes: List, + signedAtMs: Long, + token: String?, + nonce: String?, + ): String { + val scopeString = scopes.joinToString(",") + val authToken = token.orEmpty() + val version = if (nonce.isNullOrBlank()) "v1" else "v2" + val parts = + mutableListOf( + version, + deviceId, + clientId, + clientMode, + role, + scopeString, + signedAtMs.toString(), + authToken, + ) + if (!nonce.isNullOrBlank()) { + parts.add(nonce) + } + return parts.joinToString("|") + } + + private fun normalizeCanvasHostUrl(raw: String?, endpoint: GatewayEndpoint): String? { + val trimmed = raw?.trim().orEmpty() + val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { java.net.URI(it) }.getOrNull() } + val host = parsed?.host?.trim().orEmpty() + val port = parsed?.port ?: -1 + val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" } + + if (trimmed.isNotBlank() && !isLoopbackHost(host)) { + return trimmed + } + + val fallbackHost = + endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() } + ?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() } + ?: endpoint.host.trim() + if (fallbackHost.isEmpty()) return trimmed.ifBlank { null } + + val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793 + val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost + return "$scheme://$formattedHost:$fallbackPort" + } + + private fun isLoopbackHost(raw: String?): Boolean { + val host = raw?.trim()?.lowercase().orEmpty() + if (host.isEmpty()) return false + if (host == "localhost") return true + if (host == "::1") return true + if (host == "0.0.0.0" || host == "::") return true + return host.startsWith("127.") + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + when (this) { + is JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun JsonElement?.asBooleanOrNull(): Boolean? = + when (this) { + is JsonPrimitive -> { + val c = content.trim() + when { + c.equals("true", ignoreCase = true) -> true + c.equals("false", ignoreCase = true) -> false + else -> null + } + } + else -> null + } + +private fun JsonElement?.asLongOrNull(): Long? = + when (this) { + is JsonPrimitive -> content.toLongOrNull() + else -> null + } + +private fun parseJsonOrNull(payload: String): JsonElement? { + val trimmed = payload.trim() + if (trimmed.isEmpty()) return null + return try { + Json.parseToJsonElement(trimmed) + } catch (_: Throwable) { + null + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayTls.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayTls.kt new file mode 100644 index 00000000000..673d60c8fee --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewayTls.kt @@ -0,0 +1,90 @@ +package bot.molt.android.gateway + +import android.annotation.SuppressLint +import java.security.MessageDigest +import java.security.SecureRandom +import java.security.cert.CertificateException +import java.security.cert.X509Certificate +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLSocketFactory +import javax.net.ssl.TrustManagerFactory +import javax.net.ssl.X509TrustManager + +data class GatewayTlsParams( + val required: Boolean, + val expectedFingerprint: String?, + val allowTOFU: Boolean, + val stableId: String, +) + +data class GatewayTlsConfig( + val sslSocketFactory: SSLSocketFactory, + val trustManager: X509TrustManager, + val hostnameVerifier: HostnameVerifier, +) + +fun buildGatewayTlsConfig( + params: GatewayTlsParams?, + onStore: ((String) -> Unit)? = null, +): GatewayTlsConfig? { + if (params == null) return null + val expected = params.expectedFingerprint?.let(::normalizeFingerprint) + val defaultTrust = defaultTrustManager() + @SuppressLint("CustomX509TrustManager") + val trustManager = + object : X509TrustManager { + override fun checkClientTrusted(chain: Array, authType: String) { + defaultTrust.checkClientTrusted(chain, authType) + } + + override fun checkServerTrusted(chain: Array, authType: String) { + if (chain.isEmpty()) throw CertificateException("empty certificate chain") + val fingerprint = sha256Hex(chain[0].encoded) + if (expected != null) { + if (fingerprint != expected) { + throw CertificateException("gateway TLS fingerprint mismatch") + } + return + } + if (params.allowTOFU) { + onStore?.invoke(fingerprint) + return + } + defaultTrust.checkServerTrusted(chain, authType) + } + + override fun getAcceptedIssuers(): Array = defaultTrust.acceptedIssuers + } + + val context = SSLContext.getInstance("TLS") + context.init(null, arrayOf(trustManager), SecureRandom()) + return GatewayTlsConfig( + sslSocketFactory = context.socketFactory, + trustManager = trustManager, + hostnameVerifier = HostnameVerifier { _, _ -> true }, + ) +} + +private fun defaultTrustManager(): X509TrustManager { + val factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) + factory.init(null as java.security.KeyStore?) + val trust = + factory.trustManagers.firstOrNull { it is X509TrustManager } as? X509TrustManager + return trust ?: throw IllegalStateException("No default X509TrustManager found") +} + +private fun sha256Hex(data: ByteArray): String { + val digest = MessageDigest.getInstance("SHA-256").digest(data) + val out = StringBuilder(digest.size * 2) + for (byte in digest) { + out.append(String.format("%02x", byte)) + } + return out.toString() +} + +private fun normalizeFingerprint(raw: String): String { + val stripped = raw.trim() + .replace(Regex("^sha-?256\\s*:?\\s*", RegexOption.IGNORE_CASE), "") + return stripped.lowercase().filter { it in '0'..'9' || it in 'a'..'f' } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/node/CameraCaptureManager.kt b/apps/android/app/src/main/java/bot/molt/android/node/CameraCaptureManager.kt new file mode 100644 index 00000000000..cb15a3915b2 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/CameraCaptureManager.kt @@ -0,0 +1,316 @@ +package bot.molt.android.node + +import android.Manifest +import android.content.Context +import android.annotation.SuppressLint +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.util.Base64 +import android.content.pm.PackageManager +import androidx.exifinterface.media.ExifInterface +import androidx.lifecycle.LifecycleOwner +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.video.FileOutputOptions +import androidx.camera.video.Recorder +import androidx.camera.video.Recording +import androidx.camera.video.VideoCapture +import androidx.camera.video.VideoRecordEvent +import androidx.core.content.ContextCompat +import androidx.core.content.ContextCompat.checkSelfPermission +import androidx.core.graphics.scale +import bot.molt.android.PermissionRequester +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.concurrent.Executor +import kotlin.math.roundToInt +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +class CameraCaptureManager(private val context: Context) { + data class Payload(val payloadJson: String) + + @Volatile private var lifecycleOwner: LifecycleOwner? = null + @Volatile private var permissionRequester: PermissionRequester? = null + + fun attachLifecycleOwner(owner: LifecycleOwner) { + lifecycleOwner = owner + } + + fun attachPermissionRequester(requester: PermissionRequester) { + permissionRequester = requester + } + + private suspend fun ensureCameraPermission() { + val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA)) + if (results[Manifest.permission.CAMERA] != true) { + throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") + } + } + + private suspend fun ensureMicPermission() { + val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = permissionRequester + ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO)) + if (results[Manifest.permission.RECORD_AUDIO] != true) { + throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } + } + + suspend fun snap(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + ensureCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0) + val maxWidth = parseMaxWidth(paramsJson) + + val provider = context.cameraProvider() + val capture = ImageCapture.Builder().build() + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, capture) + + val (bytes, orientation) = capture.takeJpegWithExif(context.mainExecutor()) + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + ?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image") + val rotated = rotateBitmapByExif(decoded, orientation) + val scaled = + if (maxWidth != null && maxWidth > 0 && rotated.width > maxWidth) { + val h = + (rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble())) + .toInt() + .coerceAtLeast(1) + rotated.scale(maxWidth, h) + } else { + rotated + } + + val maxPayloadBytes = 5 * 1024 * 1024 + // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit). + val maxEncodedBytes = (maxPayloadBytes / 4) * 3 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = scaled.width, + initialHeight = scaled.height, + startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100), + maxBytes = maxEncodedBytes, + encode = { width, height, q -> + val bitmap = + if (width == scaled.width && height == scaled.height) { + scaled + } else { + scaled.scale(width, height) + } + val out = ByteArrayOutputStream() + if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) { + if (bitmap !== scaled) bitmap.recycle() + throw IllegalStateException("UNAVAILABLE: failed to encode JPEG") + } + if (bitmap !== scaled) { + bitmap.recycle() + } + out.toByteArray() + }, + ) + val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP) + Payload( + """{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""", + ) + } + + @SuppressLint("MissingPermission") + suspend fun clip(paramsJson: String?): Payload = + withContext(Dispatchers.Main) { + ensureCameraPermission() + val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") + val facing = parseFacing(paramsJson) ?: "front" + val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000) + val includeAudio = parseIncludeAudio(paramsJson) ?: true + if (includeAudio) ensureMicPermission() + + val provider = context.cameraProvider() + val recorder = Recorder.Builder().build() + val videoCapture = VideoCapture.withOutput(recorder) + val selector = + if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA + + provider.unbindAll() + provider.bindToLifecycle(owner, selector, videoCapture) + + val file = File.createTempFile("moltbot-clip-", ".mp4") + val outputOptions = FileOutputOptions.Builder(file).build() + + val finalized = kotlinx.coroutines.CompletableDeferred() + val recording: Recording = + videoCapture.output + .prepareRecording(context, outputOptions) + .apply { + if (includeAudio) withAudioEnabled() + } + .start(context.mainExecutor()) { event -> + if (event is VideoRecordEvent.Finalize) { + finalized.complete(event) + } + } + + try { + kotlinx.coroutines.delay(durationMs.toLong()) + } finally { + recording.stop() + } + + val finalizeEvent = + try { + withTimeout(10_000) { finalized.await() } + } catch (err: Throwable) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out") + } + if (finalizeEvent.hasError()) { + file.delete() + throw IllegalStateException("UNAVAILABLE: camera clip failed") + } + + val bytes = file.readBytes() + file.delete() + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + Payload( + """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"hasAudio":${includeAudio}}""", + ) + } + + private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap { + val matrix = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { + matrix.postRotate(90f) + matrix.postScale(-1f, 1f) + } + ExifInterface.ORIENTATION_TRANSVERSE -> { + matrix.postRotate(-90f) + matrix.postScale(-1f, 1f) + } + else -> return bitmap + } + val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) + if (rotated !== bitmap) { + bitmap.recycle() + } + return rotated + } + + private fun parseFacing(paramsJson: String?): String? = + when { + paramsJson?.contains("\"front\"") == true -> "front" + paramsJson?.contains("\"back\"") == true -> "back" + else -> null + } + + private fun parseQuality(paramsJson: String?): Double? = + parseNumber(paramsJson, key = "quality")?.toDoubleOrNull() + + private fun parseMaxWidth(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "maxWidth")?.toIntOrNull() + + private fun parseDurationMs(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() + + private fun parseIncludeAudio(paramsJson: String?): Boolean? { + val raw = paramsJson ?: return null + val key = "\"includeAudio\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return when { + tail.startsWith("true") -> true + tail.startsWith("false") -> false + else -> null + } + } + + private fun parseNumber(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return tail.takeWhile { it.isDigit() || it == '.' } + } + + private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this) +} + +private suspend fun Context.cameraProvider(): ProcessCameraProvider = + suspendCancellableCoroutine { cont -> + val future = ProcessCameraProvider.getInstance(this) + future.addListener( + { + try { + cont.resume(future.get()) + } catch (e: Exception) { + cont.resumeWithException(e) + } + }, + ContextCompat.getMainExecutor(this), + ) + } + +/** Returns (jpegBytes, exifOrientation) so caller can rotate the decoded bitmap. */ +private suspend fun ImageCapture.takeJpegWithExif(executor: Executor): Pair = + suspendCancellableCoroutine { cont -> + val file = File.createTempFile("moltbot-snap-", ".jpg") + val options = ImageCapture.OutputFileOptions.Builder(file).build() + takePicture( + options, + executor, + object : ImageCapture.OnImageSavedCallback { + override fun onError(exception: ImageCaptureException) { + file.delete() + cont.resumeWithException(exception) + } + + override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { + try { + val exif = ExifInterface(file.absolutePath) + val orientation = exif.getAttributeInt( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_NORMAL, + ) + val bytes = file.readBytes() + cont.resume(Pair(bytes, orientation)) + } catch (e: Exception) { + cont.resumeWithException(e) + } finally { + file.delete() + } + } + }, + ) + } diff --git a/apps/android/app/src/main/java/bot/molt/android/node/CanvasController.kt b/apps/android/app/src/main/java/bot/molt/android/node/CanvasController.kt new file mode 100644 index 00000000000..4d33ed0a655 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/CanvasController.kt @@ -0,0 +1,264 @@ +package bot.molt.android.node + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.os.Looper +import android.util.Log +import android.webkit.WebView +import androidx.core.graphics.createBitmap +import androidx.core.graphics.scale +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import android.util.Base64 +import org.json.JSONObject +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import bot.molt.android.BuildConfig +import kotlin.coroutines.resume + +class CanvasController { + enum class SnapshotFormat(val rawValue: String) { + Png("png"), + Jpeg("jpeg"), + } + + @Volatile private var webView: WebView? = null + @Volatile private var url: String? = null + @Volatile private var debugStatusEnabled: Boolean = false + @Volatile private var debugStatusTitle: String? = null + @Volatile private var debugStatusSubtitle: String? = null + + private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html" + + private fun clampJpegQuality(quality: Double?): Int { + val q = (quality ?: 0.82).coerceIn(0.1, 1.0) + return (q * 100.0).toInt().coerceIn(1, 100) + } + + fun attach(webView: WebView) { + this.webView = webView + reload() + applyDebugStatus() + } + + fun navigate(url: String) { + val trimmed = url.trim() + this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed + reload() + } + + fun currentUrl(): String? = url + + fun isDefaultCanvas(): Boolean = url == null + + fun setDebugStatusEnabled(enabled: Boolean) { + debugStatusEnabled = enabled + applyDebugStatus() + } + + fun setDebugStatus(title: String?, subtitle: String?) { + debugStatusTitle = title + debugStatusSubtitle = subtitle + applyDebugStatus() + } + + fun onPageFinished() { + applyDebugStatus() + } + + private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) { + val wv = webView ?: return + if (Looper.myLooper() == Looper.getMainLooper()) { + block(wv) + } else { + wv.post { block(wv) } + } + } + + private fun reload() { + val currentUrl = url + withWebViewOnMain { wv -> + if (currentUrl == null) { + if (BuildConfig.DEBUG) { + Log.d("MoltbotCanvas", "load scaffold: $scaffoldAssetUrl") + } + wv.loadUrl(scaffoldAssetUrl) + } else { + if (BuildConfig.DEBUG) { + Log.d("MoltbotCanvas", "load url: $currentUrl") + } + wv.loadUrl(currentUrl) + } + } + } + + private fun applyDebugStatus() { + val enabled = debugStatusEnabled + val title = debugStatusTitle + val subtitle = debugStatusSubtitle + withWebViewOnMain { wv -> + val titleJs = title?.let { JSONObject.quote(it) } ?: "null" + val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null" + val js = """ + (() => { + try { + const api = globalThis.__moltbot; + if (!api) return; + if (typeof api.setDebugStatusEnabled === 'function') { + api.setDebugStatusEnabled(${if (enabled) "true" else "false"}); + } + if (!${if (enabled) "true" else "false"}) return; + if (typeof api.setStatus === 'function') { + api.setStatus($titleJs, $subtitleJs); + } + } catch (_) {} + })(); + """.trimIndent() + wv.evaluateJavascript(js, null) + } + } + + suspend fun eval(javaScript: String): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + suspendCancellableCoroutine { cont -> + wv.evaluateJavascript(javaScript) { result -> + cont.resume(result ?: "") + } + } + } + + suspend fun snapshotPngBase64(maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + bmp.scale(maxWidth, h) + } else { + bmp + } + + val out = ByteArrayOutputStream() + scaled.compress(Bitmap.CompressFormat.PNG, 100, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + + suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String = + withContext(Dispatchers.Main) { + val wv = webView ?: throw IllegalStateException("no webview") + val bmp = wv.captureBitmap() + val scaled = + if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { + val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) + bmp.scale(maxWidth, h) + } else { + bmp + } + + val out = ByteArrayOutputStream() + val (compressFormat, compressQuality) = + when (format) { + SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100 + SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality) + } + scaled.compress(compressFormat, compressQuality, out) + Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) + } + + private suspend fun WebView.captureBitmap(): Bitmap = + suspendCancellableCoroutine { cont -> + val width = width.coerceAtLeast(1) + val height = height.coerceAtLeast(1) + val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888) + + // WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable + // cross-version snapshot for this lightweight "canvas" use-case. + draw(Canvas(bitmap)) + cont.resume(bitmap) + } + + companion object { + data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?) + + fun parseNavigateUrl(paramsJson: String?): String { + val obj = parseParamsObject(paramsJson) ?: return "" + return obj.string("url").trim() + } + + fun parseEvalJs(paramsJson: String?): String? { + val obj = parseParamsObject(paramsJson) ?: return null + val js = obj.string("javaScript").trim() + return js.takeIf { it.isNotBlank() } + } + + fun parseSnapshotMaxWidth(paramsJson: String?): Int? { + val obj = parseParamsObject(paramsJson) ?: return null + if (!obj.containsKey("maxWidth")) return null + val width = obj.int("maxWidth") ?: 0 + return width.takeIf { it > 0 } + } + + fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat { + val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg + val raw = obj.string("format").trim().lowercase() + return when (raw) { + "png" -> SnapshotFormat.Png + "jpeg", "jpg" -> SnapshotFormat.Jpeg + "" -> SnapshotFormat.Jpeg + else -> SnapshotFormat.Jpeg + } + } + + fun parseSnapshotQuality(paramsJson: String?): Double? { + val obj = parseParamsObject(paramsJson) ?: return null + if (!obj.containsKey("quality")) return null + val q = obj.double("quality") ?: Double.NaN + if (!q.isFinite()) return null + return q.coerceIn(0.1, 1.0) + } + + fun parseSnapshotParams(paramsJson: String?): SnapshotParams { + return SnapshotParams( + format = parseSnapshotFormat(paramsJson), + quality = parseSnapshotQuality(paramsJson), + maxWidth = parseSnapshotMaxWidth(paramsJson), + ) + } + + private val json = Json { ignoreUnknownKeys = true } + + private fun parseParamsObject(paramsJson: String?): JsonObject? { + val raw = paramsJson?.trim().orEmpty() + if (raw.isEmpty()) return null + return try { + json.parseToJsonElement(raw).asObjectOrNull() + } catch (_: Throwable) { + null + } + } + + private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + + private fun JsonObject.string(key: String): String { + val prim = this[key] as? JsonPrimitive ?: return "" + val raw = prim.content + return raw.takeIf { it != "null" }.orEmpty() + } + + private fun JsonObject.int(key: String): Int? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toIntOrNull() + } + + private fun JsonObject.double(key: String): Double? { + val prim = this[key] as? JsonPrimitive ?: return null + return prim.content.toDoubleOrNull() + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/node/JpegSizeLimiter.kt b/apps/android/app/src/main/java/bot/molt/android/node/JpegSizeLimiter.kt new file mode 100644 index 00000000000..8fb6c35d406 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/JpegSizeLimiter.kt @@ -0,0 +1,61 @@ +package bot.molt.android.node + +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +internal data class JpegSizeLimiterResult( + val bytes: ByteArray, + val width: Int, + val height: Int, + val quality: Int, +) + +internal object JpegSizeLimiter { + fun compressToLimit( + initialWidth: Int, + initialHeight: Int, + startQuality: Int, + maxBytes: Int, + minQuality: Int = 20, + minSize: Int = 256, + scaleStep: Double = 0.85, + maxScaleAttempts: Int = 6, + maxQualityAttempts: Int = 6, + encode: (width: Int, height: Int, quality: Int) -> ByteArray, + ): JpegSizeLimiterResult { + require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" } + require(maxBytes > 0) { "Invalid maxBytes" } + + var width = initialWidth + var height = initialHeight + val clampedStartQuality = startQuality.coerceIn(minQuality, 100) + var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality) + if (best.bytes.size <= maxBytes) return best + + repeat(maxScaleAttempts) { + var quality = clampedStartQuality + repeat(maxQualityAttempts) { + val bytes = encode(width, height, quality) + best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality) + if (bytes.size <= maxBytes) return best + if (quality <= minQuality) return@repeat + quality = max(minQuality, (quality * 0.75).roundToInt()) + } + + val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0) + val nextScale = max(scaleStep, minScale) + val nextWidth = max(minSize, (width * nextScale).roundToInt()) + val nextHeight = max(minSize, (height * nextScale).roundToInt()) + if (nextWidth == width && nextHeight == height) return@repeat + width = min(nextWidth, width) + height = min(nextHeight, height) + } + + if (best.bytes.size > maxBytes) { + throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes") + } + + return best + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/node/LocationCaptureManager.kt b/apps/android/app/src/main/java/bot/molt/android/node/LocationCaptureManager.kt new file mode 100644 index 00000000000..c56eee03abe --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/LocationCaptureManager.kt @@ -0,0 +1,117 @@ +package bot.molt.android.node + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.location.Location +import android.location.LocationManager +import android.os.CancellationSignal +import androidx.core.content.ContextCompat +import java.time.Instant +import java.time.format.DateTimeFormatter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlinx.coroutines.suspendCancellableCoroutine + +class LocationCaptureManager(private val context: Context) { + data class Payload(val payloadJson: String) + + suspend fun getLocation( + desiredProviders: List, + maxAgeMs: Long?, + timeoutMs: Long, + isPrecise: Boolean, + ): Payload = + withContext(Dispatchers.Main) { + val manager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager + if (!manager.isProviderEnabled(LocationManager.GPS_PROVIDER) && + !manager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) + ) { + throw IllegalStateException("LOCATION_UNAVAILABLE: no location providers enabled") + } + + val cached = bestLastKnown(manager, desiredProviders, maxAgeMs) + val location = + cached ?: requestCurrent(manager, desiredProviders, timeoutMs) + + val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(location.time)) + val source = location.provider + val altitudeMeters = if (location.hasAltitude()) location.altitude else null + val speedMps = if (location.hasSpeed()) location.speed.toDouble() else null + val headingDeg = if (location.hasBearing()) location.bearing.toDouble() else null + Payload( + buildString { + append("{\"lat\":") + append(location.latitude) + append(",\"lon\":") + append(location.longitude) + append(",\"accuracyMeters\":") + append(location.accuracy.toDouble()) + if (altitudeMeters != null) append(",\"altitudeMeters\":").append(altitudeMeters) + if (speedMps != null) append(",\"speedMps\":").append(speedMps) + if (headingDeg != null) append(",\"headingDeg\":").append(headingDeg) + append(",\"timestamp\":\"").append(timestamp).append('"') + append(",\"isPrecise\":").append(isPrecise) + append(",\"source\":\"").append(source).append('"') + append('}') + }, + ) + } + + private fun bestLastKnown( + manager: LocationManager, + providers: List, + maxAgeMs: Long?, + ): Location? { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!fineOk && !coarseOk) { + throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") + } + val now = System.currentTimeMillis() + val candidates = + providers.mapNotNull { provider -> manager.getLastKnownLocation(provider) } + val freshest = candidates.maxByOrNull { it.time } ?: return null + if (maxAgeMs != null && now - freshest.time > maxAgeMs) return null + return freshest + } + + private suspend fun requestCurrent( + manager: LocationManager, + providers: List, + timeoutMs: Long, + ): Location { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!fineOk && !coarseOk) { + throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") + } + val resolved = + providers.firstOrNull { manager.isProviderEnabled(it) } + ?: throw IllegalStateException("LOCATION_UNAVAILABLE: no providers available") + return withTimeout(timeoutMs.coerceAtLeast(1)) { + suspendCancellableCoroutine { cont -> + val signal = CancellationSignal() + cont.invokeOnCancellation { signal.cancel() } + manager.getCurrentLocation(resolved, signal, context.mainExecutor) { location -> + if (location != null) { + cont.resume(location) + } else { + cont.resumeWithException(IllegalStateException("LOCATION_UNAVAILABLE: no fix")) + } + } + } + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/node/ScreenRecordManager.kt b/apps/android/app/src/main/java/bot/molt/android/node/ScreenRecordManager.kt new file mode 100644 index 00000000000..0e785c2454f --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/ScreenRecordManager.kt @@ -0,0 +1,199 @@ +package bot.molt.android.node + +import android.content.Context +import android.hardware.display.DisplayManager +import android.media.MediaRecorder +import android.media.projection.MediaProjectionManager +import android.os.Build +import android.util.Base64 +import bot.molt.android.ScreenCaptureRequester +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import java.io.File +import kotlin.math.roundToInt + +class ScreenRecordManager(private val context: Context) { + data class Payload(val payloadJson: String) + + @Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null + @Volatile private var permissionRequester: bot.molt.android.PermissionRequester? = null + + fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) { + screenCaptureRequester = requester + } + + fun attachPermissionRequester(requester: bot.molt.android.PermissionRequester) { + permissionRequester = requester + } + + suspend fun record(paramsJson: String?): Payload = + withContext(Dispatchers.Default) { + val requester = + screenCaptureRequester + ?: throw IllegalStateException( + "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", + ) + + val durationMs = (parseDurationMs(paramsJson) ?: 10_000).coerceIn(250, 60_000) + val fps = (parseFps(paramsJson) ?: 10.0).coerceIn(1.0, 60.0) + val fpsInt = fps.roundToInt().coerceIn(1, 60) + val screenIndex = parseScreenIndex(paramsJson) + val includeAudio = parseIncludeAudio(paramsJson) ?: true + val format = parseString(paramsJson, key = "format") + if (format != null && format.lowercase() != "mp4") { + throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4") + } + if (screenIndex != null && screenIndex != 0) { + throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android") + } + + val capture = requester.requestCapture() + ?: throw IllegalStateException( + "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", + ) + + val mgr = + context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager + val projection = mgr.getMediaProjection(capture.resultCode, capture.data) + ?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable") + + val metrics = context.resources.displayMetrics + val width = metrics.widthPixels + val height = metrics.heightPixels + val densityDpi = metrics.densityDpi + + val file = File.createTempFile("moltbot-screen-", ".mp4") + if (includeAudio) ensureMicPermission() + + val recorder = createMediaRecorder() + var virtualDisplay: android.hardware.display.VirtualDisplay? = null + try { + if (includeAudio) { + recorder.setAudioSource(MediaRecorder.AudioSource.MIC) + } + recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE) + recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) + recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264) + if (includeAudio) { + recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC) + recorder.setAudioChannels(1) + recorder.setAudioSamplingRate(44_100) + recorder.setAudioEncodingBitRate(96_000) + } + recorder.setVideoSize(width, height) + recorder.setVideoFrameRate(fpsInt) + recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt)) + recorder.setOutputFile(file.absolutePath) + recorder.prepare() + + val surface = recorder.surface + virtualDisplay = + projection.createVirtualDisplay( + "moltbot-screen", + width, + height, + densityDpi, + DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, + surface, + null, + null, + ) + + recorder.start() + delay(durationMs.toLong()) + } finally { + try { + recorder.stop() + } catch (_: Throwable) { + // ignore + } + recorder.reset() + recorder.release() + virtualDisplay?.release() + projection.stop() + } + + val bytes = withContext(Dispatchers.IO) { file.readBytes() } + file.delete() + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + Payload( + """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""", + ) + } + + private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context) + + private suspend fun ensureMicPermission() { + val granted = + androidx.core.content.ContextCompat.checkSelfPermission( + context, + android.Manifest.permission.RECORD_AUDIO, + ) == android.content.pm.PackageManager.PERMISSION_GRANTED + if (granted) return + + val requester = + permissionRequester + ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO)) + if (results[android.Manifest.permission.RECORD_AUDIO] != true) { + throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") + } + } + + private fun parseDurationMs(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() + + private fun parseFps(paramsJson: String?): Double? = + parseNumber(paramsJson, key = "fps")?.toDoubleOrNull() + + private fun parseScreenIndex(paramsJson: String?): Int? = + parseNumber(paramsJson, key = "screenIndex")?.toIntOrNull() + + private fun parseIncludeAudio(paramsJson: String?): Boolean? { + val raw = paramsJson ?: return null + val key = "\"includeAudio\"" + val idx = raw.indexOf(key) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + key.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return when { + tail.startsWith("true") -> true + tail.startsWith("false") -> false + else -> null + } + } + + private fun parseNumber(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + return tail.takeWhile { it.isDigit() || it == '.' || it == '-' } + } + + private fun parseString(paramsJson: String?, key: String): String? { + val raw = paramsJson ?: return null + val needle = "\"$key\"" + val idx = raw.indexOf(needle) + if (idx < 0) return null + val colon = raw.indexOf(':', idx + needle.length) + if (colon < 0) return null + val tail = raw.substring(colon + 1).trimStart() + if (!tail.startsWith('\"')) return null + val rest = tail.drop(1) + val end = rest.indexOf('\"') + if (end < 0) return null + return rest.substring(0, end) + } + + private fun estimateBitrate(width: Int, height: Int, fps: Int): Int { + val pixels = width.toLong() * height.toLong() + val raw = (pixels * fps.toLong() * 2L).toInt() + return raw.coerceIn(1_000_000, 12_000_000) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/node/SmsManager.kt b/apps/android/app/src/main/java/bot/molt/android/node/SmsManager.kt new file mode 100644 index 00000000000..0314ee1a788 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/node/SmsManager.kt @@ -0,0 +1,230 @@ +package bot.molt.android.node + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.telephony.SmsManager as AndroidSmsManager +import androidx.core.content.ContextCompat +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.encodeToString +import bot.molt.android.PermissionRequester + +/** + * Sends SMS messages via the Android SMS API. + * Requires SEND_SMS permission to be granted. + */ +class SmsManager(private val context: Context) { + + private val json = JsonConfig + @Volatile private var permissionRequester: PermissionRequester? = null + + data class SendResult( + val ok: Boolean, + val to: String, + val message: String?, + val error: String? = null, + val payloadJson: String, + ) + + internal data class ParsedParams( + val to: String, + val message: String, + ) + + internal sealed class ParseResult { + data class Ok(val params: ParsedParams) : ParseResult() + data class Error( + val error: String, + val to: String = "", + val message: String? = null, + ) : ParseResult() + } + + internal data class SendPlan( + val parts: List, + val useMultipart: Boolean, + ) + + companion object { + internal val JsonConfig = Json { ignoreUnknownKeys = true } + + internal fun parseParams(paramsJson: String?, json: Json = JsonConfig): ParseResult { + val params = paramsJson?.trim().orEmpty() + if (params.isEmpty()) { + return ParseResult.Error(error = "INVALID_REQUEST: paramsJSON required") + } + + val obj = try { + json.parseToJsonElement(params).jsonObject + } catch (_: Throwable) { + null + } + + if (obj == null) { + return ParseResult.Error(error = "INVALID_REQUEST: expected JSON object") + } + + val to = (obj["to"] as? JsonPrimitive)?.content?.trim().orEmpty() + val message = (obj["message"] as? JsonPrimitive)?.content.orEmpty() + + if (to.isEmpty()) { + return ParseResult.Error( + error = "INVALID_REQUEST: 'to' phone number required", + message = message, + ) + } + + if (message.isEmpty()) { + return ParseResult.Error( + error = "INVALID_REQUEST: 'message' text required", + to = to, + ) + } + + return ParseResult.Ok(ParsedParams(to = to, message = message)) + } + + internal fun buildSendPlan( + message: String, + divider: (String) -> List, + ): SendPlan { + val parts = divider(message).ifEmpty { listOf(message) } + return SendPlan(parts = parts, useMultipart = parts.size > 1) + } + + internal fun buildPayloadJson( + json: Json = JsonConfig, + ok: Boolean, + to: String, + error: String?, + ): String { + val payload = + mutableMapOf( + "ok" to JsonPrimitive(ok), + "to" to JsonPrimitive(to), + ) + if (!ok) { + payload["error"] = JsonPrimitive(error ?: "SMS_SEND_FAILED") + } + return json.encodeToString(JsonObject.serializer(), JsonObject(payload)) + } + } + + fun hasSmsPermission(): Boolean { + return ContextCompat.checkSelfPermission( + context, + Manifest.permission.SEND_SMS + ) == PackageManager.PERMISSION_GRANTED + } + + fun canSendSms(): Boolean { + return hasSmsPermission() && hasTelephonyFeature() + } + + fun hasTelephonyFeature(): Boolean { + return context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true + } + + fun attachPermissionRequester(requester: PermissionRequester) { + permissionRequester = requester + } + + /** + * Send an SMS message. + * + * @param paramsJson JSON with "to" (phone number) and "message" (text) fields + * @return SendResult indicating success or failure + */ + suspend fun send(paramsJson: String?): SendResult { + if (!hasTelephonyFeature()) { + return errorResult( + error = "SMS_UNAVAILABLE: telephony not available", + ) + } + + if (!ensureSmsPermission()) { + return errorResult( + error = "SMS_PERMISSION_REQUIRED: grant SMS permission", + ) + } + + val parseResult = parseParams(paramsJson, json) + if (parseResult is ParseResult.Error) { + return errorResult( + error = parseResult.error, + to = parseResult.to, + message = parseResult.message, + ) + } + val params = (parseResult as ParseResult.Ok).params + + return try { + val smsManager = context.getSystemService(AndroidSmsManager::class.java) + ?: throw IllegalStateException("SMS_UNAVAILABLE: SmsManager not available") + + val plan = buildSendPlan(params.message) { smsManager.divideMessage(it) } + if (plan.useMultipart) { + smsManager.sendMultipartTextMessage( + params.to, // destination + null, // service center (null = default) + ArrayList(plan.parts), // message parts + null, // sent intents + null, // delivery intents + ) + } else { + smsManager.sendTextMessage( + params.to, // destination + null, // service center (null = default) + params.message,// message + null, // sent intent + null, // delivery intent + ) + } + + okResult(to = params.to, message = params.message) + } catch (e: SecurityException) { + errorResult( + error = "SMS_PERMISSION_REQUIRED: ${e.message}", + to = params.to, + message = params.message, + ) + } catch (e: Throwable) { + errorResult( + error = "SMS_SEND_FAILED: ${e.message ?: "unknown error"}", + to = params.to, + message = params.message, + ) + } + } + + private suspend fun ensureSmsPermission(): Boolean { + if (hasSmsPermission()) return true + val requester = permissionRequester ?: return false + val results = requester.requestIfMissing(listOf(Manifest.permission.SEND_SMS)) + return results[Manifest.permission.SEND_SMS] == true + } + + private fun okResult(to: String, message: String): SendResult { + return SendResult( + ok = true, + to = to, + message = message, + error = null, + payloadJson = buildPayloadJson(json = json, ok = true, to = to, error = null), + ) + } + + private fun errorResult(error: String, to: String = "", message: String? = null): SendResult { + return SendResult( + ok = false, + to = to, + message = message, + error = error, + payloadJson = buildPayloadJson(json = json, ok = false, to = to, error = error), + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotCanvasA2UIAction.kt b/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotCanvasA2UIAction.kt new file mode 100644 index 00000000000..f73879bb2a6 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotCanvasA2UIAction.kt @@ -0,0 +1,66 @@ +package bot.molt.android.protocol + +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +object MoltbotCanvasA2UIAction { + fun extractActionName(userAction: JsonObject): String? { + val name = + (userAction["name"] as? JsonPrimitive) + ?.content + ?.trim() + .orEmpty() + if (name.isNotEmpty()) return name + val action = + (userAction["action"] as? JsonPrimitive) + ?.content + ?.trim() + .orEmpty() + return action.ifEmpty { null } + } + + fun sanitizeTagValue(value: String): String { + val trimmed = value.trim().ifEmpty { "-" } + val normalized = trimmed.replace(" ", "_") + val out = StringBuilder(normalized.length) + for (c in normalized) { + val ok = + c.isLetterOrDigit() || + c == '_' || + c == '-' || + c == '.' || + c == ':' + out.append(if (ok) c else '_') + } + return out.toString() + } + + fun formatAgentMessage( + actionName: String, + sessionKey: String, + surfaceId: String, + sourceComponentId: String, + host: String, + instanceId: String, + contextJson: String?, + ): String { + val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty() + return listOf( + "CANVAS_A2UI", + "action=${sanitizeTagValue(actionName)}", + "session=${sanitizeTagValue(sessionKey)}", + "surface=${sanitizeTagValue(surfaceId)}", + "component=${sanitizeTagValue(sourceComponentId)}", + "host=${sanitizeTagValue(host)}", + "instance=${sanitizeTagValue(instanceId)}$ctxSuffix", + "default=update_canvas", + ).joinToString(separator = " ") + } + + fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String { + val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"") + val okLiteral = if (ok) "true" else "false" + val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"") + return "window.dispatchEvent(new CustomEvent('moltbot:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));" + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotProtocolConstants.kt b/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotProtocolConstants.kt new file mode 100644 index 00000000000..27d46c3f1be --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/protocol/ClawdbotProtocolConstants.kt @@ -0,0 +1,71 @@ +package bot.molt.android.protocol + +enum class MoltbotCapability(val rawValue: String) { + Canvas("canvas"), + Camera("camera"), + Screen("screen"), + Sms("sms"), + VoiceWake("voiceWake"), + Location("location"), +} + +enum class MoltbotCanvasCommand(val rawValue: String) { + Present("canvas.present"), + Hide("canvas.hide"), + Navigate("canvas.navigate"), + Eval("canvas.eval"), + Snapshot("canvas.snapshot"), + ; + + companion object { + const val NamespacePrefix: String = "canvas." + } +} + +enum class MoltbotCanvasA2UICommand(val rawValue: String) { + Push("canvas.a2ui.push"), + PushJSONL("canvas.a2ui.pushJSONL"), + Reset("canvas.a2ui.reset"), + ; + + companion object { + const val NamespacePrefix: String = "canvas.a2ui." + } +} + +enum class MoltbotCameraCommand(val rawValue: String) { + Snap("camera.snap"), + Clip("camera.clip"), + ; + + companion object { + const val NamespacePrefix: String = "camera." + } +} + +enum class MoltbotScreenCommand(val rawValue: String) { + Record("screen.record"), + ; + + companion object { + const val NamespacePrefix: String = "screen." + } +} + +enum class MoltbotSmsCommand(val rawValue: String) { + Send("sms.send"), + ; + + companion object { + const val NamespacePrefix: String = "sms." + } +} + +enum class MoltbotLocationCommand(val rawValue: String) { + Get("location.get"), + ; + + companion object { + const val NamespacePrefix: String = "location." + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/tools/ToolDisplay.kt b/apps/android/app/src/main/java/bot/molt/android/tools/ToolDisplay.kt new file mode 100644 index 00000000000..6f486288755 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/tools/ToolDisplay.kt @@ -0,0 +1,222 @@ +package bot.molt.android.tools + +import android.content.Context +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull + +@Serializable +private data class ToolDisplayActionSpec( + val label: String? = null, + val detailKeys: List? = null, +) + +@Serializable +private data class ToolDisplaySpec( + val emoji: String? = null, + val title: String? = null, + val label: String? = null, + val detailKeys: List? = null, + val actions: Map? = null, +) + +@Serializable +private data class ToolDisplayConfig( + val version: Int? = null, + val fallback: ToolDisplaySpec? = null, + val tools: Map? = null, +) + +data class ToolDisplaySummary( + val name: String, + val emoji: String, + val title: String, + val label: String, + val verb: String?, + val detail: String?, +) { + val detailLine: String? + get() { + val parts = mutableListOf() + if (!verb.isNullOrBlank()) parts.add(verb) + if (!detail.isNullOrBlank()) parts.add(detail) + return if (parts.isEmpty()) null else parts.joinToString(" · ") + } + + val summaryLine: String + get() = if (detailLine != null) "${emoji} ${label}: ${detailLine}" else "${emoji} ${label}" +} + +object ToolDisplayRegistry { + private const val CONFIG_ASSET = "tool-display.json" + + private val json = Json { ignoreUnknownKeys = true } + @Volatile private var cachedConfig: ToolDisplayConfig? = null + + fun resolve( + context: Context, + name: String?, + args: JsonObject?, + meta: String? = null, + ): ToolDisplaySummary { + val trimmedName = name?.trim().orEmpty().ifEmpty { "tool" } + val key = trimmedName.lowercase() + val config = loadConfig(context) + val spec = config.tools?.get(key) + val fallback = config.fallback + + val emoji = spec?.emoji ?: fallback?.emoji ?: "🧩" + val title = spec?.title ?: titleFromName(trimmedName) + val label = spec?.label ?: trimmedName + + val actionRaw = args?.get("action")?.asStringOrNull()?.trim() + val action = actionRaw?.takeIf { it.isNotEmpty() } + val actionSpec = action?.let { spec?.actions?.get(it) } + val verb = normalizeVerb(actionSpec?.label ?: action) + + var detail: String? = null + if (key == "read") { + detail = readDetail(args) + } else if (key == "write" || key == "edit" || key == "attach") { + detail = pathDetail(args) + } + + val detailKeys = actionSpec?.detailKeys ?: spec?.detailKeys ?: fallback?.detailKeys ?: emptyList() + if (detail == null) { + detail = firstValue(args, detailKeys) + } + + if (detail == null) { + detail = meta + } + + if (detail != null) { + detail = shortenHomeInString(detail) + } + + return ToolDisplaySummary( + name = trimmedName, + emoji = emoji, + title = title, + label = label, + verb = verb, + detail = detail, + ) + } + + private fun loadConfig(context: Context): ToolDisplayConfig { + val existing = cachedConfig + if (existing != null) return existing + return try { + val jsonString = context.assets.open(CONFIG_ASSET).bufferedReader().use { it.readText() } + val decoded = json.decodeFromString(ToolDisplayConfig.serializer(), jsonString) + cachedConfig = decoded + decoded + } catch (_: Throwable) { + val fallback = ToolDisplayConfig() + cachedConfig = fallback + fallback + } + } + + private fun titleFromName(name: String): String { + val cleaned = name.replace("_", " ").trim() + if (cleaned.isEmpty()) return "Tool" + return cleaned + .split(Regex("\\s+")) + .joinToString(" ") { part -> + val upper = part.uppercase() + if (part.length <= 2 && part == upper) part + else upper.firstOrNull()?.toString().orEmpty() + part.lowercase().drop(1) + } + } + + private fun normalizeVerb(value: String?): String? { + val trimmed = value?.trim().orEmpty() + if (trimmed.isEmpty()) return null + return trimmed.replace("_", " ") + } + + private fun readDetail(args: JsonObject?): String? { + val path = args?.get("path")?.asStringOrNull() ?: return null + val offset = args["offset"].asNumberOrNull() + val limit = args["limit"].asNumberOrNull() + return if (offset != null && limit != null) { + val end = offset + limit + "${path}:${offset.toInt()}-${end.toInt()}" + } else { + path + } + } + + private fun pathDetail(args: JsonObject?): String? { + return args?.get("path")?.asStringOrNull() + } + + private fun firstValue(args: JsonObject?, keys: List): String? { + for (key in keys) { + val value = valueForPath(args, key) + val rendered = renderValue(value) + if (!rendered.isNullOrBlank()) return rendered + } + return null + } + + private fun valueForPath(args: JsonObject?, path: String): JsonElement? { + var current: JsonElement? = args + for (segment in path.split(".")) { + if (segment.isBlank()) return null + val obj = current as? JsonObject ?: return null + current = obj[segment] + } + return current + } + + private fun renderValue(value: JsonElement?): String? { + if (value == null) return null + if (value is JsonPrimitive) { + if (value.isString) { + val trimmed = value.contentOrNull?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val firstLine = trimmed.lineSequence().firstOrNull()?.trim().orEmpty() + if (firstLine.isEmpty()) return null + return if (firstLine.length > 160) "${firstLine.take(157)}…" else firstLine + } + val raw = value.contentOrNull?.trim().orEmpty() + raw.toBooleanStrictOrNull()?.let { return it.toString() } + raw.toLongOrNull()?.let { return it.toString() } + raw.toDoubleOrNull()?.let { return it.toString() } + } + if (value is JsonArray) { + val items = value.mapNotNull { renderValue(it) } + if (items.isEmpty()) return null + val preview = items.take(3).joinToString(", ") + return if (items.size > 3) "${preview}…" else preview + } + return null + } + + private fun shortenHomeInString(value: String): String { + val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() } + ?: System.getenv("HOME")?.takeIf { it.isNotBlank() } + if (home.isNullOrEmpty()) return value + return value.replace(home, "~") + .replace(Regex("/Users/[^/]+"), "~") + .replace(Regex("/home/[^/]+"), "~") + } + + private fun JsonElement?.asStringOrNull(): String? { + val primitive = this as? JsonPrimitive ?: return null + return if (primitive.isString) primitive.contentOrNull else primitive.toString() + } + + private fun JsonElement?.asNumberOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + val raw = primitive.contentOrNull ?: return null + return raw.toDoubleOrNull() + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/CameraHudOverlay.kt b/apps/android/app/src/main/java/bot/molt/android/ui/CameraHudOverlay.kt new file mode 100644 index 00000000000..7b45efae9e7 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/CameraHudOverlay.kt @@ -0,0 +1,44 @@ +package bot.molt.android.ui + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Color +import kotlinx.coroutines.delay + +@Composable +fun CameraFlashOverlay( + token: Long, + modifier: Modifier = Modifier, +) { + Box(modifier = modifier.fillMaxSize()) { + CameraFlash(token = token) + } +} + +@Composable +private fun CameraFlash(token: Long) { + var alpha by remember { mutableFloatStateOf(0f) } + LaunchedEffect(token) { + if (token == 0L) return@LaunchedEffect + alpha = 0.85f + delay(110) + alpha = 0f + } + + Box( + modifier = + Modifier + .fillMaxSize() + .alpha(alpha) + .background(Color.White), + ) +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/ChatSheet.kt b/apps/android/app/src/main/java/bot/molt/android/ui/ChatSheet.kt new file mode 100644 index 00000000000..21af1a4c65b --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/ChatSheet.kt @@ -0,0 +1,10 @@ +package bot.molt.android.ui + +import androidx.compose.runtime.Composable +import bot.molt.android.MainViewModel +import bot.molt.android.ui.chat.ChatSheetContent + +@Composable +fun ChatSheet(viewModel: MainViewModel) { + ChatSheetContent(viewModel = viewModel) +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/ClawdbotTheme.kt b/apps/android/app/src/main/java/bot/molt/android/ui/ClawdbotTheme.kt new file mode 100644 index 00000000000..c292aa25dc2 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/ClawdbotTheme.kt @@ -0,0 +1,32 @@ +package bot.molt.android.ui + +import androidx.compose.foundation.isSystemInDarkTheme +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.dynamicDarkColorScheme +import androidx.compose.material3.dynamicLightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext + +@Composable +fun MoltbotTheme(content: @Composable () -> Unit) { + val context = LocalContext.current + val isDark = isSystemInDarkTheme() + val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) + + MaterialTheme(colorScheme = colorScheme, content = content) +} + +@Composable +fun overlayContainerColor(): Color { + val scheme = MaterialTheme.colorScheme + val isDark = isSystemInDarkTheme() + val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh + // Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare. + return if (isDark) base else base.copy(alpha = 0.88f) +} + +@Composable +fun overlayIconColor(): Color { + return MaterialTheme.colorScheme.onSurfaceVariant +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt b/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt new file mode 100644 index 00000000000..67d76b82fac --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt @@ -0,0 +1,449 @@ +package bot.molt.android.ui + +import android.annotation.SuppressLint +import android.Manifest +import android.content.pm.PackageManager +import android.graphics.Color +import android.util.Log +import android.view.View +import android.webkit.JavascriptInterface +import android.webkit.ConsoleMessage +import android.webkit.WebChromeClient +import android.webkit.WebView +import android.webkit.WebSettings +import android.webkit.WebResourceError +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebViewClient +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.webkit.WebSettingsCompat +import androidx.webkit.WebViewFeature +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.WindowInsetsSides +import androidx.compose.foundation.layout.only +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.safeDrawing +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.windowInsetsPadding +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.LocalContentColor +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.filled.ScreenShare +import androidx.compose.material.icons.filled.ChatBubble +import androidx.compose.material.icons.filled.CheckCircle +import androidx.compose.material.icons.filled.Error +import androidx.compose.material.icons.filled.FiberManualRecord +import androidx.compose.material.icons.filled.PhotoCamera +import androidx.compose.material.icons.filled.RecordVoiceOver +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material.icons.filled.Report +import androidx.compose.material.icons.filled.Settings +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color as ComposeColor +import androidx.compose.ui.graphics.lerp +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.compose.ui.window.Popup +import androidx.compose.ui.window.PopupProperties +import androidx.core.content.ContextCompat +import bot.molt.android.CameraHudKind +import bot.molt.android.MainViewModel + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun RootScreen(viewModel: MainViewModel) { + var sheet by remember { mutableStateOf(null) } + val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal) + val context = LocalContext.current + val serverName by viewModel.serverName.collectAsState() + val statusText by viewModel.statusText.collectAsState() + val cameraHud by viewModel.cameraHud.collectAsState() + val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() + val screenRecordActive by viewModel.screenRecordActive.collectAsState() + val isForeground by viewModel.isForeground.collectAsState() + val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() + val talkEnabled by viewModel.talkEnabled.collectAsState() + val talkStatusText by viewModel.talkStatusText.collectAsState() + val talkIsListening by viewModel.talkIsListening.collectAsState() + val talkIsSpeaking by viewModel.talkIsSpeaking.collectAsState() + val seamColorArgb by viewModel.seamColorArgb.collectAsState() + val seamColor = remember(seamColorArgb) { ComposeColor(seamColorArgb) } + val audioPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + if (granted) viewModel.setTalkEnabled(true) + } + val activity = + remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) { + // Status pill owns transient activity state so it doesn't overlap the connection indicator. + if (!isForeground) { + return@remember StatusActivity( + title = "Foreground required", + icon = Icons.Default.Report, + contentDescription = "Foreground required", + ) + } + + val lowerStatus = statusText.lowercase() + if (lowerStatus.contains("repair")) { + return@remember StatusActivity( + title = "Repairing…", + icon = Icons.Default.Refresh, + contentDescription = "Repairing", + ) + } + if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) { + return@remember StatusActivity( + title = "Approval pending", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Approval pending", + ) + } + // Avoid duplicating the primary gateway status ("Connecting…") in the activity slot. + + if (screenRecordActive) { + return@remember StatusActivity( + title = "Recording screen…", + icon = Icons.AutoMirrored.Filled.ScreenShare, + contentDescription = "Recording screen", + tint = androidx.compose.ui.graphics.Color.Red, + ) + } + + cameraHud?.let { hud -> + return@remember when (hud.kind) { + CameraHudKind.Photo -> + StatusActivity( + title = hud.message, + icon = Icons.Default.PhotoCamera, + contentDescription = "Taking photo", + ) + CameraHudKind.Recording -> + StatusActivity( + title = hud.message, + icon = Icons.Default.FiberManualRecord, + contentDescription = "Recording", + tint = androidx.compose.ui.graphics.Color.Red, + ) + CameraHudKind.Success -> + StatusActivity( + title = hud.message, + icon = Icons.Default.CheckCircle, + contentDescription = "Capture finished", + ) + CameraHudKind.Error -> + StatusActivity( + title = hud.message, + icon = Icons.Default.Error, + contentDescription = "Capture failed", + tint = androidx.compose.ui.graphics.Color.Red, + ) + } + } + + if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) { + return@remember StatusActivity( + title = "Mic permission", + icon = Icons.Default.Error, + contentDescription = "Mic permission required", + ) + } + if (voiceWakeStatusText == "Paused") { + val suffix = if (!isForeground) " (background)" else "" + return@remember StatusActivity( + title = "Voice Wake paused$suffix", + icon = Icons.Default.RecordVoiceOver, + contentDescription = "Voice Wake paused", + ) + } + + null + } + + val gatewayState = + remember(serverName, statusText) { + when { + serverName != null -> GatewayState.Connected + statusText.contains("connecting", ignoreCase = true) || + statusText.contains("reconnecting", ignoreCase = true) -> GatewayState.Connecting + statusText.contains("error", ignoreCase = true) -> GatewayState.Error + else -> GatewayState.Disconnected + } + } + + val voiceEnabled = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + + Box(modifier = Modifier.fillMaxSize()) { + CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize()) + } + + // Camera flash must be in a Popup to render above the WebView. + Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { + CameraFlashOverlay(token = cameraFlashToken, modifier = Modifier.fillMaxSize()) + } + + // Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches. + Popup(alignment = Alignment.TopStart, properties = PopupProperties(focusable = false)) { + StatusPill( + gateway = gatewayState, + voiceEnabled = voiceEnabled, + activity = activity, + onClick = { sheet = Sheet.Settings }, + modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp), + ) + } + + Popup(alignment = Alignment.TopEnd, properties = PopupProperties(focusable = false)) { + Column( + modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(end = 12.dp, top = 12.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + horizontalAlignment = Alignment.End, + ) { + OverlayIconButton( + onClick = { sheet = Sheet.Chat }, + icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") }, + ) + + // Talk mode gets a dedicated side bubble instead of burying it in settings. + val baseOverlay = overlayContainerColor() + val talkContainer = + lerp( + baseOverlay, + seamColor.copy(alpha = baseOverlay.alpha), + if (talkEnabled) 0.35f else 0.22f, + ) + val talkContent = if (talkEnabled) seamColor else overlayIconColor() + OverlayIconButton( + onClick = { + val next = !talkEnabled + if (next) { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setTalkEnabled(true) + } else { + viewModel.setTalkEnabled(false) + } + }, + containerColor = talkContainer, + contentColor = talkContent, + icon = { + Icon( + Icons.Default.RecordVoiceOver, + contentDescription = "Talk Mode", + ) + }, + ) + + OverlayIconButton( + onClick = { sheet = Sheet.Settings }, + icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") }, + ) + } + } + + if (talkEnabled) { + Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { + TalkOrbOverlay( + seamColor = seamColor, + statusText = talkStatusText, + isListening = talkIsListening, + isSpeaking = talkIsSpeaking, + ) + } + } + + val currentSheet = sheet + if (currentSheet != null) { + ModalBottomSheet( + onDismissRequest = { sheet = null }, + sheetState = sheetState, + ) { + when (currentSheet) { + Sheet.Chat -> ChatSheet(viewModel = viewModel) + Sheet.Settings -> SettingsSheet(viewModel = viewModel) + } + } + } +} + +private enum class Sheet { + Chat, + Settings, +} + +@Composable +private fun OverlayIconButton( + onClick: () -> Unit, + icon: @Composable () -> Unit, + containerColor: ComposeColor? = null, + contentColor: ComposeColor? = null, +) { + FilledTonalIconButton( + onClick = onClick, + modifier = Modifier.size(44.dp), + colors = + IconButtonDefaults.filledTonalIconButtonColors( + containerColor = containerColor ?: overlayContainerColor(), + contentColor = contentColor ?: overlayIconColor(), + ), + ) { + icon() + } +} + +@SuppressLint("SetJavaScriptEnabled") +@Composable +private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier) { + val context = LocalContext.current + val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0 + AndroidView( + modifier = modifier, + factory = { + WebView(context).apply { + settings.javaScriptEnabled = true + // Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage. + settings.domStorageEnabled = true + settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE + if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) { + WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false) + } else { + disableForceDarkIfSupported(settings) + } + if (isDebuggable) { + Log.d("MoltbotWebView", "userAgent: ${settings.userAgentString}") + } + isScrollContainer = true + overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS + isVerticalScrollBarEnabled = true + isHorizontalScrollBarEnabled = true + webViewClient = + object : WebViewClient() { + override fun onReceivedError( + view: WebView, + request: WebResourceRequest, + error: WebResourceError, + ) { + if (!isDebuggable) return + if (!request.isForMainFrame) return + Log.e("MoltbotWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}") + } + + override fun onReceivedHttpError( + view: WebView, + request: WebResourceRequest, + errorResponse: WebResourceResponse, + ) { + if (!isDebuggable) return + if (!request.isForMainFrame) return + Log.e( + "MoltbotWebView", + "onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}", + ) + } + + override fun onPageFinished(view: WebView, url: String?) { + if (isDebuggable) { + Log.d("MoltbotWebView", "onPageFinished: $url") + } + viewModel.canvas.onPageFinished() + } + + override fun onRenderProcessGone( + view: WebView, + detail: android.webkit.RenderProcessGoneDetail, + ): Boolean { + if (isDebuggable) { + Log.e( + "MoltbotWebView", + "onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}", + ) + } + return true + } + } + webChromeClient = + object : WebChromeClient() { + override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean { + if (!isDebuggable) return false + val msg = consoleMessage ?: return false + Log.d( + "MoltbotWebView", + "console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}", + ) + return false + } + } + // Use default layer/background; avoid forcing a black fill over WebView content. + + val a2uiBridge = + CanvasA2UIActionBridge { payload -> + viewModel.handleCanvasA2UIActionFromWebView(payload) + } + addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName) + addJavascriptInterface( + CanvasA2UIActionLegacyBridge(a2uiBridge), + CanvasA2UIActionLegacyBridge.interfaceName, + ) + viewModel.canvas.attach(this) + } + }, + ) +} + +private fun disableForceDarkIfSupported(settings: WebSettings) { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) return + @Suppress("DEPRECATION") + WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF) +} + +private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) { + @JavascriptInterface + fun postMessage(payload: String?) { + val msg = payload?.trim().orEmpty() + if (msg.isEmpty()) return + onMessage(msg) + } + + companion object { + const val interfaceName: String = "moltbotCanvasA2UIAction" + } +} + +private class CanvasA2UIActionLegacyBridge(private val bridge: CanvasA2UIActionBridge) { + @JavascriptInterface + fun canvasAction(payload: String?) { + bridge.postMessage(payload) + } + + @JavascriptInterface + fun postMessage(payload: String?) { + bridge.postMessage(payload) + } + + companion object { + const val interfaceName: String = "Android" + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt b/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt new file mode 100644 index 00000000000..f96731acfb5 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt @@ -0,0 +1,686 @@ +package bot.molt.android.ui + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.os.Build +import android.provider.Settings +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.WindowInsetsSides +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.only +import androidx.compose.foundation.layout.safeDrawing +import androidx.compose.foundation.layout.windowInsetsPadding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ExpandLess +import androidx.compose.material.icons.filled.ExpandMore +import androidx.compose.material3.Button +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Icon +import androidx.compose.material3.ListItem +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.focus.onFocusChanged +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.text.input.ImeAction +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import bot.molt.android.BuildConfig +import bot.molt.android.LocationMode +import bot.molt.android.MainViewModel +import bot.molt.android.NodeForegroundService +import bot.molt.android.VoiceWakeMode +import bot.molt.android.WakeWords + +@Composable +fun SettingsSheet(viewModel: MainViewModel) { + val context = LocalContext.current + val instanceId by viewModel.instanceId.collectAsState() + val displayName by viewModel.displayName.collectAsState() + val cameraEnabled by viewModel.cameraEnabled.collectAsState() + val locationMode by viewModel.locationMode.collectAsState() + val locationPreciseEnabled by viewModel.locationPreciseEnabled.collectAsState() + val preventSleep by viewModel.preventSleep.collectAsState() + val wakeWords by viewModel.wakeWords.collectAsState() + val voiceWakeMode by viewModel.voiceWakeMode.collectAsState() + val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() + val isConnected by viewModel.isConnected.collectAsState() + val manualEnabled by viewModel.manualEnabled.collectAsState() + val manualHost by viewModel.manualHost.collectAsState() + val manualPort by viewModel.manualPort.collectAsState() + val manualTls by viewModel.manualTls.collectAsState() + val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState() + val statusText by viewModel.statusText.collectAsState() + val serverName by viewModel.serverName.collectAsState() + val remoteAddress by viewModel.remoteAddress.collectAsState() + val gateways by viewModel.gateways.collectAsState() + val discoveryStatusText by viewModel.discoveryStatusText.collectAsState() + + val listState = rememberLazyListState() + val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") } + val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) } + val focusManager = LocalFocusManager.current + var wakeWordsHadFocus by remember { mutableStateOf(false) } + val deviceModel = + remember { + listOfNotNull(Build.MANUFACTURER, Build.MODEL) + .joinToString(" ") + .trim() + .ifEmpty { "Android" } + } + val appVersion = + remember { + val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } + if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { + "$versionName-dev" + } else { + versionName + } + } + + LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) } + val commitWakeWords = { + val parsed = WakeWords.parseIfChanged(wakeWordsText, wakeWords) + if (parsed != null) { + viewModel.setWakeWords(parsed) + } + } + + val permissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> + val cameraOk = perms[Manifest.permission.CAMERA] == true + viewModel.setCameraEnabled(cameraOk) + } + + var pendingLocationMode by remember { mutableStateOf(null) } + var pendingPreciseToggle by remember { mutableStateOf(false) } + + val locationPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> + val fineOk = perms[Manifest.permission.ACCESS_FINE_LOCATION] == true + val coarseOk = perms[Manifest.permission.ACCESS_COARSE_LOCATION] == true + val granted = fineOk || coarseOk + val requestedMode = pendingLocationMode + pendingLocationMode = null + + if (pendingPreciseToggle) { + pendingPreciseToggle = false + viewModel.setLocationPreciseEnabled(fineOk) + return@rememberLauncherForActivityResult + } + + if (!granted) { + viewModel.setLocationMode(LocationMode.Off) + return@rememberLauncherForActivityResult + } + + if (requestedMode != null) { + viewModel.setLocationMode(requestedMode) + if (requestedMode == LocationMode.Always) { + val backgroundOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!backgroundOk) { + openAppSettings(context) + } + } + } + } + + val audioPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { _ -> + // Status text is handled by NodeRuntime. + } + + val smsPermissionAvailable = + remember { + context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true + } + var smsPermissionGranted by + remember { + mutableStateOf( + ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) == + PackageManager.PERMISSION_GRANTED, + ) + } + val smsPermissionLauncher = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + smsPermissionGranted = granted + viewModel.refreshGatewayConnection() + } + + fun setCameraEnabledChecked(checked: Boolean) { + if (!checked) { + viewModel.setCameraEnabled(false) + return + } + + val cameraOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == + PackageManager.PERMISSION_GRANTED + if (cameraOk) { + viewModel.setCameraEnabled(true) + } else { + permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)) + } + } + + fun requestLocationPermissions(targetMode: LocationMode) { + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + val coarseOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (fineOk || coarseOk) { + viewModel.setLocationMode(targetMode) + if (targetMode == LocationMode.Always) { + val backgroundOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (!backgroundOk) { + openAppSettings(context) + } + } + } else { + pendingLocationMode = targetMode + locationPermissionLauncher.launch( + arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION), + ) + } + } + + fun setPreciseLocationChecked(checked: Boolean) { + if (!checked) { + viewModel.setLocationPreciseEnabled(false) + return + } + val fineOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED + if (fineOk) { + viewModel.setLocationPreciseEnabled(true) + } else { + pendingPreciseToggle = true + locationPermissionLauncher.launch(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION)) + } + } + + val visibleGateways = + if (isConnected && remoteAddress != null) { + gateways.filterNot { "${it.host}:${it.port}" == remoteAddress } + } else { + gateways + } + + val gatewayDiscoveryFooterText = + if (visibleGateways.isEmpty()) { + discoveryStatusText + } else if (isConnected) { + "Discovery active • ${visibleGateways.size} other gateway${if (visibleGateways.size == 1) "" else "s"} found" + } else { + "Discovery active • ${visibleGateways.size} gateway${if (visibleGateways.size == 1) "" else "s"} found" + } + + LazyColumn( + state = listState, + modifier = + Modifier + .fillMaxWidth() + .fillMaxHeight() + .imePadding() + .windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)), + contentPadding = PaddingValues(16.dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + // Order parity: Node → Gateway → Voice → Camera → Messaging → Location → Screen. + item { Text("Node", style = MaterialTheme.typography.titleSmall) } + item { + OutlinedTextField( + value = displayName, + onValueChange = viewModel::setDisplayName, + label = { Text("Name") }, + modifier = Modifier.fillMaxWidth(), + ) + } + item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) } + item { Text("Device: $deviceModel", color = MaterialTheme.colorScheme.onSurfaceVariant) } + item { Text("Version: $appVersion", color = MaterialTheme.colorScheme.onSurfaceVariant) } + + item { HorizontalDivider() } + + // Gateway + item { Text("Gateway", style = MaterialTheme.typography.titleSmall) } + item { ListItem(headlineContent = { Text("Status") }, supportingContent = { Text(statusText) }) } + if (serverName != null) { + item { ListItem(headlineContent = { Text("Server") }, supportingContent = { Text(serverName!!) }) } + } + if (remoteAddress != null) { + item { ListItem(headlineContent = { Text("Address") }, supportingContent = { Text(remoteAddress!!) }) } + } + item { + // UI sanity: "Disconnect" only when we have an active remote. + if (isConnected && remoteAddress != null) { + Button( + onClick = { + viewModel.disconnect() + NodeForegroundService.stop(context) + }, + ) { + Text("Disconnect") + } + } + } + + item { HorizontalDivider() } + + if (!isConnected || visibleGateways.isNotEmpty()) { + item { + Text( + if (isConnected) "Other Gateways" else "Discovered Gateways", + style = MaterialTheme.typography.titleSmall, + ) + } + if (!isConnected && visibleGateways.isEmpty()) { + item { Text("No gateways found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) } + } else { + items(items = visibleGateways, key = { it.stableId }) { gateway -> + val detailLines = + buildList { + add("IP: ${gateway.host}:${gateway.port}") + gateway.lanHost?.let { add("LAN: $it") } + gateway.tailnetDns?.let { add("Tailnet: $it") } + if (gateway.gatewayPort != null || gateway.canvasPort != null) { + val gw = (gateway.gatewayPort ?: gateway.port).toString() + val canvas = gateway.canvasPort?.toString() ?: "—" + add("Ports: gw $gw · canvas $canvas") + } + } + ListItem( + headlineContent = { Text(gateway.name) }, + supportingContent = { + Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { + detailLines.forEach { line -> + Text(line, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + }, + trailingContent = { + Button( + onClick = { + NodeForegroundService.start(context) + viewModel.connect(gateway) + }, + ) { + Text("Connect") + } + }, + ) + } + } + item { + Text( + gatewayDiscoveryFooterText, + modifier = Modifier.fillMaxWidth(), + textAlign = TextAlign.Center, + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + + item { HorizontalDivider() } + + item { + ListItem( + headlineContent = { Text("Advanced") }, + supportingContent = { Text("Manual gateway connection") }, + trailingContent = { + Icon( + imageVector = if (advancedExpanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore, + contentDescription = if (advancedExpanded) "Collapse" else "Expand", + ) + }, + modifier = + Modifier.clickable { + setAdvancedExpanded(!advancedExpanded) + }, + ) + } + item { + AnimatedVisibility(visible = advancedExpanded) { + Column(verticalArrangement = Arrangement.spacedBy(10.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Use Manual Gateway") }, + supportingContent = { Text("Use this when discovery is blocked.") }, + trailingContent = { Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) }, + ) + + OutlinedTextField( + value = manualHost, + onValueChange = viewModel::setManualHost, + label = { Text("Host") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + OutlinedTextField( + value = manualPort.toString(), + onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) }, + label = { Text("Port") }, + modifier = Modifier.fillMaxWidth(), + enabled = manualEnabled, + ) + ListItem( + headlineContent = { Text("Require TLS") }, + supportingContent = { Text("Pin the gateway certificate on first connect.") }, + trailingContent = { Switch(checked = manualTls, onCheckedChange = viewModel::setManualTls, enabled = manualEnabled) }, + modifier = Modifier.alpha(if (manualEnabled) 1f else 0.5f), + ) + + val hostOk = manualHost.trim().isNotEmpty() + val portOk = manualPort in 1..65535 + Button( + onClick = { + NodeForegroundService.start(context) + viewModel.connectManual() + }, + enabled = manualEnabled && hostOk && portOk, + ) { + Text("Connect (Manual)") + } + } + } + } + + item { HorizontalDivider() } + + // Voice + item { Text("Voice", style = MaterialTheme.typography.titleSmall) } + item { + val enabled = voiceWakeMode != VoiceWakeMode.Off + ListItem( + headlineContent = { Text("Voice Wake") }, + supportingContent = { Text(voiceWakeStatusText) }, + trailingContent = { + Switch( + checked = enabled, + onCheckedChange = { on -> + if (on) { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) + } else { + viewModel.setVoiceWakeMode(VoiceWakeMode.Off) + } + }, + ) + }, + ) + } + item { + AnimatedVisibility(visible = voiceWakeMode != VoiceWakeMode.Off) { + Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Foreground Only") }, + supportingContent = { Text("Listens only while Moltbot is open.") }, + trailingContent = { + RadioButton( + selected = voiceWakeMode == VoiceWakeMode.Foreground, + onClick = { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) + }, + ) + }, + ) + ListItem( + headlineContent = { Text("Always") }, + supportingContent = { Text("Keeps listening in the background (shows a persistent notification).") }, + trailingContent = { + RadioButton( + selected = voiceWakeMode == VoiceWakeMode.Always, + onClick = { + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + viewModel.setVoiceWakeMode(VoiceWakeMode.Always) + }, + ) + }, + ) + } + } + } + item { + OutlinedTextField( + value = wakeWordsText, + onValueChange = setWakeWordsText, + label = { Text("Wake Words (comma-separated)") }, + modifier = + Modifier.fillMaxWidth().onFocusChanged { focusState -> + if (focusState.isFocused) { + wakeWordsHadFocus = true + } else if (wakeWordsHadFocus) { + wakeWordsHadFocus = false + commitWakeWords() + } + }, + singleLine = true, + keyboardOptions = KeyboardOptions(imeAction = ImeAction.Done), + keyboardActions = + KeyboardActions( + onDone = { + commitWakeWords() + focusManager.clearFocus() + }, + ), + ) + } + item { Button(onClick = viewModel::resetWakeWordsDefaults) { Text("Reset defaults") } } + item { + Text( + if (isConnected) { + "Any node can edit wake words. Changes sync via the gateway." + } else { + "Connect to a gateway to sync wake words globally." + }, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Camera + item { Text("Camera", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Allow Camera") }, + supportingContent = { Text("Allows the gateway to request photos or short video clips (foreground only).") }, + trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) }, + ) + } + item { + Text( + "Tip: grant Microphone permission for video clips with audio.", + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Messaging + item { Text("Messaging", style = MaterialTheme.typography.titleSmall) } + item { + val buttonLabel = + when { + !smsPermissionAvailable -> "Unavailable" + smsPermissionGranted -> "Manage" + else -> "Grant" + } + ListItem( + headlineContent = { Text("SMS Permission") }, + supportingContent = { + Text( + if (smsPermissionAvailable) { + "Allow the gateway to send SMS from this device." + } else { + "SMS requires a device with telephony hardware." + }, + ) + }, + trailingContent = { + Button( + onClick = { + if (!smsPermissionAvailable) return@Button + if (smsPermissionGranted) { + openAppSettings(context) + } else { + smsPermissionLauncher.launch(Manifest.permission.SEND_SMS) + } + }, + enabled = smsPermissionAvailable, + ) { + Text(buttonLabel) + } + }, + ) + } + + item { HorizontalDivider() } + + // Location + item { Text("Location", style = MaterialTheme.typography.titleSmall) } + item { + Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { + ListItem( + headlineContent = { Text("Off") }, + supportingContent = { Text("Disable location sharing.") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.Off, + onClick = { viewModel.setLocationMode(LocationMode.Off) }, + ) + }, + ) + ListItem( + headlineContent = { Text("While Using") }, + supportingContent = { Text("Only while Moltbot is open.") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.WhileUsing, + onClick = { requestLocationPermissions(LocationMode.WhileUsing) }, + ) + }, + ) + ListItem( + headlineContent = { Text("Always") }, + supportingContent = { Text("Allow background location (requires system permission).") }, + trailingContent = { + RadioButton( + selected = locationMode == LocationMode.Always, + onClick = { requestLocationPermissions(LocationMode.Always) }, + ) + }, + ) + } + } + item { + ListItem( + headlineContent = { Text("Precise Location") }, + supportingContent = { Text("Use precise GPS when available.") }, + trailingContent = { + Switch( + checked = locationPreciseEnabled, + onCheckedChange = ::setPreciseLocationChecked, + enabled = locationMode != LocationMode.Off, + ) + }, + ) + } + item { + Text( + "Always may require Android Settings to allow background location.", + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + item { HorizontalDivider() } + + // Screen + item { Text("Screen", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Prevent Sleep") }, + supportingContent = { Text("Keeps the screen awake while Moltbot is open.") }, + trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) }, + ) + } + + item { HorizontalDivider() } + + // Debug + item { Text("Debug", style = MaterialTheme.typography.titleSmall) } + item { + ListItem( + headlineContent = { Text("Debug Canvas Status") }, + supportingContent = { Text("Show status text in the canvas when debug is enabled.") }, + trailingContent = { + Switch( + checked = canvasDebugStatusEnabled, + onCheckedChange = viewModel::setCanvasDebugStatusEnabled, + ) + }, + ) + } + + item { Spacer(modifier = Modifier.height(20.dp)) } + } +} + +private fun openAppSettings(context: Context) { + val intent = + Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", context.packageName, null), + ) + context.startActivity(intent) +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/StatusPill.kt b/apps/android/app/src/main/java/bot/molt/android/ui/StatusPill.kt new file mode 100644 index 00000000000..199bcbf820d --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/StatusPill.kt @@ -0,0 +1,114 @@ +package bot.molt.android.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.MicOff +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.VerticalDivider +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp + +@Composable +fun StatusPill( + gateway: GatewayState, + voiceEnabled: Boolean, + onClick: () -> Unit, + modifier: Modifier = Modifier, + activity: StatusActivity? = null, +) { + Surface( + onClick = onClick, + modifier = modifier, + shape = RoundedCornerShape(14.dp), + color = overlayContainerColor(), + tonalElevation = 3.dp, + shadowElevation = 0.dp, + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp), + horizontalArrangement = Arrangement.spacedBy(10.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) { + Surface( + modifier = Modifier.size(9.dp), + shape = CircleShape, + color = gateway.color, + ) {} + + Text( + text = gateway.title, + style = MaterialTheme.typography.labelLarge, + ) + } + + VerticalDivider( + modifier = Modifier.height(14.dp).alpha(0.35f), + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + + if (activity != null) { + Row( + horizontalArrangement = Arrangement.spacedBy(6.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon( + imageVector = activity.icon, + contentDescription = activity.contentDescription, + tint = activity.tint ?: overlayIconColor(), + modifier = Modifier.size(18.dp), + ) + Text( + text = activity.title, + style = MaterialTheme.typography.labelLarge, + maxLines = 1, + ) + } + } else { + Icon( + imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff, + contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled", + tint = + if (voiceEnabled) { + overlayIconColor() + } else { + MaterialTheme.colorScheme.onSurfaceVariant + }, + modifier = Modifier.size(18.dp), + ) + } + + Spacer(modifier = Modifier.width(2.dp)) + } + } +} + +data class StatusActivity( + val title: String, + val icon: androidx.compose.ui.graphics.vector.ImageVector, + val contentDescription: String, + val tint: Color? = null, +) + +enum class GatewayState(val title: String, val color: Color) { + Connected("Connected", Color(0xFF2ECC71)), + Connecting("Connecting…", Color(0xFFF1C40F)), + Error("Error", Color(0xFFE74C3C)), + Disconnected("Offline", Color(0xFF9E9E9E)), +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/TalkOrbOverlay.kt b/apps/android/app/src/main/java/bot/molt/android/ui/TalkOrbOverlay.kt new file mode 100644 index 00000000000..9098c06ff3c --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/TalkOrbOverlay.kt @@ -0,0 +1,134 @@ +package bot.molt.android.ui + +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp + +@Composable +fun TalkOrbOverlay( + seamColor: Color, + statusText: String, + isListening: Boolean, + isSpeaking: Boolean, + modifier: Modifier = Modifier, +) { + val transition = rememberInfiniteTransition(label = "talk-orb") + val t by + transition.animateFloat( + initialValue = 0f, + targetValue = 1f, + animationSpec = + infiniteRepeatable( + animation = tween(durationMillis = 1500, easing = LinearEasing), + repeatMode = RepeatMode.Restart, + ), + label = "pulse", + ) + + val trimmed = statusText.trim() + val showStatus = trimmed.isNotEmpty() && trimmed != "Off" + val phase = + when { + isSpeaking -> "Speaking" + isListening -> "Listening" + else -> "Thinking" + } + + Column( + modifier = modifier.padding(24.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + Box(contentAlignment = Alignment.Center) { + Canvas(modifier = Modifier.size(360.dp)) { + val center = this.center + val baseRadius = size.minDimension * 0.30f + + val ring1 = 1.05f + (t * 0.25f) + val ring2 = 1.20f + (t * 0.55f) + val ringAlpha1 = (1f - t) * 0.34f + val ringAlpha2 = (1f - t) * 0.22f + + drawCircle( + color = seamColor.copy(alpha = ringAlpha1), + radius = baseRadius * ring1, + center = center, + style = Stroke(width = 3.dp.toPx()), + ) + drawCircle( + color = seamColor.copy(alpha = ringAlpha2), + radius = baseRadius * ring2, + center = center, + style = Stroke(width = 3.dp.toPx()), + ) + + drawCircle( + brush = + Brush.radialGradient( + colors = + listOf( + seamColor.copy(alpha = 0.92f), + seamColor.copy(alpha = 0.40f), + Color.Black.copy(alpha = 0.56f), + ), + center = center, + radius = baseRadius * 1.35f, + ), + radius = baseRadius, + center = center, + ) + + drawCircle( + color = seamColor.copy(alpha = 0.34f), + radius = baseRadius, + center = center, + style = Stroke(width = 1.dp.toPx()), + ) + } + } + + if (showStatus) { + Surface( + color = Color.Black.copy(alpha = 0.40f), + shape = CircleShape, + ) { + Text( + text = trimmed, + modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp), + color = Color.White.copy(alpha = 0.92f), + style = MaterialTheme.typography.labelLarge, + fontWeight = FontWeight.SemiBold, + ) + } + } else { + Text( + text = phase, + color = Color.White.copy(alpha = 0.80f), + style = MaterialTheme.typography.labelLarge, + fontWeight = FontWeight.SemiBold, + ) + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatComposer.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatComposer.kt new file mode 100644 index 00000000000..bc0d9917f55 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatComposer.kt @@ -0,0 +1,285 @@ +package bot.molt.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.horizontalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowUpward +import androidx.compose.material.icons.filled.AttachFile +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material.icons.filled.Stop +import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.FilledTonalButton +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import bot.molt.android.chat.ChatSessionEntry + +@Composable +fun ChatComposer( + sessionKey: String, + sessions: List, + mainSessionKey: String, + healthOk: Boolean, + thinkingLevel: String, + pendingRunCount: Int, + errorText: String?, + attachments: List, + onPickImages: () -> Unit, + onRemoveAttachment: (id: String) -> Unit, + onSetThinkingLevel: (level: String) -> Unit, + onSelectSession: (sessionKey: String) -> Unit, + onRefresh: () -> Unit, + onAbort: () -> Unit, + onSend: (text: String) -> Unit, +) { + var input by rememberSaveable { mutableStateOf("") } + var showThinkingMenu by remember { mutableStateOf(false) } + var showSessionMenu by remember { mutableStateOf(false) } + + val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey) + val currentSessionLabel = + sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey + + val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk + + Surface( + shape = MaterialTheme.shapes.large, + color = MaterialTheme.colorScheme.surfaceContainer, + tonalElevation = 0.dp, + shadowElevation = 0.dp, + ) { + Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Box { + FilledTonalButton( + onClick = { showSessionMenu = true }, + contentPadding = ButtonDefaults.ContentPadding, + ) { + Text("Session: $currentSessionLabel") + } + + DropdownMenu(expanded = showSessionMenu, onDismissRequest = { showSessionMenu = false }) { + for (entry in sessionOptions) { + DropdownMenuItem( + text = { Text(entry.displayName ?: entry.key) }, + onClick = { + onSelectSession(entry.key) + showSessionMenu = false + }, + trailingIcon = { + if (entry.key == sessionKey) { + Text("✓") + } else { + Spacer(modifier = Modifier.width(10.dp)) + } + }, + ) + } + } + } + + Box { + FilledTonalButton( + onClick = { showThinkingMenu = true }, + contentPadding = ButtonDefaults.ContentPadding, + ) { + Text("Thinking: ${thinkingLabel(thinkingLevel)}") + } + + DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) { + ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } + } + } + + Spacer(modifier = Modifier.weight(1f)) + + FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) { + Icon(Icons.Default.Refresh, contentDescription = "Refresh") + } + + FilledTonalIconButton(onClick = onPickImages, modifier = Modifier.size(42.dp)) { + Icon(Icons.Default.AttachFile, contentDescription = "Add image") + } + } + + if (attachments.isNotEmpty()) { + AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment) + } + + OutlinedTextField( + value = input, + onValueChange = { input = it }, + modifier = Modifier.fillMaxWidth(), + placeholder = { Text("Message Clawd…") }, + minLines = 2, + maxLines = 6, + ) + + Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { + ConnectionPill(sessionLabel = currentSessionLabel, healthOk = healthOk) + Spacer(modifier = Modifier.weight(1f)) + + if (pendingRunCount > 0) { + FilledTonalIconButton( + onClick = onAbort, + colors = + IconButtonDefaults.filledTonalIconButtonColors( + containerColor = Color(0x33E74C3C), + contentColor = Color(0xFFE74C3C), + ), + ) { + Icon(Icons.Default.Stop, contentDescription = "Abort") + } + } else { + FilledTonalIconButton(onClick = { + val text = input + input = "" + onSend(text) + }, enabled = canSend) { + Icon(Icons.Default.ArrowUpward, contentDescription = "Send") + } + } + } + + if (!errorText.isNullOrBlank()) { + Text( + text = errorText, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + maxLines = 2, + ) + } + } + } +} + +@Composable +private fun ConnectionPill(sessionLabel: String, healthOk: Boolean) { + Surface( + shape = RoundedCornerShape(999.dp), + color = MaterialTheme.colorScheme.surfaceContainerHighest, + ) { + Row( + modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Surface( + modifier = Modifier.size(7.dp), + shape = androidx.compose.foundation.shape.CircleShape, + color = if (healthOk) Color(0xFF2ECC71) else Color(0xFFF39C12), + ) {} + Text(sessionLabel, style = MaterialTheme.typography.labelSmall) + Text( + if (healthOk) "Connected" else "Connecting…", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } +} + +@Composable +private fun ThinkingMenuItem( + value: String, + current: String, + onSet: (String) -> Unit, + onDismiss: () -> Unit, +) { + DropdownMenuItem( + text = { Text(thinkingLabel(value)) }, + onClick = { + onSet(value) + onDismiss() + }, + trailingIcon = { + if (value == current.trim().lowercase()) { + Text("✓") + } else { + Spacer(modifier = Modifier.width(10.dp)) + } + }, + ) +} + +private fun thinkingLabel(raw: String): String { + return when (raw.trim().lowercase()) { + "low" -> "Low" + "medium" -> "Medium" + "high" -> "High" + else -> "Off" + } +} + +@Composable +private fun AttachmentsStrip( + attachments: List, + onRemoveAttachment: (id: String) -> Unit, +) { + Row( + modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + for (att in attachments) { + AttachmentChip( + fileName = att.fileName, + onRemove = { onRemoveAttachment(att.id) }, + ) + } + } +} + +@Composable +private fun AttachmentChip(fileName: String, onRemove: () -> Unit) { + Surface( + shape = RoundedCornerShape(999.dp), + color = MaterialTheme.colorScheme.primary.copy(alpha = 0.10f), + ) { + Row( + modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text(text = fileName, style = MaterialTheme.typography.bodySmall, maxLines = 1) + FilledTonalIconButton( + onClick = onRemove, + modifier = Modifier.size(30.dp), + ) { + Text("×") + } + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMarkdown.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMarkdown.kt new file mode 100644 index 00000000000..10cf25b8136 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMarkdown.kt @@ -0,0 +1,215 @@ +package bot.molt.android.ui.chat + +import android.graphics.BitmapFactory +import android.util.Base64 +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.text.selection.SelectionContainer +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.text.SpanStyle +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontStyle +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.unit.dp +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +@Composable +fun ChatMarkdown(text: String, textColor: Color) { + val blocks = remember(text) { splitMarkdown(text) } + val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow + + Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + for (b in blocks) { + when (b) { + is ChatMarkdownBlock.Text -> { + val trimmed = b.text.trimEnd() + if (trimmed.isEmpty()) continue + Text( + text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg), + style = MaterialTheme.typography.bodyMedium, + color = textColor, + ) + } + is ChatMarkdownBlock.Code -> { + SelectionContainer(modifier = Modifier.fillMaxWidth()) { + ChatCodeBlock(code = b.code, language = b.language) + } + } + is ChatMarkdownBlock.InlineImage -> { + InlineBase64Image(base64 = b.base64, mimeType = b.mimeType) + } + } + } + } +} + +private sealed interface ChatMarkdownBlock { + data class Text(val text: String) : ChatMarkdownBlock + data class Code(val code: String, val language: String?) : ChatMarkdownBlock + data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock +} + +private fun splitMarkdown(raw: String): List { + if (raw.isEmpty()) return emptyList() + + val out = ArrayList() + var idx = 0 + while (idx < raw.length) { + val fenceStart = raw.indexOf("```", startIndex = idx) + if (fenceStart < 0) { + out.addAll(splitInlineImages(raw.substring(idx))) + break + } + + if (fenceStart > idx) { + out.addAll(splitInlineImages(raw.substring(idx, fenceStart))) + } + + val langLineStart = fenceStart + 3 + val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it } + val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null } + + val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd + val fenceEnd = raw.indexOf("```", startIndex = codeStart) + if (fenceEnd < 0) { + out.addAll(splitInlineImages(raw.substring(fenceStart))) + break + } + val code = raw.substring(codeStart, fenceEnd) + out.add(ChatMarkdownBlock.Code(code = code, language = language)) + + idx = fenceEnd + 3 + } + + return out +} + +private fun splitInlineImages(text: String): List { + if (text.isEmpty()) return emptyList() + val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)") + val out = ArrayList() + + var idx = 0 + while (idx < text.length) { + val m = regex.find(text, startIndex = idx) ?: break + val start = m.range.first + val end = m.range.last + 1 + if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start))) + + val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png") + val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty() + if (b64.isNotEmpty()) { + out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64)) + } + idx = end + } + + if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx))) + return out +} + +private fun parseInlineMarkdown(text: String, inlineCodeBg: androidx.compose.ui.graphics.Color): AnnotatedString { + if (text.isEmpty()) return AnnotatedString("") + + val out = buildAnnotatedString { + var i = 0 + while (i < text.length) { + if (text.startsWith("**", startIndex = i)) { + val end = text.indexOf("**", startIndex = i + 2) + if (end > i + 2) { + withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { + append(text.substring(i + 2, end)) + } + i = end + 2 + continue + } + } + + if (text[i] == '`') { + val end = text.indexOf('`', startIndex = i + 1) + if (end > i + 1) { + withStyle( + SpanStyle( + fontFamily = FontFamily.Monospace, + background = inlineCodeBg, + ), + ) { + append(text.substring(i + 1, end)) + } + i = end + 1 + continue + } + } + + if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) { + val end = text.indexOf('*', startIndex = i + 1) + if (end > i + 1) { + withStyle(SpanStyle(fontStyle = FontStyle.Italic)) { + append(text.substring(i + 1, end)) + } + i = end + 1 + continue + } + } + + append(text[i]) + i += 1 + } + } + return out +} + +@Composable +private fun InlineBase64Image(base64: String, mimeType: String?) { + var image by remember(base64) { mutableStateOf(null) } + var failed by remember(base64) { mutableStateOf(false) } + + LaunchedEffect(base64) { + failed = false + image = + withContext(Dispatchers.Default) { + try { + val bytes = Base64.decode(base64, Base64.DEFAULT) + val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null + bitmap.asImageBitmap() + } catch (_: Throwable) { + null + } + } + if (image == null) failed = true + } + + if (image != null) { + Image( + bitmap = image!!, + contentDescription = mimeType ?: "image", + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxWidth(), + ) + } else if (failed) { + Text( + text = "Image unavailable", + modifier = Modifier.padding(vertical = 2.dp), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageListCard.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageListCard.kt new file mode 100644 index 00000000000..1091de6c802 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageListCard.kt @@ -0,0 +1,111 @@ +package bot.molt.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowCircleDown +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.unit.dp +import bot.molt.android.chat.ChatMessage +import bot.molt.android.chat.ChatPendingToolCall + +@Composable +fun ChatMessageListCard( + messages: List, + pendingRunCount: Int, + pendingToolCalls: List, + streamingAssistantText: String?, + modifier: Modifier = Modifier, +) { + val listState = rememberLazyListState() + + LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) { + val total = + messages.size + + (if (pendingRunCount > 0) 1 else 0) + + (if (pendingToolCalls.isNotEmpty()) 1 else 0) + + (if (!streamingAssistantText.isNullOrBlank()) 1 else 0) + if (total <= 0) return@LaunchedEffect + listState.animateScrollToItem(index = total - 1) + } + + Card( + modifier = modifier.fillMaxWidth(), + shape = MaterialTheme.shapes.large, + colors = + CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceContainer, + ), + elevation = CardDefaults.cardElevation(defaultElevation = 0.dp), + ) { + Box(modifier = Modifier.fillMaxSize()) { + LazyColumn( + modifier = Modifier.fillMaxSize(), + state = listState, + verticalArrangement = Arrangement.spacedBy(14.dp), + contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp), + ) { + items(count = messages.size, key = { idx -> messages[idx].id }) { idx -> + ChatMessageBubble(message = messages[idx]) + } + + if (pendingRunCount > 0) { + item(key = "typing") { + ChatTypingIndicatorBubble() + } + } + + if (pendingToolCalls.isNotEmpty()) { + item(key = "tools") { + ChatPendingToolsBubble(toolCalls = pendingToolCalls) + } + } + + val stream = streamingAssistantText?.trim() + if (!stream.isNullOrEmpty()) { + item(key = "stream") { + ChatStreamingAssistantBubble(text = stream) + } + } + } + + if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) { + EmptyChatHint(modifier = Modifier.align(Alignment.Center)) + } + } + } +} + +@Composable +private fun EmptyChatHint(modifier: Modifier = Modifier) { + Row( + modifier = modifier.alpha(0.7f), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + imageVector = Icons.Default.ArrowCircleDown, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Text( + text = "Message Clawd…", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageViews.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageViews.kt new file mode 100644 index 00000000000..59445be374d --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatMessageViews.kt @@ -0,0 +1,252 @@ +package bot.molt.android.ui.chat + +import android.graphics.BitmapFactory +import android.util.Base64 +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.unit.dp +import androidx.compose.foundation.Image +import bot.molt.android.chat.ChatMessage +import bot.molt.android.chat.ChatMessageContent +import bot.molt.android.chat.ChatPendingToolCall +import bot.molt.android.tools.ToolDisplayRegistry +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import androidx.compose.ui.platform.LocalContext + +@Composable +fun ChatMessageBubble(message: ChatMessage) { + val isUser = message.role.lowercase() == "user" + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start, + ) { + Surface( + shape = RoundedCornerShape(16.dp), + tonalElevation = 0.dp, + shadowElevation = 0.dp, + color = Color.Transparent, + modifier = Modifier.fillMaxWidth(0.92f), + ) { + Box( + modifier = + Modifier + .background(bubbleBackground(isUser)) + .padding(horizontal = 12.dp, vertical = 10.dp), + ) { + val textColor = textColorOverBubble(isUser) + ChatMessageBody(content = message.content, textColor = textColor) + } + } + } +} + +@Composable +private fun ChatMessageBody(content: List, textColor: Color) { + Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { + for (part in content) { + when (part.type) { + "text" -> { + val text = part.text ?: continue + ChatMarkdown(text = text, textColor = textColor) + } + else -> { + val b64 = part.base64 ?: continue + ChatBase64Image(base64 = b64, mimeType = part.mimeType) + } + } + } + } +} + +@Composable +fun ChatTypingIndicatorBubble() { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + DotPulse() + Text("Thinking…", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } +} + +@Composable +fun ChatPendingToolsBubble(toolCalls: List) { + val context = LocalContext.current + val displays = + remember(toolCalls, context) { + toolCalls.map { ToolDisplayRegistry.resolve(context, it.name, it.args) } + } + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Column(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) { + Text("Running tools…", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurface) + for (display in displays.take(6)) { + Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { + Text( + "${display.emoji} ${display.label}", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontFamily = FontFamily.Monospace, + ) + display.detailLine?.let { detail -> + Text( + detail, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontFamily = FontFamily.Monospace, + ) + } + } + } + if (toolCalls.size > 6) { + Text( + "… +${toolCalls.size - 6} more", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + } + } +} + +@Composable +fun ChatStreamingAssistantBubble(text: String) { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surfaceContainer, + ) { + Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) { + ChatMarkdown(text = text, textColor = MaterialTheme.colorScheme.onSurface) + } + } + } +} + +@Composable +private fun bubbleBackground(isUser: Boolean): Brush { + return if (isUser) { + Brush.linearGradient( + colors = listOf(MaterialTheme.colorScheme.primary, MaterialTheme.colorScheme.primary.copy(alpha = 0.78f)), + ) + } else { + Brush.linearGradient( + colors = listOf(MaterialTheme.colorScheme.surfaceContainer, MaterialTheme.colorScheme.surfaceContainerHigh), + ) + } +} + +@Composable +private fun textColorOverBubble(isUser: Boolean): Color { + return if (isUser) { + MaterialTheme.colorScheme.onPrimary + } else { + MaterialTheme.colorScheme.onSurface + } +} + +@Composable +private fun ChatBase64Image(base64: String, mimeType: String?) { + var image by remember(base64) { mutableStateOf(null) } + var failed by remember(base64) { mutableStateOf(false) } + + LaunchedEffect(base64) { + failed = false + image = + withContext(Dispatchers.Default) { + try { + val bytes = Base64.decode(base64, Base64.DEFAULT) + val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null + bitmap.asImageBitmap() + } catch (_: Throwable) { + null + } + } + if (image == null) failed = true + } + + if (image != null) { + Image( + bitmap = image!!, + contentDescription = mimeType ?: "attachment", + contentScale = ContentScale.Fit, + modifier = Modifier.fillMaxWidth(), + ) + } else if (failed) { + Text("Unsupported attachment", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant) + } +} + +@Composable +private fun DotPulse() { + Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) { + PulseDot(alpha = 0.38f) + PulseDot(alpha = 0.62f) + PulseDot(alpha = 0.90f) + } +} + +@Composable +private fun PulseDot(alpha: Float) { + Surface( + modifier = Modifier.size(6.dp).alpha(alpha), + shape = CircleShape, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) {} +} + +@Composable +fun ChatCodeBlock(code: String, language: String?) { + Surface( + shape = RoundedCornerShape(12.dp), + color = MaterialTheme.colorScheme.surfaceContainerLowest, + modifier = Modifier.fillMaxWidth(), + ) { + Text( + text = code.trimEnd(), + modifier = Modifier.padding(10.dp), + fontFamily = FontFamily.Monospace, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurface, + ) + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSessionsDialog.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSessionsDialog.kt new file mode 100644 index 00000000000..377a13daa6e --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSessionsDialog.kt @@ -0,0 +1,92 @@ +package bot.molt.android.ui.chat + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Refresh +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.FilledTonalIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import bot.molt.android.chat.ChatSessionEntry + +@Composable +fun ChatSessionsDialog( + currentSessionKey: String, + sessions: List, + onDismiss: () -> Unit, + onRefresh: () -> Unit, + onSelect: (sessionKey: String) -> Unit, +) { + AlertDialog( + onDismissRequest = onDismiss, + confirmButton = {}, + title = { + Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) { + Text("Sessions", style = MaterialTheme.typography.titleMedium) + Spacer(modifier = Modifier.weight(1f)) + FilledTonalIconButton(onClick = onRefresh) { + Icon(Icons.Default.Refresh, contentDescription = "Refresh") + } + } + }, + text = { + if (sessions.isEmpty()) { + Text("No sessions", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } else { + LazyColumn(verticalArrangement = Arrangement.spacedBy(8.dp)) { + items(sessions, key = { it.key }) { entry -> + SessionRow( + entry = entry, + isCurrent = entry.key == currentSessionKey, + onClick = { onSelect(entry.key) }, + ) + } + } + } + }, + ) +} + +@Composable +private fun SessionRow( + entry: ChatSessionEntry, + isCurrent: Boolean, + onClick: () -> Unit, +) { + Surface( + onClick = onClick, + shape = MaterialTheme.shapes.medium, + color = + if (isCurrent) { + MaterialTheme.colorScheme.primary.copy(alpha = 0.14f) + } else { + MaterialTheme.colorScheme.surfaceContainer + }, + modifier = Modifier.fillMaxWidth(), + ) { + Row( + modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(10.dp), + ) { + Text(entry.displayName ?: entry.key, style = MaterialTheme.typography.bodyMedium) + Spacer(modifier = Modifier.weight(1f)) + if (isCurrent) { + Text("Current", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSheetContent.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSheetContent.kt new file mode 100644 index 00000000000..5632be70fe5 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/ChatSheetContent.kt @@ -0,0 +1,147 @@ +package bot.molt.android.ui.chat + +import android.content.ContentResolver +import android.net.Uri +import android.util.Base64 +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.unit.dp +import bot.molt.android.MainViewModel +import bot.molt.android.chat.OutgoingAttachment +import java.io.ByteArrayOutputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +@Composable +fun ChatSheetContent(viewModel: MainViewModel) { + val messages by viewModel.chatMessages.collectAsState() + val errorText by viewModel.chatError.collectAsState() + val pendingRunCount by viewModel.pendingRunCount.collectAsState() + val healthOk by viewModel.chatHealthOk.collectAsState() + val sessionKey by viewModel.chatSessionKey.collectAsState() + val mainSessionKey by viewModel.mainSessionKey.collectAsState() + val thinkingLevel by viewModel.chatThinkingLevel.collectAsState() + val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState() + val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState() + val sessions by viewModel.chatSessions.collectAsState() + + LaunchedEffect(mainSessionKey) { + viewModel.loadChat(mainSessionKey) + viewModel.refreshChatSessions(limit = 200) + } + + val context = LocalContext.current + val resolver = context.contentResolver + val scope = rememberCoroutineScope() + + val attachments = remember { mutableStateListOf() } + + val pickImages = + rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris -> + if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult + scope.launch(Dispatchers.IO) { + val next = + uris.take(8).mapNotNull { uri -> + try { + loadImageAttachment(resolver, uri) + } catch (_: Throwable) { + null + } + } + withContext(Dispatchers.Main) { + attachments.addAll(next) + } + } + } + + Column( + modifier = + Modifier + .fillMaxSize() + .padding(horizontal = 12.dp, vertical = 12.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + ) { + ChatMessageListCard( + messages = messages, + pendingRunCount = pendingRunCount, + pendingToolCalls = pendingToolCalls, + streamingAssistantText = streamingAssistantText, + modifier = Modifier.weight(1f, fill = true), + ) + + ChatComposer( + sessionKey = sessionKey, + sessions = sessions, + mainSessionKey = mainSessionKey, + healthOk = healthOk, + thinkingLevel = thinkingLevel, + pendingRunCount = pendingRunCount, + errorText = errorText, + attachments = attachments, + onPickImages = { pickImages.launch("image/*") }, + onRemoveAttachment = { id -> attachments.removeAll { it.id == id } }, + onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) }, + onSelectSession = { key -> viewModel.switchChatSession(key) }, + onRefresh = { + viewModel.refreshChat() + viewModel.refreshChatSessions(limit = 200) + }, + onAbort = { viewModel.abortChat() }, + onSend = { text -> + val outgoing = + attachments.map { att -> + OutgoingAttachment( + type = "image", + mimeType = att.mimeType, + fileName = att.fileName, + base64 = att.base64, + ) + } + viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing) + attachments.clear() + }, + ) + } +} + +data class PendingImageAttachment( + val id: String, + val fileName: String, + val mimeType: String, + val base64: String, +) + +private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment { + val mimeType = resolver.getType(uri) ?: "image/*" + val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/') + val bytes = + withContext(Dispatchers.IO) { + resolver.openInputStream(uri)?.use { input -> + val out = ByteArrayOutputStream() + input.copyTo(out) + out.toByteArray() + } ?: ByteArray(0) + } + if (bytes.isEmpty()) throw IllegalStateException("empty attachment") + val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) + return PendingImageAttachment( + id = uri.toString() + "#" + System.currentTimeMillis().toString(), + fileName = fileName, + mimeType = mimeType, + base64 = base64, + ) +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/chat/SessionFilters.kt b/apps/android/app/src/main/java/bot/molt/android/ui/chat/SessionFilters.kt new file mode 100644 index 00000000000..227fb0a02a7 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/ui/chat/SessionFilters.kt @@ -0,0 +1,49 @@ +package bot.molt.android.ui.chat + +import bot.molt.android.chat.ChatSessionEntry + +private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L + +fun resolveSessionChoices( + currentSessionKey: String, + sessions: List, + mainSessionKey: String, + nowMs: Long = System.currentTimeMillis(), +): List { + val mainKey = mainSessionKey.trim().ifEmpty { "main" } + val current = currentSessionKey.trim().let { if (it == "main" && mainKey != "main") mainKey else it } + val aliasKey = if (mainKey == "main") null else "main" + val cutoff = nowMs - RECENT_WINDOW_MS + val sorted = sessions.sortedByDescending { it.updatedAtMs ?: 0L } + val recent = mutableListOf() + val seen = mutableSetOf() + for (entry in sorted) { + if (aliasKey != null && entry.key == aliasKey) continue + if (!seen.add(entry.key)) continue + if ((entry.updatedAtMs ?: 0L) < cutoff) continue + recent.add(entry) + } + + val result = mutableListOf() + val included = mutableSetOf() + val mainEntry = sorted.firstOrNull { it.key == mainKey } + if (mainEntry != null) { + result.add(mainEntry) + included.add(mainKey) + } else if (current == mainKey) { + result.add(ChatSessionEntry(key = mainKey, updatedAtMs = null)) + included.add(mainKey) + } + + for (entry in recent) { + if (included.add(entry.key)) { + result.add(entry) + } + } + + if (current.isNotEmpty() && !included.contains(current)) { + result.add(ChatSessionEntry(key = current, updatedAtMs = null)) + } + + return result +} diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/StreamingMediaDataSource.kt b/apps/android/app/src/main/java/bot/molt/android/voice/StreamingMediaDataSource.kt new file mode 100644 index 00000000000..7a7f61165f2 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/voice/StreamingMediaDataSource.kt @@ -0,0 +1,98 @@ +package bot.molt.android.voice + +import android.media.MediaDataSource +import kotlin.math.min + +internal class StreamingMediaDataSource : MediaDataSource() { + private data class Chunk(val start: Long, val data: ByteArray) + + private val lock = Object() + private val chunks = ArrayList() + private var totalSize: Long = 0 + private var closed = false + private var finished = false + private var lastReadIndex = 0 + + fun append(data: ByteArray) { + if (data.isEmpty()) return + synchronized(lock) { + if (closed || finished) return + val chunk = Chunk(totalSize, data) + chunks.add(chunk) + totalSize += data.size.toLong() + lock.notifyAll() + } + } + + fun finish() { + synchronized(lock) { + if (closed) return + finished = true + lock.notifyAll() + } + } + + fun fail() { + synchronized(lock) { + closed = true + lock.notifyAll() + } + } + + override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { + if (position < 0) return -1 + synchronized(lock) { + while (!closed && !finished && position >= totalSize) { + lock.wait() + } + if (closed) return -1 + if (position >= totalSize && finished) return -1 + + val available = (totalSize - position).toInt() + val toRead = min(size, available) + var remaining = toRead + var destOffset = offset + var pos = position + + var index = findChunkIndex(pos) + while (remaining > 0 && index < chunks.size) { + val chunk = chunks[index] + val inChunkOffset = (pos - chunk.start).toInt() + if (inChunkOffset >= chunk.data.size) { + index++ + continue + } + val copyLen = min(remaining, chunk.data.size - inChunkOffset) + System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen) + remaining -= copyLen + destOffset += copyLen + pos += copyLen + if (inChunkOffset + copyLen >= chunk.data.size) { + index++ + } + } + + return toRead - remaining + } + } + + override fun getSize(): Long = -1 + + override fun close() { + synchronized(lock) { + closed = true + lock.notifyAll() + } + } + + private fun findChunkIndex(position: Long): Int { + var index = lastReadIndex + while (index < chunks.size) { + val chunk = chunks[index] + if (position < chunk.start + chunk.data.size) break + index++ + } + lastReadIndex = index + return index + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/TalkDirectiveParser.kt b/apps/android/app/src/main/java/bot/molt/android/voice/TalkDirectiveParser.kt new file mode 100644 index 00000000000..0d969e4d144 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/voice/TalkDirectiveParser.kt @@ -0,0 +1,191 @@ +package bot.molt.android.voice + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +private val directiveJson = Json { ignoreUnknownKeys = true } + +data class TalkDirective( + val voiceId: String? = null, + val modelId: String? = null, + val speed: Double? = null, + val rateWpm: Int? = null, + val stability: Double? = null, + val similarity: Double? = null, + val style: Double? = null, + val speakerBoost: Boolean? = null, + val seed: Long? = null, + val normalize: String? = null, + val language: String? = null, + val outputFormat: String? = null, + val latencyTier: Int? = null, + val once: Boolean? = null, +) + +data class TalkDirectiveParseResult( + val directive: TalkDirective?, + val stripped: String, + val unknownKeys: List, +) + +object TalkDirectiveParser { + fun parse(text: String): TalkDirectiveParseResult { + val normalized = text.replace("\r\n", "\n") + val lines = normalized.split("\n").toMutableList() + if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList()) + + val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() } + if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList()) + + val head = lines[firstNonEmpty].trim() + if (!head.startsWith("{") || !head.endsWith("}")) { + return TalkDirectiveParseResult(null, text, emptyList()) + } + + val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList()) + + val speakerBoost = + boolValue(obj, listOf("speaker_boost", "speakerBoost")) + ?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not() + + val directive = TalkDirective( + voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")), + modelId = stringValue(obj, listOf("model", "model_id", "modelId")), + speed = doubleValue(obj, listOf("speed")), + rateWpm = intValue(obj, listOf("rate", "wpm")), + stability = doubleValue(obj, listOf("stability")), + similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")), + style = doubleValue(obj, listOf("style")), + speakerBoost = speakerBoost, + seed = longValue(obj, listOf("seed")), + normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")), + language = stringValue(obj, listOf("lang", "language_code", "language")), + outputFormat = stringValue(obj, listOf("output_format", "format")), + latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")), + once = boolValue(obj, listOf("once")), + ) + + val hasDirective = listOf( + directive.voiceId, + directive.modelId, + directive.speed, + directive.rateWpm, + directive.stability, + directive.similarity, + directive.style, + directive.speakerBoost, + directive.seed, + directive.normalize, + directive.language, + directive.outputFormat, + directive.latencyTier, + directive.once, + ).any { it != null } + + if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList()) + + val knownKeys = setOf( + "voice", "voice_id", "voiceid", + "model", "model_id", "modelid", + "speed", "rate", "wpm", + "stability", "similarity", "similarity_boost", "similarityboost", + "style", + "speaker_boost", "speakerboost", + "no_speaker_boost", "nospeakerboost", + "seed", + "normalize", "apply_text_normalization", + "lang", "language_code", "language", + "output_format", "format", + "latency", "latency_tier", "latencytier", + "once", + ) + val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted() + + lines.removeAt(firstNonEmpty) + if (firstNonEmpty < lines.size) { + if (lines[firstNonEmpty].trim().isEmpty()) { + lines.removeAt(firstNonEmpty) + } + } + + return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys) + } + + private fun parseJsonObject(line: String): JsonObject? { + return try { + directiveJson.parseToJsonElement(line) as? JsonObject + } catch (_: Throwable) { + null + } + } + + private fun stringValue(obj: JsonObject, keys: List): String? { + for (key in keys) { + val value = obj[key].asStringOrNull()?.trim() + if (!value.isNullOrEmpty()) return value + } + return null + } + + private fun doubleValue(obj: JsonObject, keys: List): Double? { + for (key in keys) { + val value = obj[key].asDoubleOrNull() + if (value != null) return value + } + return null + } + + private fun intValue(obj: JsonObject, keys: List): Int? { + for (key in keys) { + val value = obj[key].asIntOrNull() + if (value != null) return value + } + return null + } + + private fun longValue(obj: JsonObject, keys: List): Long? { + for (key in keys) { + val value = obj[key].asLongOrNull() + if (value != null) return value + } + return null + } + + private fun boolValue(obj: JsonObject, keys: List): Boolean? { + for (key in keys) { + val value = obj[key].asBooleanOrNull() + if (value != null) return value + } + return null + } +} + +private fun JsonElement?.asStringOrNull(): String? = + (this as? JsonPrimitive)?.takeIf { it.isString }?.content + +private fun JsonElement?.asDoubleOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toDoubleOrNull() +} + +private fun JsonElement?.asIntOrNull(): Int? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toIntOrNull() +} + +private fun JsonElement?.asLongOrNull(): Long? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toLongOrNull() +} + +private fun JsonElement?.asBooleanOrNull(): Boolean? { + val primitive = this as? JsonPrimitive ?: return null + val content = primitive.content.trim().lowercase() + return when (content) { + "true", "yes", "1" -> true + "false", "no", "0" -> false + else -> null + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt b/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt new file mode 100644 index 00000000000..f050f8bd240 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt @@ -0,0 +1,1257 @@ +package bot.molt.android.voice + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioTrack +import android.media.MediaPlayer +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import android.util.Log +import androidx.core.content.ContextCompat +import bot.molt.android.gateway.GatewaySession +import bot.molt.android.isCanonicalMainSessionKey +import bot.molt.android.normalizeMainKey +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonObject +import kotlin.math.max + +class TalkModeManager( + private val context: Context, + private val scope: CoroutineScope, + private val session: GatewaySession, + private val supportsChatSubscribe: Boolean, + private val isConnected: () -> Boolean, +) { + companion object { + private const val tag = "TalkMode" + private const val defaultModelIdFallback = "eleven_v3" + private const val defaultOutputFormatFallback = "pcm_24000" + } + + private val mainHandler = Handler(Looper.getMainLooper()) + private val json = Json { ignoreUnknownKeys = true } + + private val _isEnabled = MutableStateFlow(false) + val isEnabled: StateFlow = _isEnabled + + private val _isListening = MutableStateFlow(false) + val isListening: StateFlow = _isListening + + private val _isSpeaking = MutableStateFlow(false) + val isSpeaking: StateFlow = _isSpeaking + + private val _statusText = MutableStateFlow("Off") + val statusText: StateFlow = _statusText + + private val _lastAssistantText = MutableStateFlow(null) + val lastAssistantText: StateFlow = _lastAssistantText + + private val _usingFallbackTts = MutableStateFlow(false) + val usingFallbackTts: StateFlow = _usingFallbackTts + + private var recognizer: SpeechRecognizer? = null + private var restartJob: Job? = null + private var stopRequested = false + private var listeningMode = false + + private var silenceJob: Job? = null + private val silenceWindowMs = 700L + private var lastTranscript: String = "" + private var lastHeardAtMs: Long? = null + private var lastSpokenText: String? = null + private var lastInterruptedAtSeconds: Double? = null + + private var defaultVoiceId: String? = null + private var currentVoiceId: String? = null + private var fallbackVoiceId: String? = null + private var defaultModelId: String? = null + private var currentModelId: String? = null + private var defaultOutputFormat: String? = null + private var apiKey: String? = null + private var voiceAliases: Map = emptyMap() + private var interruptOnSpeech: Boolean = true + private var voiceOverrideActive = false + private var modelOverrideActive = false + private var mainSessionKey: String = "main" + + private var pendingRunId: String? = null + private var pendingFinal: CompletableDeferred? = null + private var chatSubscribedSessionKey: String? = null + + private var player: MediaPlayer? = null + private var streamingSource: StreamingMediaDataSource? = null + private var pcmTrack: AudioTrack? = null + @Volatile private var pcmStopRequested = false + private var systemTts: TextToSpeech? = null + private var systemTtsPending: CompletableDeferred? = null + private var systemTtsPendingId: String? = null + + fun setMainSessionKey(sessionKey: String?) { + val trimmed = sessionKey?.trim().orEmpty() + if (trimmed.isEmpty()) return + if (isCanonicalMainSessionKey(mainSessionKey)) return + mainSessionKey = trimmed + } + + fun setEnabled(enabled: Boolean) { + if (_isEnabled.value == enabled) return + _isEnabled.value = enabled + if (enabled) { + Log.d(tag, "enabled") + start() + } else { + Log.d(tag, "disabled") + stop() + } + } + + fun handleGatewayEvent(event: String, payloadJson: String?) { + if (event != "chat") return + if (payloadJson.isNullOrBlank()) return + val pending = pendingRunId ?: return + val obj = + try { + json.parseToJsonElement(payloadJson).asObjectOrNull() + } catch (_: Throwable) { + null + } ?: return + val runId = obj["runId"].asStringOrNull() ?: return + if (runId != pending) return + val state = obj["state"].asStringOrNull() ?: return + if (state == "final") { + pendingFinal?.complete(true) + pendingFinal = null + pendingRunId = null + } + } + + private fun start() { + mainHandler.post { + if (_isListening.value) return@post + stopRequested = false + listeningMode = true + Log.d(tag, "start") + + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + _statusText.value = "Speech recognizer unavailable" + Log.w(tag, "speech recognizer unavailable") + return@post + } + + val micOk = + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED + if (!micOk) { + _statusText.value = "Microphone permission required" + Log.w(tag, "microphone permission required") + return@post + } + + try { + recognizer?.destroy() + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + startListeningInternal(markListening = true) + startSilenceMonitor() + Log.d(tag, "listening") + } catch (err: Throwable) { + _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" + Log.w(tag, "start failed: ${err.message ?: err::class.simpleName}") + } + } + } + + private fun stop() { + stopRequested = true + listeningMode = false + restartJob?.cancel() + restartJob = null + silenceJob?.cancel() + silenceJob = null + lastTranscript = "" + lastHeardAtMs = null + _isListening.value = false + _statusText.value = "Off" + stopSpeaking() + _usingFallbackTts.value = false + chatSubscribedSessionKey = null + + mainHandler.post { + recognizer?.cancel() + recognizer?.destroy() + recognizer = null + } + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + } + + private fun startListeningInternal(markListening: Boolean) { + val r = recognizer ?: return + val intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + } + + if (markListening) { + _statusText.value = "Listening" + _isListening.value = true + } + r.startListening(intent) + } + + private fun scheduleRestart(delayMs: Long = 350) { + if (stopRequested) return + restartJob?.cancel() + restartJob = + scope.launch { + delay(delayMs) + mainHandler.post { + if (stopRequested) return@post + try { + recognizer?.cancel() + val shouldListen = listeningMode + val shouldInterrupt = _isSpeaking.value && interruptOnSpeech + if (!shouldListen && !shouldInterrupt) return@post + startListeningInternal(markListening = shouldListen) + } catch (_: Throwable) { + // handled by onError + } + } + } + } + + private fun handleTranscript(text: String, isFinal: Boolean) { + val trimmed = text.trim() + if (_isSpeaking.value && interruptOnSpeech) { + if (shouldInterrupt(trimmed)) { + stopSpeaking() + } + return + } + + if (!_isListening.value) return + + if (trimmed.isNotEmpty()) { + lastTranscript = trimmed + lastHeardAtMs = SystemClock.elapsedRealtime() + } + + if (isFinal) { + lastTranscript = trimmed + } + } + + private fun startSilenceMonitor() { + silenceJob?.cancel() + silenceJob = + scope.launch { + while (_isEnabled.value) { + delay(200) + checkSilence() + } + } + } + + private fun checkSilence() { + if (!_isListening.value) return + val transcript = lastTranscript.trim() + if (transcript.isEmpty()) return + val lastHeard = lastHeardAtMs ?: return + val elapsed = SystemClock.elapsedRealtime() - lastHeard + if (elapsed < silenceWindowMs) return + scope.launch { finalizeTranscript(transcript) } + } + + private suspend fun finalizeTranscript(transcript: String) { + listeningMode = false + _isListening.value = false + _statusText.value = "Thinking…" + lastTranscript = "" + lastHeardAtMs = null + + reloadConfig() + val prompt = buildPrompt(transcript) + if (!isConnected()) { + _statusText.value = "Gateway not connected" + Log.w(tag, "finalize: gateway not connected") + start() + return + } + + try { + val startedAt = System.currentTimeMillis().toDouble() / 1000.0 + subscribeChatIfNeeded(session = session, sessionKey = mainSessionKey) + Log.d(tag, "chat.send start sessionKey=${mainSessionKey.ifBlank { "main" }} chars=${prompt.length}") + val runId = sendChat(prompt, session) + Log.d(tag, "chat.send ok runId=$runId") + val ok = waitForChatFinal(runId) + if (!ok) { + Log.w(tag, "chat final timeout runId=$runId; attempting history fallback") + } + val assistant = waitForAssistantText(session, startedAt, if (ok) 12_000 else 25_000) + if (assistant.isNullOrBlank()) { + _statusText.value = "No reply" + Log.w(tag, "assistant text timeout runId=$runId") + start() + return + } + Log.d(tag, "assistant text ok chars=${assistant.length}") + playAssistant(assistant) + } catch (err: Throwable) { + _statusText.value = "Talk failed: ${err.message ?: err::class.simpleName}" + Log.w(tag, "finalize failed: ${err.message ?: err::class.simpleName}") + } + + if (_isEnabled.value) { + start() + } + } + + private suspend fun subscribeChatIfNeeded(session: GatewaySession, sessionKey: String) { + if (!supportsChatSubscribe) return + val key = sessionKey.trim() + if (key.isEmpty()) return + if (chatSubscribedSessionKey == key) return + try { + session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") + chatSubscribedSessionKey = key + Log.d(tag, "chat.subscribe ok sessionKey=$key") + } catch (err: Throwable) { + Log.w(tag, "chat.subscribe failed sessionKey=$key err=${err.message ?: err::class.java.simpleName}") + } + } + + private fun buildPrompt(transcript: String): String { + val lines = mutableListOf( + "Talk Mode active. Reply in a concise, spoken tone.", + "You may optionally prefix the response with JSON (first line) to set ElevenLabs voice (id or alias), e.g. {\"voice\":\"\",\"once\":true}.", + ) + lastInterruptedAtSeconds?.let { + lines.add("Assistant speech interrupted at ${"%.1f".format(it)}s.") + lastInterruptedAtSeconds = null + } + lines.add("") + lines.add(transcript) + return lines.joinToString("\n") + } + + private suspend fun sendChat(message: String, session: GatewaySession): String { + val runId = UUID.randomUUID().toString() + val params = + buildJsonObject { + put("sessionKey", JsonPrimitive(mainSessionKey.ifBlank { "main" })) + put("message", JsonPrimitive(message)) + put("thinking", JsonPrimitive("low")) + put("timeoutMs", JsonPrimitive(30_000)) + put("idempotencyKey", JsonPrimitive(runId)) + } + val res = session.request("chat.send", params.toString()) + val parsed = parseRunId(res) ?: runId + if (parsed != runId) { + pendingRunId = parsed + } + return parsed + } + + private suspend fun waitForChatFinal(runId: String): Boolean { + pendingFinal?.cancel() + val deferred = CompletableDeferred() + pendingRunId = runId + pendingFinal = deferred + + val result = + withContext(Dispatchers.IO) { + try { + kotlinx.coroutines.withTimeout(120_000) { deferred.await() } + } catch (_: Throwable) { + false + } + } + + if (!result) { + pendingFinal = null + pendingRunId = null + } + return result + } + + private suspend fun waitForAssistantText( + session: GatewaySession, + sinceSeconds: Double, + timeoutMs: Long, + ): String? { + val deadline = SystemClock.elapsedRealtime() + timeoutMs + while (SystemClock.elapsedRealtime() < deadline) { + val text = fetchLatestAssistantText(session, sinceSeconds) + if (!text.isNullOrBlank()) return text + delay(300) + } + return null + } + + private suspend fun fetchLatestAssistantText( + session: GatewaySession, + sinceSeconds: Double? = null, + ): String? { + val key = mainSessionKey.ifBlank { "main" } + val res = session.request("chat.history", "{\"sessionKey\":\"$key\"}") + val root = json.parseToJsonElement(res).asObjectOrNull() ?: return null + val messages = root["messages"] as? JsonArray ?: return null + for (item in messages.reversed()) { + val obj = item.asObjectOrNull() ?: continue + if (obj["role"].asStringOrNull() != "assistant") continue + if (sinceSeconds != null) { + val timestamp = obj["timestamp"].asDoubleOrNull() + if (timestamp != null && !TalkModeRuntime.isMessageTimestampAfter(timestamp, sinceSeconds)) continue + } + val content = obj["content"] as? JsonArray ?: continue + val text = + content.mapNotNull { entry -> + entry.asObjectOrNull()?.get("text")?.asStringOrNull()?.trim() + }.filter { it.isNotEmpty() } + if (text.isNotEmpty()) return text.joinToString("\n") + } + return null + } + + private suspend fun playAssistant(text: String) { + val parsed = TalkDirectiveParser.parse(text) + if (parsed.unknownKeys.isNotEmpty()) { + Log.w(tag, "Unknown talk directive keys: ${parsed.unknownKeys}") + } + val directive = parsed.directive + val cleaned = parsed.stripped.trim() + if (cleaned.isEmpty()) return + _lastAssistantText.value = cleaned + + val requestedVoice = directive?.voiceId?.trim()?.takeIf { it.isNotEmpty() } + val resolvedVoice = resolveVoiceAlias(requestedVoice) + if (requestedVoice != null && resolvedVoice == null) { + Log.w(tag, "unknown voice alias: $requestedVoice") + } + + if (directive?.voiceId != null) { + if (directive.once != true) { + currentVoiceId = resolvedVoice + voiceOverrideActive = true + } + } + if (directive?.modelId != null) { + if (directive.once != true) { + currentModelId = directive.modelId + modelOverrideActive = true + } + } + + val apiKey = + apiKey?.trim()?.takeIf { it.isNotEmpty() } + ?: System.getenv("ELEVENLABS_API_KEY")?.trim() + val preferredVoice = resolvedVoice ?: currentVoiceId ?: defaultVoiceId + val voiceId = + if (!apiKey.isNullOrEmpty()) { + resolveVoiceId(preferredVoice, apiKey) + } else { + null + } + + _statusText.value = "Speaking…" + _isSpeaking.value = true + lastSpokenText = cleaned + ensureInterruptListener() + + try { + val canUseElevenLabs = !voiceId.isNullOrBlank() && !apiKey.isNullOrEmpty() + if (!canUseElevenLabs) { + if (voiceId.isNullOrBlank()) { + Log.w(tag, "missing voiceId; falling back to system voice") + } + if (apiKey.isNullOrEmpty()) { + Log.w(tag, "missing ELEVENLABS_API_KEY; falling back to system voice") + } + _usingFallbackTts.value = true + _statusText.value = "Speaking (System)…" + speakWithSystemTts(cleaned) + } else { + _usingFallbackTts.value = false + val ttsStarted = SystemClock.elapsedRealtime() + val modelId = directive?.modelId ?: currentModelId ?: defaultModelId + val request = + ElevenLabsRequest( + text = cleaned, + modelId = modelId, + outputFormat = + TalkModeRuntime.validatedOutputFormat(directive?.outputFormat ?: defaultOutputFormat), + speed = TalkModeRuntime.resolveSpeed(directive?.speed, directive?.rateWpm), + stability = TalkModeRuntime.validatedStability(directive?.stability, modelId), + similarity = TalkModeRuntime.validatedUnit(directive?.similarity), + style = TalkModeRuntime.validatedUnit(directive?.style), + speakerBoost = directive?.speakerBoost, + seed = TalkModeRuntime.validatedSeed(directive?.seed), + normalize = TalkModeRuntime.validatedNormalize(directive?.normalize), + language = TalkModeRuntime.validatedLanguage(directive?.language), + latencyTier = TalkModeRuntime.validatedLatencyTier(directive?.latencyTier), + ) + streamAndPlay(voiceId = voiceId!!, apiKey = apiKey!!, request = request) + Log.d(tag, "elevenlabs stream ok durMs=${SystemClock.elapsedRealtime() - ttsStarted}") + } + } catch (err: Throwable) { + Log.w(tag, "speak failed: ${err.message ?: err::class.simpleName}; falling back to system voice") + try { + _usingFallbackTts.value = true + _statusText.value = "Speaking (System)…" + speakWithSystemTts(cleaned) + } catch (fallbackErr: Throwable) { + _statusText.value = "Speak failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}" + Log.w(tag, "system voice failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}") + } + } + + _isSpeaking.value = false + } + + private suspend fun streamAndPlay(voiceId: String, apiKey: String, request: ElevenLabsRequest) { + stopSpeaking(resetInterrupt = false) + + pcmStopRequested = false + val pcmSampleRate = TalkModeRuntime.parsePcmSampleRate(request.outputFormat) + if (pcmSampleRate != null) { + try { + streamAndPlayPcm(voiceId = voiceId, apiKey = apiKey, request = request, sampleRate = pcmSampleRate) + return + } catch (err: Throwable) { + if (pcmStopRequested) return + Log.w(tag, "pcm playback failed; falling back to mp3: ${err.message ?: err::class.simpleName}") + } + } + + streamAndPlayMp3(voiceId = voiceId, apiKey = apiKey, request = request) + } + + private suspend fun streamAndPlayMp3(voiceId: String, apiKey: String, request: ElevenLabsRequest) { + val dataSource = StreamingMediaDataSource() + streamingSource = dataSource + + val player = MediaPlayer() + this.player = player + + val prepared = CompletableDeferred() + val finished = CompletableDeferred() + + player.setAudioAttributes( + AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .build(), + ) + player.setOnPreparedListener { + it.start() + prepared.complete(Unit) + } + player.setOnCompletionListener { + finished.complete(Unit) + } + player.setOnErrorListener { _, _, _ -> + finished.completeExceptionally(IllegalStateException("MediaPlayer error")) + true + } + + player.setDataSource(dataSource) + withContext(Dispatchers.Main) { + player.prepareAsync() + } + + val fetchError = CompletableDeferred() + val fetchJob = + scope.launch(Dispatchers.IO) { + try { + streamTts(voiceId = voiceId, apiKey = apiKey, request = request, sink = dataSource) + fetchError.complete(null) + } catch (err: Throwable) { + dataSource.fail() + fetchError.complete(err) + } + } + + Log.d(tag, "play start") + try { + prepared.await() + finished.await() + fetchError.await()?.let { throw it } + } finally { + fetchJob.cancel() + cleanupPlayer() + } + Log.d(tag, "play done") + } + + private suspend fun streamAndPlayPcm( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + sampleRate: Int, + ) { + val minBuffer = + AudioTrack.getMinBufferSize( + sampleRate, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + if (minBuffer <= 0) { + throw IllegalStateException("AudioTrack buffer size invalid: $minBuffer") + } + + val bufferSize = max(minBuffer * 2, 8 * 1024) + val track = + AudioTrack( + AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .build(), + AudioFormat.Builder() + .setSampleRate(sampleRate) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .build(), + bufferSize, + AudioTrack.MODE_STREAM, + AudioManager.AUDIO_SESSION_ID_GENERATE, + ) + if (track.state != AudioTrack.STATE_INITIALIZED) { + track.release() + throw IllegalStateException("AudioTrack init failed") + } + pcmTrack = track + track.play() + + Log.d(tag, "pcm play start sampleRate=$sampleRate bufferSize=$bufferSize") + try { + streamPcm(voiceId = voiceId, apiKey = apiKey, request = request, track = track) + } finally { + cleanupPcmTrack() + } + Log.d(tag, "pcm play done") + } + + private suspend fun speakWithSystemTts(text: String) { + val trimmed = text.trim() + if (trimmed.isEmpty()) return + val ok = ensureSystemTts() + if (!ok) { + throw IllegalStateException("system TTS unavailable") + } + + val tts = systemTts ?: throw IllegalStateException("system TTS unavailable") + val utteranceId = "talk-${UUID.randomUUID()}" + val deferred = CompletableDeferred() + systemTtsPending?.cancel() + systemTtsPending = deferred + systemTtsPendingId = utteranceId + + withContext(Dispatchers.Main) { + val params = Bundle() + tts.speak(trimmed, TextToSpeech.QUEUE_FLUSH, params, utteranceId) + } + + withContext(Dispatchers.IO) { + try { + kotlinx.coroutines.withTimeout(180_000) { deferred.await() } + } catch (err: Throwable) { + throw err + } + } + } + + private suspend fun ensureSystemTts(): Boolean { + if (systemTts != null) return true + return withContext(Dispatchers.Main) { + val deferred = CompletableDeferred() + val tts = + try { + TextToSpeech(context) { status -> + deferred.complete(status == TextToSpeech.SUCCESS) + } + } catch (_: Throwable) { + deferred.complete(false) + null + } + if (tts == null) return@withContext false + + tts.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + + override fun onDone(utteranceId: String?) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.complete(Unit) + systemTtsPending = null + systemTtsPendingId = null + } + + @Suppress("OVERRIDE_DEPRECATION") + @Deprecated("Deprecated in Java") + override fun onError(utteranceId: String?) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error")) + systemTtsPending = null + systemTtsPendingId = null + } + + override fun onError(utteranceId: String?, errorCode: Int) { + if (utteranceId == null) return + if (utteranceId != systemTtsPendingId) return + systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error $errorCode")) + systemTtsPending = null + systemTtsPendingId = null + } + }, + ) + + val ok = + try { + deferred.await() + } catch (_: Throwable) { + false + } + if (ok) { + systemTts = tts + } else { + tts.shutdown() + } + ok + } + } + + private fun stopSpeaking(resetInterrupt: Boolean = true) { + pcmStopRequested = true + if (!_isSpeaking.value) { + cleanupPlayer() + cleanupPcmTrack() + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + return + } + if (resetInterrupt) { + val currentMs = player?.currentPosition?.toDouble() ?: 0.0 + lastInterruptedAtSeconds = currentMs / 1000.0 + } + cleanupPlayer() + cleanupPcmTrack() + systemTts?.stop() + systemTtsPending?.cancel() + systemTtsPending = null + systemTtsPendingId = null + _isSpeaking.value = false + } + + private fun cleanupPlayer() { + player?.stop() + player?.release() + player = null + streamingSource?.close() + streamingSource = null + } + + private fun cleanupPcmTrack() { + val track = pcmTrack ?: return + try { + track.pause() + track.flush() + track.stop() + } catch (_: Throwable) { + // ignore cleanup errors + } finally { + track.release() + } + pcmTrack = null + } + + private fun shouldInterrupt(transcript: String): Boolean { + val trimmed = transcript.trim() + if (trimmed.length < 3) return false + val spoken = lastSpokenText?.lowercase() + if (spoken != null && spoken.contains(trimmed.lowercase())) return false + return true + } + + private suspend fun reloadConfig() { + val envVoice = System.getenv("ELEVENLABS_VOICE_ID")?.trim() + val sagVoice = System.getenv("SAG_VOICE_ID")?.trim() + val envKey = System.getenv("ELEVENLABS_API_KEY")?.trim() + try { + val res = session.request("config.get", "{}") + val root = json.parseToJsonElement(res).asObjectOrNull() + val config = root?.get("config").asObjectOrNull() + val talk = config?.get("talk").asObjectOrNull() + val sessionCfg = config?.get("session").asObjectOrNull() + val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) + val voice = talk?.get("voiceId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val aliases = + talk?.get("voiceAliases").asObjectOrNull()?.entries?.mapNotNull { (key, value) -> + val id = value.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + normalizeAliasKey(key).takeIf { it.isNotEmpty() }?.let { it to id } + }?.toMap().orEmpty() + val model = talk?.get("modelId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val outputFormat = talk?.get("outputFormat")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val key = talk?.get("apiKey")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } + val interrupt = talk?.get("interruptOnSpeech")?.asBooleanOrNull() + + if (!isCanonicalMainSessionKey(mainSessionKey)) { + mainSessionKey = mainKey + } + defaultVoiceId = voice ?: envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } + voiceAliases = aliases + if (!voiceOverrideActive) currentVoiceId = defaultVoiceId + defaultModelId = model ?: defaultModelIdFallback + if (!modelOverrideActive) currentModelId = defaultModelId + defaultOutputFormat = outputFormat ?: defaultOutputFormatFallback + apiKey = key ?: envKey?.takeIf { it.isNotEmpty() } + if (interrupt != null) interruptOnSpeech = interrupt + } catch (_: Throwable) { + defaultVoiceId = envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } + defaultModelId = defaultModelIdFallback + if (!modelOverrideActive) currentModelId = defaultModelId + apiKey = envKey?.takeIf { it.isNotEmpty() } + voiceAliases = emptyMap() + defaultOutputFormat = defaultOutputFormatFallback + } + } + + private fun parseRunId(jsonString: String): String? { + val obj = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return null + return obj["runId"].asStringOrNull() + } + + private suspend fun streamTts( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + sink: StreamingMediaDataSource, + ) { + withContext(Dispatchers.IO) { + val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) + try { + val payload = buildRequestPayload(request) + conn.outputStream.use { it.write(payload.toByteArray()) } + + val code = conn.responseCode + if (code >= 400) { + val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" + sink.fail() + throw IllegalStateException("ElevenLabs failed: $code $message") + } + + val buffer = ByteArray(8 * 1024) + conn.inputStream.use { input -> + while (true) { + val read = input.read(buffer) + if (read <= 0) break + sink.append(buffer.copyOf(read)) + } + } + sink.finish() + } finally { + conn.disconnect() + } + } + } + + private suspend fun streamPcm( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + track: AudioTrack, + ) { + withContext(Dispatchers.IO) { + val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) + try { + val payload = buildRequestPayload(request) + conn.outputStream.use { it.write(payload.toByteArray()) } + + val code = conn.responseCode + if (code >= 400) { + val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" + throw IllegalStateException("ElevenLabs failed: $code $message") + } + + val buffer = ByteArray(8 * 1024) + conn.inputStream.use { input -> + while (true) { + if (pcmStopRequested) return@withContext + val read = input.read(buffer) + if (read <= 0) break + var offset = 0 + while (offset < read) { + if (pcmStopRequested) return@withContext + val wrote = + try { + track.write(buffer, offset, read - offset) + } catch (err: Throwable) { + if (pcmStopRequested) return@withContext + throw err + } + if (wrote <= 0) { + if (pcmStopRequested) return@withContext + throw IllegalStateException("AudioTrack write failed: $wrote") + } + offset += wrote + } + } + } + } finally { + conn.disconnect() + } + } + } + + private fun openTtsConnection( + voiceId: String, + apiKey: String, + request: ElevenLabsRequest, + ): HttpURLConnection { + val baseUrl = "https://api.elevenlabs.io/v1/text-to-speech/$voiceId/stream" + val latencyTier = request.latencyTier + val url = + if (latencyTier != null) { + URL("$baseUrl?optimize_streaming_latency=$latencyTier") + } else { + URL(baseUrl) + } + val conn = url.openConnection() as HttpURLConnection + conn.requestMethod = "POST" + conn.connectTimeout = 30_000 + conn.readTimeout = 30_000 + conn.setRequestProperty("Content-Type", "application/json") + conn.setRequestProperty("Accept", resolveAcceptHeader(request.outputFormat)) + conn.setRequestProperty("xi-api-key", apiKey) + conn.doOutput = true + return conn + } + + private fun resolveAcceptHeader(outputFormat: String?): String { + val normalized = outputFormat?.trim()?.lowercase().orEmpty() + return if (normalized.startsWith("pcm_")) "audio/pcm" else "audio/mpeg" + } + + private fun buildRequestPayload(request: ElevenLabsRequest): String { + val voiceSettingsEntries = + buildJsonObject { + request.speed?.let { put("speed", JsonPrimitive(it)) } + request.stability?.let { put("stability", JsonPrimitive(it)) } + request.similarity?.let { put("similarity_boost", JsonPrimitive(it)) } + request.style?.let { put("style", JsonPrimitive(it)) } + request.speakerBoost?.let { put("use_speaker_boost", JsonPrimitive(it)) } + } + + val payload = + buildJsonObject { + put("text", JsonPrimitive(request.text)) + request.modelId?.takeIf { it.isNotEmpty() }?.let { put("model_id", JsonPrimitive(it)) } + request.outputFormat?.takeIf { it.isNotEmpty() }?.let { put("output_format", JsonPrimitive(it)) } + request.seed?.let { put("seed", JsonPrimitive(it)) } + request.normalize?.let { put("apply_text_normalization", JsonPrimitive(it)) } + request.language?.let { put("language_code", JsonPrimitive(it)) } + if (voiceSettingsEntries.isNotEmpty()) { + put("voice_settings", voiceSettingsEntries) + } + } + + return payload.toString() + } + + private data class ElevenLabsRequest( + val text: String, + val modelId: String?, + val outputFormat: String?, + val speed: Double?, + val stability: Double?, + val similarity: Double?, + val style: Double?, + val speakerBoost: Boolean?, + val seed: Long?, + val normalize: String?, + val language: String?, + val latencyTier: Int?, + ) + + private object TalkModeRuntime { + fun resolveSpeed(speed: Double?, rateWpm: Int?): Double? { + if (rateWpm != null && rateWpm > 0) { + val resolved = rateWpm.toDouble() / 175.0 + if (resolved <= 0.5 || resolved >= 2.0) return null + return resolved + } + if (speed != null) { + if (speed <= 0.5 || speed >= 2.0) return null + return speed + } + return null + } + + fun validatedUnit(value: Double?): Double? { + if (value == null) return null + if (value < 0 || value > 1) return null + return value + } + + fun validatedStability(value: Double?, modelId: String?): Double? { + if (value == null) return null + val normalized = modelId?.trim()?.lowercase() + if (normalized == "eleven_v3") { + return if (value == 0.0 || value == 0.5 || value == 1.0) value else null + } + return validatedUnit(value) + } + + fun validatedSeed(value: Long?): Long? { + if (value == null) return null + if (value < 0 || value > 4294967295L) return null + return value + } + + fun validatedNormalize(value: String?): String? { + val normalized = value?.trim()?.lowercase() ?: return null + return if (normalized in listOf("auto", "on", "off")) normalized else null + } + + fun validatedLanguage(value: String?): String? { + val normalized = value?.trim()?.lowercase() ?: return null + if (normalized.length != 2) return null + if (!normalized.all { it in 'a'..'z' }) return null + return normalized + } + + fun validatedOutputFormat(value: String?): String? { + val trimmed = value?.trim()?.lowercase() ?: return null + if (trimmed.isEmpty()) return null + if (trimmed.startsWith("mp3_")) return trimmed + return if (parsePcmSampleRate(trimmed) != null) trimmed else null + } + + fun validatedLatencyTier(value: Int?): Int? { + if (value == null) return null + if (value < 0 || value > 4) return null + return value + } + + fun parsePcmSampleRate(value: String?): Int? { + val trimmed = value?.trim()?.lowercase() ?: return null + if (!trimmed.startsWith("pcm_")) return null + val suffix = trimmed.removePrefix("pcm_") + val digits = suffix.takeWhile { it.isDigit() } + val rate = digits.toIntOrNull() ?: return null + return if (rate in setOf(16000, 22050, 24000, 44100)) rate else null + } + + fun isMessageTimestampAfter(timestamp: Double, sinceSeconds: Double): Boolean { + val sinceMs = sinceSeconds * 1000 + return if (timestamp > 10_000_000_000) { + timestamp >= sinceMs - 500 + } else { + timestamp >= sinceSeconds - 0.5 + } + } + } + + private fun ensureInterruptListener() { + if (!interruptOnSpeech || !_isEnabled.value) return + mainHandler.post { + if (stopRequested) return@post + if (!SpeechRecognizer.isRecognitionAvailable(context)) return@post + try { + if (recognizer == null) { + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + } + recognizer?.cancel() + startListeningInternal(markListening = false) + } catch (_: Throwable) { + // ignore + } + } + } + + private fun resolveVoiceAlias(value: String?): String? { + val trimmed = value?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val normalized = normalizeAliasKey(trimmed) + voiceAliases[normalized]?.let { return it } + if (voiceAliases.values.any { it.equals(trimmed, ignoreCase = true) }) return trimmed + return if (isLikelyVoiceId(trimmed)) trimmed else null + } + + private suspend fun resolveVoiceId(preferred: String?, apiKey: String): String? { + val trimmed = preferred?.trim().orEmpty() + if (trimmed.isNotEmpty()) { + val resolved = resolveVoiceAlias(trimmed) + if (resolved != null) return resolved + Log.w(tag, "unknown voice alias $trimmed") + } + fallbackVoiceId?.let { return it } + + return try { + val voices = listVoices(apiKey) + val first = voices.firstOrNull() ?: return null + fallbackVoiceId = first.voiceId + if (defaultVoiceId.isNullOrBlank()) { + defaultVoiceId = first.voiceId + } + if (!voiceOverrideActive) { + currentVoiceId = first.voiceId + } + val name = first.name ?: "unknown" + Log.d(tag, "default voice selected $name (${first.voiceId})") + first.voiceId + } catch (err: Throwable) { + Log.w(tag, "list voices failed: ${err.message ?: err::class.simpleName}") + null + } + } + + private suspend fun listVoices(apiKey: String): List { + return withContext(Dispatchers.IO) { + val url = URL("https://api.elevenlabs.io/v1/voices") + val conn = url.openConnection() as HttpURLConnection + conn.requestMethod = "GET" + conn.connectTimeout = 15_000 + conn.readTimeout = 15_000 + conn.setRequestProperty("xi-api-key", apiKey) + + val code = conn.responseCode + val stream = if (code >= 400) conn.errorStream else conn.inputStream + val data = stream.readBytes() + if (code >= 400) { + val message = data.toString(Charsets.UTF_8) + throw IllegalStateException("ElevenLabs voices failed: $code $message") + } + + val root = json.parseToJsonElement(data.toString(Charsets.UTF_8)).asObjectOrNull() + val voices = (root?.get("voices") as? JsonArray) ?: JsonArray(emptyList()) + voices.mapNotNull { entry -> + val obj = entry.asObjectOrNull() ?: return@mapNotNull null + val voiceId = obj["voice_id"].asStringOrNull() ?: return@mapNotNull null + val name = obj["name"].asStringOrNull() + ElevenLabsVoice(voiceId, name) + } + } + } + + private fun isLikelyVoiceId(value: String): Boolean { + if (value.length < 10) return false + return value.all { it.isLetterOrDigit() || it == '-' || it == '_' } + } + + private fun normalizeAliasKey(value: String): String = + value.trim().lowercase() + + private data class ElevenLabsVoice(val voiceId: String, val name: String?) + + private val listener = + object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + if (_isEnabled.value) { + _statusText.value = if (_isListening.value) "Listening" else _statusText.value + } + } + + override fun onBeginningOfSpeech() {} + + override fun onRmsChanged(rmsdB: Float) {} + + override fun onBufferReceived(buffer: ByteArray?) {} + + override fun onEndOfSpeech() { + scheduleRestart() + } + + override fun onError(error: Int) { + if (stopRequested) return + _isListening.value = false + if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { + _statusText.value = "Microphone permission required" + return + } + + _statusText.value = + when (error) { + SpeechRecognizer.ERROR_AUDIO -> "Audio error" + SpeechRecognizer.ERROR_CLIENT -> "Client error" + SpeechRecognizer.ERROR_NETWORK -> "Network error" + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" + SpeechRecognizer.ERROR_NO_MATCH -> "Listening" + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" + SpeechRecognizer.ERROR_SERVER -> "Server error" + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" + else -> "Speech error ($error)" + } + scheduleRestart(delayMs = 600) + } + + override fun onResults(results: Bundle?) { + val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let { handleTranscript(it, isFinal = true) } + scheduleRestart() + } + + override fun onPartialResults(partialResults: Bundle?) { + val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let { handleTranscript(it, isFinal = false) } + } + + override fun onEvent(eventType: Int, params: Bundle?) {} + } +} + +private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +private fun JsonElement?.asStringOrNull(): String? = + (this as? JsonPrimitive)?.takeIf { it.isString }?.content + +private fun JsonElement?.asDoubleOrNull(): Double? { + val primitive = this as? JsonPrimitive ?: return null + return primitive.content.toDoubleOrNull() +} + +private fun JsonElement?.asBooleanOrNull(): Boolean? { + val primitive = this as? JsonPrimitive ?: return null + val content = primitive.content.trim().lowercase() + return when (content) { + "true", "yes", "1" -> true + "false", "no", "0" -> false + else -> null + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeCommandExtractor.kt b/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeCommandExtractor.kt new file mode 100644 index 00000000000..8da4e328944 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeCommandExtractor.kt @@ -0,0 +1,40 @@ +package bot.molt.android.voice + +object VoiceWakeCommandExtractor { + fun extractCommand(text: String, triggerWords: List): String? { + val raw = text.trim() + if (raw.isEmpty()) return null + + val triggers = + triggerWords + .map { it.trim().lowercase() } + .filter { it.isNotEmpty() } + .distinct() + if (triggers.isEmpty()) return null + + val alternation = triggers.joinToString("|") { Regex.escape(it) } + // Match: " " + val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$") + val match = regex.find(raw) ?: return null + val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty() + if (extracted.isEmpty()) return null + + val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim() + if (cleaned.isEmpty()) return null + return cleaned + } +} + +private fun Char.isPunctuation(): Boolean { + return when (Character.getType(this)) { + Character.CONNECTOR_PUNCTUATION.toInt(), + Character.DASH_PUNCTUATION.toInt(), + Character.START_PUNCTUATION.toInt(), + Character.END_PUNCTUATION.toInt(), + Character.INITIAL_QUOTE_PUNCTUATION.toInt(), + Character.FINAL_QUOTE_PUNCTUATION.toInt(), + Character.OTHER_PUNCTUATION.toInt(), + -> true + else -> false + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeManager.kt b/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeManager.kt new file mode 100644 index 00000000000..b27d0e3c704 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/voice/VoiceWakeManager.kt @@ -0,0 +1,173 @@ +package bot.molt.android.voice + +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.launch + +class VoiceWakeManager( + private val context: Context, + private val scope: CoroutineScope, + private val onCommand: suspend (String) -> Unit, +) { + private val mainHandler = Handler(Looper.getMainLooper()) + + private val _isListening = MutableStateFlow(false) + val isListening: StateFlow = _isListening + + private val _statusText = MutableStateFlow("Off") + val statusText: StateFlow = _statusText + + var triggerWords: List = emptyList() + private set + + private var recognizer: SpeechRecognizer? = null + private var restartJob: Job? = null + private var lastDispatched: String? = null + private var stopRequested = false + + fun setTriggerWords(words: List) { + triggerWords = words + } + + fun start() { + mainHandler.post { + if (_isListening.value) return@post + stopRequested = false + + if (!SpeechRecognizer.isRecognitionAvailable(context)) { + _isListening.value = false + _statusText.value = "Speech recognizer unavailable" + return@post + } + + try { + recognizer?.destroy() + recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } + startListeningInternal() + } catch (err: Throwable) { + _isListening.value = false + _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" + } + } + } + + fun stop(statusText: String = "Off") { + stopRequested = true + restartJob?.cancel() + restartJob = null + mainHandler.post { + _isListening.value = false + _statusText.value = statusText + recognizer?.cancel() + recognizer?.destroy() + recognizer = null + } + } + + private fun startListeningInternal() { + val r = recognizer ?: return + val intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + } + + _statusText.value = "Listening" + _isListening.value = true + r.startListening(intent) + } + + private fun scheduleRestart(delayMs: Long = 350) { + if (stopRequested) return + restartJob?.cancel() + restartJob = + scope.launch { + delay(delayMs) + mainHandler.post { + if (stopRequested) return@post + try { + recognizer?.cancel() + startListeningInternal() + } catch (_: Throwable) { + // Will be picked up by onError and retry again. + } + } + } + } + + private fun handleTranscription(text: String) { + val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return + if (command == lastDispatched) return + lastDispatched = command + + scope.launch { onCommand(command) } + _statusText.value = "Triggered" + scheduleRestart(delayMs = 650) + } + + private val listener = + object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { + _statusText.value = "Listening" + } + + override fun onBeginningOfSpeech() {} + + override fun onRmsChanged(rmsdB: Float) {} + + override fun onBufferReceived(buffer: ByteArray?) {} + + override fun onEndOfSpeech() { + scheduleRestart() + } + + override fun onError(error: Int) { + if (stopRequested) return + _isListening.value = false + if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { + _statusText.value = "Microphone permission required" + return + } + + _statusText.value = + when (error) { + SpeechRecognizer.ERROR_AUDIO -> "Audio error" + SpeechRecognizer.ERROR_CLIENT -> "Client error" + SpeechRecognizer.ERROR_NETWORK -> "Network error" + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" + SpeechRecognizer.ERROR_NO_MATCH -> "Listening" + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" + SpeechRecognizer.ERROR_SERVER -> "Server error" + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" + else -> "Speech error ($error)" + } + scheduleRestart(delayMs = 600) + } + + override fun onResults(results: Bundle?) { + val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let(::handleTranscription) + scheduleRestart() + } + + override fun onPartialResults(partialResults: Bundle?) { + val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let(::handleTranscription) + } + + override fun onEvent(eventType: Int, params: Bundle?) {} + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/NodeForegroundServiceTest.kt b/apps/android/app/src/test/java/bot/molt/android/NodeForegroundServiceTest.kt new file mode 100644 index 00000000000..77ab3ae36b5 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/NodeForegroundServiceTest.kt @@ -0,0 +1,43 @@ +package bot.molt.android + +import android.app.Notification +import android.content.Intent +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class NodeForegroundServiceTest { + @Test + fun buildNotificationSetsLaunchIntent() { + val service = Robolectric.buildService(NodeForegroundService::class.java).get() + val notification = buildNotification(service) + + val pendingIntent = notification.contentIntent + assertNotNull(pendingIntent) + + val savedIntent = Shadows.shadowOf(pendingIntent).savedIntent + assertNotNull(savedIntent) + assertEquals(MainActivity::class.java.name, savedIntent.component?.className) + + val expectedFlags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP + assertEquals(expectedFlags, savedIntent.flags and expectedFlags) + } + + private fun buildNotification(service: NodeForegroundService): Notification { + val method = + NodeForegroundService::class.java.getDeclaredMethod( + "buildNotification", + String::class.java, + String::class.java, + ) + method.isAccessible = true + return method.invoke(service, "Title", "Text") as Notification + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/WakeWordsTest.kt b/apps/android/app/src/test/java/bot/molt/android/WakeWordsTest.kt new file mode 100644 index 00000000000..f18ba187ea2 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/WakeWordsTest.kt @@ -0,0 +1,50 @@ +package bot.molt.android + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class WakeWordsTest { + @Test + fun parseCommaSeparatedTrimsAndDropsEmpty() { + assertEquals(listOf("clawd", "claude"), WakeWords.parseCommaSeparated(" clawd , claude, , ")) + } + + @Test + fun sanitizeTrimsCapsAndFallsBack() { + val defaults = listOf("clawd", "claude") + val long = "x".repeat(WakeWords.maxWordLength + 10) + val words = listOf(" ", " hello ", long) + + val sanitized = WakeWords.sanitize(words, defaults) + assertEquals(2, sanitized.size) + assertEquals("hello", sanitized[0]) + assertEquals("x".repeat(WakeWords.maxWordLength), sanitized[1]) + + assertEquals(defaults, WakeWords.sanitize(listOf(" ", ""), defaults)) + } + + @Test + fun sanitizeLimitsWordCount() { + val defaults = listOf("clawd") + val words = (1..(WakeWords.maxWords + 5)).map { "w$it" } + val sanitized = WakeWords.sanitize(words, defaults) + assertEquals(WakeWords.maxWords, sanitized.size) + assertEquals("w1", sanitized.first()) + assertEquals("w${WakeWords.maxWords}", sanitized.last()) + } + + @Test + fun parseIfChangedSkipsWhenUnchanged() { + val current = listOf("clawd", "claude") + val parsed = WakeWords.parseIfChanged(" clawd , claude ", current) + assertNull(parsed) + } + + @Test + fun parseIfChangedReturnsUpdatedList() { + val current = listOf("clawd") + val parsed = WakeWords.parseIfChanged(" clawd , jarvis ", current) + assertEquals(listOf("clawd", "jarvis"), parsed) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/gateway/BonjourEscapesTest.kt b/apps/android/app/src/test/java/bot/molt/android/gateway/BonjourEscapesTest.kt new file mode 100644 index 00000000000..026d35a8a2c --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/gateway/BonjourEscapesTest.kt @@ -0,0 +1,19 @@ +package bot.molt.android.gateway + +import org.junit.Assert.assertEquals +import org.junit.Test + +class BonjourEscapesTest { + @Test + fun decodeNoop() { + assertEquals("", BonjourEscapes.decode("")) + assertEquals("hello", BonjourEscapes.decode("hello")) + } + + @Test + fun decodeDecodesDecimalEscapes() { + assertEquals("Moltbot Gateway", BonjourEscapes.decode("Moltbot\\032Gateway")) + assertEquals("A B", BonjourEscapes.decode("A\\032B")) + assertEquals("Peter\u2019s Mac", BonjourEscapes.decode("Peter\\226\\128\\153s Mac")) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/node/CanvasControllerSnapshotParamsTest.kt b/apps/android/app/src/test/java/bot/molt/android/node/CanvasControllerSnapshotParamsTest.kt new file mode 100644 index 00000000000..c9803c56feb --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/node/CanvasControllerSnapshotParamsTest.kt @@ -0,0 +1,43 @@ +package bot.molt.android.node + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class CanvasControllerSnapshotParamsTest { + @Test + fun parseSnapshotParamsDefaultsToJpeg() { + val params = CanvasController.parseSnapshotParams(null) + assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format) + assertNull(params.quality) + assertNull(params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesPng() { + val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""") + assertEquals(CanvasController.SnapshotFormat.Png, params.format) + assertEquals(900, params.maxWidth) + } + + @Test + fun parseSnapshotParamsParsesJpegAliases() { + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format, + ) + assertEquals( + CanvasController.SnapshotFormat.Jpeg, + CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format, + ) + } + + @Test + fun parseSnapshotParamsClampsQuality() { + val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""") + assertEquals(0.1, low.quality) + + val high = CanvasController.parseSnapshotParams("""{"quality":5}""") + assertEquals(1.0, high.quality) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/node/JpegSizeLimiterTest.kt b/apps/android/app/src/test/java/bot/molt/android/node/JpegSizeLimiterTest.kt new file mode 100644 index 00000000000..8f114b3ec69 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/node/JpegSizeLimiterTest.kt @@ -0,0 +1,47 @@ +package bot.molt.android.node + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.math.min + +class JpegSizeLimiterTest { + @Test + fun compressesLargePayloadsUnderLimit() { + val maxBytes = 5 * 1024 * 1024 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = 4000, + initialHeight = 3000, + startQuality = 95, + maxBytes = maxBytes, + encode = { width, height, quality -> + val estimated = (width.toLong() * height.toLong() * quality.toLong()) / 100 + val size = min(maxBytes.toLong() * 2, estimated).toInt() + ByteArray(size) + }, + ) + + assertTrue(result.bytes.size <= maxBytes) + assertTrue(result.width <= 4000) + assertTrue(result.height <= 3000) + assertTrue(result.quality <= 95) + } + + @Test + fun keepsSmallPayloadsAsIs() { + val maxBytes = 5 * 1024 * 1024 + val result = + JpegSizeLimiter.compressToLimit( + initialWidth = 800, + initialHeight = 600, + startQuality = 90, + maxBytes = maxBytes, + encode = { _, _, _ -> ByteArray(120_000) }, + ) + + assertEquals(800, result.width) + assertEquals(600, result.height) + assertEquals(90, result.quality) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/node/SmsManagerTest.kt b/apps/android/app/src/test/java/bot/molt/android/node/SmsManagerTest.kt new file mode 100644 index 00000000000..d09bbc6bb17 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/node/SmsManagerTest.kt @@ -0,0 +1,91 @@ +package bot.molt.android.node + +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class SmsManagerTest { + private val json = SmsManager.JsonConfig + + @Test + fun parseParamsRejectsEmptyPayload() { + val result = SmsManager.parseParams("", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: paramsJSON required", error.error) + } + + @Test + fun parseParamsRejectsInvalidJson() { + val result = SmsManager.parseParams("not-json", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: expected JSON object", error.error) + } + + @Test + fun parseParamsRejectsNonObjectJson() { + val result = SmsManager.parseParams("[]", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: expected JSON object", error.error) + } + + @Test + fun parseParamsRejectsMissingTo() { + val result = SmsManager.parseParams("{\"message\":\"Hi\"}", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: 'to' phone number required", error.error) + assertEquals("Hi", error.message) + } + + @Test + fun parseParamsRejectsMissingMessage() { + val result = SmsManager.parseParams("{\"to\":\"+1234\"}", json) + assertTrue(result is SmsManager.ParseResult.Error) + val error = result as SmsManager.ParseResult.Error + assertEquals("INVALID_REQUEST: 'message' text required", error.error) + assertEquals("+1234", error.to) + } + + @Test + fun parseParamsTrimsToField() { + val result = SmsManager.parseParams("{\"to\":\" +1555 \",\"message\":\"Hello\"}", json) + assertTrue(result is SmsManager.ParseResult.Ok) + val ok = result as SmsManager.ParseResult.Ok + assertEquals("+1555", ok.params.to) + assertEquals("Hello", ok.params.message) + } + + @Test + fun buildPayloadJsonEscapesFields() { + val payload = SmsManager.buildPayloadJson( + json = json, + ok = false, + to = "+1\"23", + error = "SMS_SEND_FAILED: \"nope\"", + ) + val parsed = json.parseToJsonElement(payload).jsonObject + assertEquals("false", parsed["ok"]?.jsonPrimitive?.content) + assertEquals("+1\"23", parsed["to"]?.jsonPrimitive?.content) + assertEquals("SMS_SEND_FAILED: \"nope\"", parsed["error"]?.jsonPrimitive?.content) + } + + @Test + fun buildSendPlanUsesMultipartWhenMultipleParts() { + val plan = SmsManager.buildSendPlan("hello") { listOf("a", "b") } + assertTrue(plan.useMultipart) + assertEquals(listOf("a", "b"), plan.parts) + } + + @Test + fun buildSendPlanFallsBackToSinglePartWhenDividerEmpty() { + val plan = SmsManager.buildSendPlan("hello") { emptyList() } + assertFalse(plan.useMultipart) + assertEquals(listOf("hello"), plan.parts) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotCanvasA2UIActionTest.kt b/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotCanvasA2UIActionTest.kt new file mode 100644 index 00000000000..5ed5e505f94 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotCanvasA2UIActionTest.kt @@ -0,0 +1,49 @@ +package bot.molt.android.protocol + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import org.junit.Assert.assertEquals +import org.junit.Test + +class MoltbotCanvasA2UIActionTest { + @Test + fun extractActionNameAcceptsNameOrAction() { + val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject + assertEquals("Hello", MoltbotCanvasA2UIAction.extractActionName(nameObj)) + + val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject + assertEquals("Wave", MoltbotCanvasA2UIAction.extractActionName(actionObj)) + + val fallbackObj = + Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject + assertEquals("Fallback", MoltbotCanvasA2UIAction.extractActionName(fallbackObj)) + } + + @Test + fun formatAgentMessageMatchesSharedSpec() { + val msg = + MoltbotCanvasA2UIAction.formatAgentMessage( + actionName = "Get Weather", + sessionKey = "main", + surfaceId = "main", + sourceComponentId = "btnWeather", + host = "Peter’s iPad", + instanceId = "ipad16,6", + contextJson = "{\"city\":\"Vienna\"}", + ) + + assertEquals( + "CANVAS_A2UI action=Get_Weather session=main surface=main component=btnWeather host=Peter_s_iPad instance=ipad16_6 ctx={\"city\":\"Vienna\"} default=update_canvas", + msg, + ) + } + + @Test + fun jsDispatchA2uiStatusIsStable() { + val js = MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId = "a1", ok = true, error = null) + assertEquals( + "window.dispatchEvent(new CustomEvent('moltbot:a2ui-action-status', { detail: { id: \"a1\", ok: true, error: \"\" } }));", + js, + ) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotProtocolConstantsTest.kt b/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotProtocolConstantsTest.kt new file mode 100644 index 00000000000..998f6600cff --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/protocol/ClawdbotProtocolConstantsTest.kt @@ -0,0 +1,35 @@ +package bot.molt.android.protocol + +import org.junit.Assert.assertEquals +import org.junit.Test + +class MoltbotProtocolConstantsTest { + @Test + fun canvasCommandsUseStableStrings() { + assertEquals("canvas.present", MoltbotCanvasCommand.Present.rawValue) + assertEquals("canvas.hide", MoltbotCanvasCommand.Hide.rawValue) + assertEquals("canvas.navigate", MoltbotCanvasCommand.Navigate.rawValue) + assertEquals("canvas.eval", MoltbotCanvasCommand.Eval.rawValue) + assertEquals("canvas.snapshot", MoltbotCanvasCommand.Snapshot.rawValue) + } + + @Test + fun a2uiCommandsUseStableStrings() { + assertEquals("canvas.a2ui.push", MoltbotCanvasA2UICommand.Push.rawValue) + assertEquals("canvas.a2ui.pushJSONL", MoltbotCanvasA2UICommand.PushJSONL.rawValue) + assertEquals("canvas.a2ui.reset", MoltbotCanvasA2UICommand.Reset.rawValue) + } + + @Test + fun capabilitiesUseStableStrings() { + assertEquals("canvas", MoltbotCapability.Canvas.rawValue) + assertEquals("camera", MoltbotCapability.Camera.rawValue) + assertEquals("screen", MoltbotCapability.Screen.rawValue) + assertEquals("voiceWake", MoltbotCapability.VoiceWake.rawValue) + } + + @Test + fun screenCommandsUseStableStrings() { + assertEquals("screen.record", MoltbotScreenCommand.Record.rawValue) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/ui/chat/SessionFiltersTest.kt b/apps/android/app/src/test/java/bot/molt/android/ui/chat/SessionFiltersTest.kt new file mode 100644 index 00000000000..e410cc36567 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/ui/chat/SessionFiltersTest.kt @@ -0,0 +1,35 @@ +package bot.molt.android.ui.chat + +import bot.molt.android.chat.ChatSessionEntry +import org.junit.Assert.assertEquals +import org.junit.Test + +class SessionFiltersTest { + @Test + fun sessionChoicesPreferMainAndRecent() { + val now = 1_700_000_000_000L + val recent1 = now - 2 * 60 * 60 * 1000L + val recent2 = now - 5 * 60 * 60 * 1000L + val stale = now - 26 * 60 * 60 * 1000L + val sessions = + listOf( + ChatSessionEntry(key = "recent-1", updatedAtMs = recent1), + ChatSessionEntry(key = "main", updatedAtMs = stale), + ChatSessionEntry(key = "old-1", updatedAtMs = stale), + ChatSessionEntry(key = "recent-2", updatedAtMs = recent2), + ) + + val result = resolveSessionChoices("main", sessions, mainSessionKey = "main", nowMs = now).map { it.key } + assertEquals(listOf("main", "recent-1", "recent-2"), result) + } + + @Test + fun sessionChoicesIncludeCurrentWhenMissing() { + val now = 1_700_000_000_000L + val recent = now - 10 * 60 * 1000L + val sessions = listOf(ChatSessionEntry(key = "main", updatedAtMs = recent)) + + val result = resolveSessionChoices("custom", sessions, mainSessionKey = "main", nowMs = now).map { it.key } + assertEquals(listOf("main", "custom"), result) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/voice/TalkDirectiveParserTest.kt b/apps/android/app/src/test/java/bot/molt/android/voice/TalkDirectiveParserTest.kt new file mode 100644 index 00000000000..8f57a9aca75 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/voice/TalkDirectiveParserTest.kt @@ -0,0 +1,55 @@ +package bot.molt.android.voice + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class TalkDirectiveParserTest { + @Test + fun parsesDirectiveAndStripsHeader() { + val input = """ + {"voice":"voice-123","once":true} + Hello from talk mode. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("voice-123", result.directive?.voiceId) + assertEquals(true, result.directive?.once) + assertEquals("Hello from talk mode.", result.stripped.trim()) + } + + @Test + fun ignoresUnknownKeysButReportsThem() { + val input = """ + {"voice":"abc","foo":1,"bar":"baz"} + Hi there. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("abc", result.directive?.voiceId) + assertTrue(result.unknownKeys.containsAll(listOf("bar", "foo"))) + } + + @Test + fun parsesAlternateKeys() { + val input = """ + {"model_id":"eleven_v3","similarity_boost":0.4,"no_speaker_boost":true,"rate":200} + Speak. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertEquals("eleven_v3", result.directive?.modelId) + assertEquals(0.4, result.directive?.similarity) + assertEquals(false, result.directive?.speakerBoost) + assertEquals(200, result.directive?.rateWpm) + } + + @Test + fun returnsNullWhenNoDirectivePresent() { + val input = """ + {} + Hello. + """.trimIndent() + val result = TalkDirectiveParser.parse(input) + assertNull(result.directive) + assertEquals(input, result.stripped) + } +} diff --git a/apps/android/app/src/test/java/bot/molt/android/voice/VoiceWakeCommandExtractorTest.kt b/apps/android/app/src/test/java/bot/molt/android/voice/VoiceWakeCommandExtractorTest.kt new file mode 100644 index 00000000000..3460ba7a8ec --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/voice/VoiceWakeCommandExtractorTest.kt @@ -0,0 +1,25 @@ +package bot.molt.android.voice + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class VoiceWakeCommandExtractorTest { + @Test + fun extractsCommandAfterTriggerWord() { + val res = VoiceWakeCommandExtractor.extractCommand("Claude take a photo", listOf("clawd", "claude")) + assertEquals("take a photo", res) + } + + @Test + fun extractsCommandWithPunctuation() { + val res = VoiceWakeCommandExtractor.extractCommand("hey clawd, what's the weather?", listOf("clawd")) + assertEquals("what's the weather?", res) + } + + @Test + fun returnsNullWhenNoCommandProvided() { + assertNull(VoiceWakeCommandExtractor.extractCommand("claude", listOf("claude"))) + assertNull(VoiceWakeCommandExtractor.extractCommand("hey claude!", listOf("claude"))) + } +} diff --git a/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift b/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift index aaba5a86304..19be913f4f2 100644 --- a/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift +++ b/apps/ios/Sources/Gateway/GatewayDiscoveryModel.swift @@ -104,7 +104,7 @@ final class GatewayDiscoveryModel { } self.browsers[domain] = browser - browser.start(queue: DispatchQueue(label: "com.clawdbot.ios.gateway-discovery.\(domain)")) + browser.start(queue: DispatchQueue(label: "bot.molt.ios.gateway-discovery.\(domain)")) } } diff --git a/apps/ios/Sources/Gateway/GatewaySettingsStore.swift b/apps/ios/Sources/Gateway/GatewaySettingsStore.swift index 52ada8d80f2..1c78b7869ea 100644 --- a/apps/ios/Sources/Gateway/GatewaySettingsStore.swift +++ b/apps/ios/Sources/Gateway/GatewaySettingsStore.swift @@ -1,9 +1,11 @@ import Foundation enum GatewaySettingsStore { - private static let gatewayService = "com.clawdbot.gateway" + private static let gatewayService = "bot.molt.gateway" + private static let legacyGatewayService = "com.clawdbot.gateway" private static let legacyBridgeService = "com.clawdbot.bridge" - private static let nodeService = "com.clawdbot.node" + private static let nodeService = "bot.molt.node" + private static let legacyNodeService = "com.clawdbot.node" private static let instanceIdDefaultsKey = "node.instanceId" private static let preferredGatewayStableIDDefaultsKey = "gateway.preferredStableID" @@ -33,8 +35,22 @@ enum GatewaySettingsStore { } static func loadStableInstanceID() -> String? { - KeychainStore.loadString(service: self.nodeService, account: self.instanceIdAccount)? - .trimmingCharacters(in: .whitespacesAndNewlines) + if let value = KeychainStore.loadString(service: self.nodeService, account: self.instanceIdAccount)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + if let legacy = KeychainStore.loadString(service: self.legacyNodeService, account: self.instanceIdAccount)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !legacy.isEmpty + { + _ = KeychainStore.saveString(legacy, service: self.nodeService, account: self.instanceIdAccount) + return legacy + } + + return nil } static func saveStableInstanceID(_ instanceId: String) { @@ -42,8 +58,29 @@ enum GatewaySettingsStore { } static func loadPreferredGatewayStableID() -> String? { - KeychainStore.loadString(service: self.gatewayService, account: self.preferredGatewayStableIDAccount)? - .trimmingCharacters(in: .whitespacesAndNewlines) + if let value = KeychainStore.loadString( + service: self.gatewayService, + account: self.preferredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + if let legacy = KeychainStore.loadString( + service: self.legacyGatewayService, + account: self.preferredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !legacy.isEmpty + { + _ = KeychainStore.saveString( + legacy, + service: self.gatewayService, + account: self.preferredGatewayStableIDAccount) + return legacy + } + + return nil } static func savePreferredGatewayStableID(_ stableID: String) { @@ -54,8 +91,29 @@ enum GatewaySettingsStore { } static func loadLastDiscoveredGatewayStableID() -> String? { - KeychainStore.loadString(service: self.gatewayService, account: self.lastDiscoveredGatewayStableIDAccount)? - .trimmingCharacters(in: .whitespacesAndNewlines) + if let value = KeychainStore.loadString( + service: self.gatewayService, + account: self.lastDiscoveredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + { + return value + } + + if let legacy = KeychainStore.loadString( + service: self.legacyGatewayService, + account: self.lastDiscoveredGatewayStableIDAccount + )?.trimmingCharacters(in: .whitespacesAndNewlines), + !legacy.isEmpty + { + _ = KeychainStore.saveString( + legacy, + service: self.gatewayService, + account: self.lastDiscoveredGatewayStableIDAccount) + return legacy + } + + return nil } static func saveLastDiscoveredGatewayStableID(_ stableID: String) { diff --git a/apps/ios/Sources/Screen/ScreenRecordService.swift b/apps/ios/Sources/Screen/ScreenRecordService.swift index 9011487ba84..849add38de8 100644 --- a/apps/ios/Sources/Screen/ScreenRecordService.swift +++ b/apps/ios/Sources/Screen/ScreenRecordService.swift @@ -55,7 +55,7 @@ final class ScreenRecordService: @unchecked Sendable { outPath: outPath) let state = CaptureState() - let recordQueue = DispatchQueue(label: "com.clawdbot.screenrecord") + let recordQueue = DispatchQueue(label: "bot.molt.screenrecord") try await self.startCapture(state: state, config: config, recordQueue: recordQueue) try await Task.sleep(nanoseconds: UInt64(config.durationMs) * 1_000_000) diff --git a/apps/ios/Sources/Voice/TalkModeManager.swift b/apps/ios/Sources/Voice/TalkModeManager.swift index 0a387242468..c0ae8b4540c 100644 --- a/apps/ios/Sources/Voice/TalkModeManager.swift +++ b/apps/ios/Sources/Voice/TalkModeManager.swift @@ -48,7 +48,7 @@ final class TalkModeManager: NSObject { private var chatSubscribedSessionKeys = Set() - private let logger = Logger(subsystem: "com.clawdbot", category: "TalkMode") + private let logger = Logger(subsystem: "bot.molt", category: "TalkMode") func attachGateway(_ gateway: GatewayNodeSession) { self.gateway = gateway diff --git a/apps/ios/Tests/GatewaySettingsStoreTests.swift b/apps/ios/Tests/GatewaySettingsStoreTests.swift index 2f4df7964c1..746bf8fdf04 100644 --- a/apps/ios/Tests/GatewaySettingsStoreTests.swift +++ b/apps/ios/Tests/GatewaySettingsStoreTests.swift @@ -7,8 +7,8 @@ private struct KeychainEntry: Hashable { let account: String } -private let gatewayService = "com.clawdbot.gateway" -private let nodeService = "com.clawdbot.node" +private let gatewayService = "bot.molt.gateway" +private let nodeService = "bot.molt.node" private let instanceIdEntry = KeychainEntry(service: nodeService, account: "instanceId") private let preferredGatewayEntry = KeychainEntry(service: gatewayService, account: "preferredStableID") private let lastGatewayEntry = KeychainEntry(service: gatewayService, account: "lastDiscoveredStableID") diff --git a/apps/ios/Tests/KeychainStoreTests.swift b/apps/ios/Tests/KeychainStoreTests.swift index 798137b7e29..8aa5ae07162 100644 --- a/apps/ios/Tests/KeychainStoreTests.swift +++ b/apps/ios/Tests/KeychainStoreTests.swift @@ -4,7 +4,7 @@ import Testing @Suite struct KeychainStoreTests { @Test func saveLoadUpdateDeleteRoundTrip() { - let service = "com.clawdbot.tests.\(UUID().uuidString)" + let service = "bot.molt.tests.\(UUID().uuidString)" let account = "value" #expect(KeychainStore.delete(service: service, account: account)) diff --git a/apps/ios/fastlane/Appfile b/apps/ios/fastlane/Appfile index 7942da6254b..adaa3fc29fb 100644 --- a/apps/ios/fastlane/Appfile +++ b/apps/ios/fastlane/Appfile @@ -1,4 +1,4 @@ -app_identifier("com.clawdbot.ios") +app_identifier("bot.molt.ios") # Auth is expected via App Store Connect API key. # Provide either: diff --git a/apps/ios/project.yml b/apps/ios/project.yml index cdd16d4d13b..a7305c26c70 100644 --- a/apps/ios/project.yml +++ b/apps/ios/project.yml @@ -1,6 +1,6 @@ name: Moltbot options: - bundleIdPrefix: com.clawdbot + bundleIdPrefix: bot.molt deploymentTarget: iOS: "18.0" xcodeVersion: "16.0" @@ -71,8 +71,8 @@ targets: CODE_SIGN_IDENTITY: "Apple Development" CODE_SIGN_STYLE: Manual DEVELOPMENT_TEAM: Y5PE65HELJ - PRODUCT_BUNDLE_IDENTIFIER: com.clawdbot.ios - PROVISIONING_PROFILE_SPECIFIER: "com.clawdbot.ios Development" + PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios + PROVISIONING_PROFILE_SPECIFIER: "bot.molt.ios Development" SWIFT_VERSION: "6.0" SWIFT_STRICT_CONCURRENCY: complete ENABLE_APPINTENTS_METADATA: NO @@ -121,7 +121,7 @@ targets: - sdk: AppIntents.framework settings: base: - PRODUCT_BUNDLE_IDENTIFIER: com.clawdbot.ios.tests + PRODUCT_BUNDLE_IDENTIFIER: bot.molt.ios.tests SWIFT_VERSION: "6.0" SWIFT_STRICT_CONCURRENCY: complete TEST_HOST: "$(BUILT_PRODUCTS_DIR)/Moltbot.app/Moltbot" diff --git a/apps/shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift b/apps/shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift index a3eff72f527..6ef6c71b684 100644 --- a/apps/shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift +++ b/apps/shared/MoltbotKit/Sources/MoltbotChatUI/ChatViewModel.swift @@ -10,7 +10,7 @@ import AppKit import UIKit #endif -private let chatUILogger = Logger(subsystem: "com.clawdbot", category: "MoltbotChatUI") +private let chatUILogger = Logger(subsystem: "bot.molt", category: "MoltbotChatUI") @MainActor @Observable diff --git a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayChannel.swift b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayChannel.swift index c1562b2d971..0ead3021ce4 100644 --- a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayChannel.swift +++ b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayChannel.swift @@ -109,7 +109,7 @@ private enum ConnectChallengeError: Error { } public actor GatewayChannelActor { - private let logger = Logger(subsystem: "com.clawdbot", category: "gateway") + private let logger = Logger(subsystem: "bot.molt", category: "gateway") private var task: WebSocketTaskBox? private var pending: [String: CheckedContinuation] = [:] private var connected = false diff --git a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayNodeSession.swift b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayNodeSession.swift index daf4397d122..570342ce40d 100644 --- a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayNodeSession.swift +++ b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayNodeSession.swift @@ -12,7 +12,7 @@ private struct NodeInvokeRequestPayload: Codable, Sendable { } public actor GatewayNodeSession { - private let logger = Logger(subsystem: "com.clawdbot", category: "node.gateway") + private let logger = Logger(subsystem: "bot.molt", category: "node.gateway") private let decoder = JSONDecoder() private let encoder = JSONEncoder() private var channel: GatewayChannelActor? diff --git a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayTLSPinning.swift b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayTLSPinning.swift index f22505eff2a..4ce98603fb3 100644 --- a/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayTLSPinning.swift +++ b/apps/shared/MoltbotKit/Sources/MoltbotKit/GatewayTLSPinning.swift @@ -17,17 +17,30 @@ public struct GatewayTLSParams: Sendable { } public enum GatewayTLSStore { - private static let suiteName = "com.clawdbot.shared" + private static let suiteName = "bot.molt.shared" + private static let legacySuiteName = "com.clawdbot.shared" private static let keyPrefix = "gateway.tls." private static var defaults: UserDefaults { UserDefaults(suiteName: suiteName) ?? .standard } + private static var legacyDefaults: UserDefaults? { + UserDefaults(suiteName: legacySuiteName) + } + public static func loadFingerprint(stableID: String) -> String? { let key = self.keyPrefix + stableID let raw = self.defaults.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) - return raw?.isEmpty == false ? raw : nil + if raw?.isEmpty == false { return raw } + + let legacy = self.legacyDefaults?.string(forKey: key)?.trimmingCharacters(in: .whitespacesAndNewlines) + if legacy?.isEmpty == false { + self.defaults.set(legacy, forKey: key) + return legacy + } + + return nil } public static func saveFingerprint(_ value: String, stableID: String) { diff --git a/apps/shared/MoltbotKit/Sources/MoltbotKit/InstanceIdentity.swift b/apps/shared/MoltbotKit/Sources/MoltbotKit/InstanceIdentity.swift index e1a52ff399b..cbc82432939 100644 --- a/apps/shared/MoltbotKit/Sources/MoltbotKit/InstanceIdentity.swift +++ b/apps/shared/MoltbotKit/Sources/MoltbotKit/InstanceIdentity.swift @@ -5,13 +5,18 @@ import UIKit #endif public enum InstanceIdentity { - private static let suiteName = "com.clawdbot.shared" + private static let suiteName = "bot.molt.shared" + private static let legacySuiteName = "com.clawdbot.shared" private static let instanceIdKey = "instanceId" private static var defaults: UserDefaults { UserDefaults(suiteName: suiteName) ?? .standard } + private static var legacyDefaults: UserDefaults? { + UserDefaults(suiteName: legacySuiteName) + } + #if canImport(UIKit) private static func readMainActor(_ body: @MainActor () -> T) -> T { if Thread.isMainThread { @@ -32,6 +37,14 @@ public enum InstanceIdentity { return existing } + if let legacy = Self.legacyDefaults?.string(forKey: instanceIdKey)? + .trimmingCharacters(in: .whitespacesAndNewlines), + !legacy.isEmpty + { + defaults.set(legacy, forKey: instanceIdKey) + return legacy + } + let id = UUID().uuidString.lowercased() defaults.set(id, forKey: instanceIdKey) return id diff --git a/docs/gateway/index.md b/docs/gateway/index.md index 65ef3d61d0d..8c5be0fa849 100644 --- a/docs/gateway/index.md +++ b/docs/gateway/index.md @@ -56,7 +56,7 @@ Usually unnecessary: one Gateway can serve multiple messaging channels and agent Supported if you isolate state + config and use unique ports. Full guide: [Multiple gateways](/gateway/multiple-gateways). Service names are profile-aware: -- macOS: `com.clawdbot.` +- macOS: `bot.molt.` (legacy `com.clawdbot.*` may still exist) - Linux: `moltbot-gateway-.service` - Windows: `Moltbot Gateway ()` @@ -181,8 +181,8 @@ See also: [Presence](/concepts/presence) for how presence is produced/deduped an - StandardOut/Err: file paths or `syslog` - On failure, launchd restarts; fatal misconfig should keep exiting so the operator notices. - LaunchAgents are per-user and require a logged-in session; for headless setups use a custom LaunchDaemon (not shipped). - - `moltbot gateway install` writes `~/Library/LaunchAgents/com.clawdbot.gateway.plist` - (or `com.clawdbot..plist`). + - `moltbot gateway install` writes `~/Library/LaunchAgents/bot.molt.gateway.plist` + (or `bot.molt..plist`; legacy `com.clawdbot.*` is cleaned up). - `moltbot doctor` audits the LaunchAgent config and can update it to current defaults. ## Gateway service management (CLI) @@ -213,11 +213,11 @@ Notes: Bundled mac app: - Moltbot.app can bundle a Node-based gateway relay and install a per-user LaunchAgent labeled - `com.clawdbot.gateway` (or `com.clawdbot.`). -- To stop it cleanly, use `moltbot gateway stop` (or `launchctl bootout gui/$UID/com.clawdbot.gateway`). -- To restart, use `moltbot gateway restart` (or `launchctl kickstart -k gui/$UID/com.clawdbot.gateway`). + `bot.molt.gateway` (or `bot.molt.`; legacy `com.clawdbot.*` labels still unload cleanly). +- To stop it cleanly, use `moltbot gateway stop` (or `launchctl bootout gui/$UID/bot.molt.gateway`). +- To restart, use `moltbot gateway restart` (or `launchctl kickstart -k gui/$UID/bot.molt.gateway`). - `launchctl` only works if the LaunchAgent is installed; otherwise use `moltbot gateway install` first. - - Replace the label with `com.clawdbot.` when running a named profile. + - Replace the label with `bot.molt.` when running a named profile. ## Supervision (systemd user unit) Moltbot installs a **systemd user service** by default on Linux/WSL2. We diff --git a/docs/gateway/remote-gateway-readme.md b/docs/gateway/remote-gateway-readme.md index f0923ba7cb9..b48a746d782 100644 --- a/docs/gateway/remote-gateway-readme.md +++ b/docs/gateway/remote-gateway-readme.md @@ -82,7 +82,7 @@ To have the SSH tunnel start automatically when you log in, create a Launch Agen ### Create the PLIST file -Save this as `~/Library/LaunchAgents/com.clawdbot.ssh-tunnel.plist`: +Save this as `~/Library/LaunchAgents/bot.molt.ssh-tunnel.plist`: ```xml @@ -90,7 +90,7 @@ Save this as `~/Library/LaunchAgents/com.clawdbot.ssh-tunnel.plist`: Label - com.clawdbot.ssh-tunnel + bot.molt.ssh-tunnel ProgramArguments /usr/bin/ssh @@ -108,7 +108,7 @@ Save this as `~/Library/LaunchAgents/com.clawdbot.ssh-tunnel.plist`: ### Load the Launch Agent ```bash -launchctl bootstrap gui/$UID ~/Library/LaunchAgents/com.clawdbot.ssh-tunnel.plist +launchctl bootstrap gui/$UID ~/Library/LaunchAgents/bot.molt.ssh-tunnel.plist ``` The tunnel will now: @@ -116,6 +116,8 @@ The tunnel will now: - Restart if it crashes - Keep running in the background +Legacy note: remove any leftover `com.clawdbot.ssh-tunnel` LaunchAgent if present. + --- ## Troubleshooting @@ -130,13 +132,13 @@ lsof -i :18789 **Restart the tunnel:** ```bash -launchctl kickstart -k gui/$UID/com.clawdbot.ssh-tunnel +launchctl kickstart -k gui/$UID/bot.molt.ssh-tunnel ``` **Stop the tunnel:** ```bash -launchctl bootout gui/$UID/com.clawdbot.ssh-tunnel +launchctl bootout gui/$UID/bot.molt.ssh-tunnel ``` --- diff --git a/docs/gateway/troubleshooting.md b/docs/gateway/troubleshooting.md index 48c14318fc5..a4b0b151d70 100644 --- a/docs/gateway/troubleshooting.md +++ b/docs/gateway/troubleshooting.md @@ -576,7 +576,7 @@ If the app disappears or shows "Abort trap 6" when you click "Allow" on a privac **Fix 1: Reset TCC Cache** ```bash -tccutil reset All com.clawdbot.mac.debug +tccutil reset All bot.molt.mac.debug ``` **Fix 2: Force New Bundle ID** @@ -591,7 +591,7 @@ If the gateway is supervised by launchd, killing the PID will just respawn it. S ```bash moltbot gateway status moltbot gateway stop -# Or: launchctl bootout gui/$UID/com.clawdbot.gateway (replace with com.clawdbot. if needed) +# Or: launchctl bootout gui/$UID/bot.molt.gateway (replace with bot.molt.; legacy com.clawdbot.* still works) ``` **Fix 2: Port is busy (find the listener)** diff --git a/docs/help/faq.md b/docs/help/faq.md index 0766f64b44a..1b4f3b7baca 100644 --- a/docs/help/faq.md +++ b/docs/help/faq.md @@ -2328,7 +2328,7 @@ Quick setup (recommended): - Set a unique `gateway.port` in each profile config (or pass `--port` for manual runs). - Install a per-profile service: `moltbot --profile gateway install`. -Profiles also suffix service names (`com.clawdbot.`, `moltbot-gateway-.service`, `Moltbot Gateway ()`). +Profiles also suffix service names (`bot.molt.`; legacy `com.clawdbot.*`, `moltbot-gateway-.service`, `Moltbot Gateway ()`). Full guide: [Multiple gateways](/gateway/multiple-gateways). ### What does invalid handshake code 1008 mean diff --git a/docs/install/nix.md b/docs/install/nix.md index ee2e099975b..b6767742351 100644 --- a/docs/install/nix.md +++ b/docs/install/nix.md @@ -57,7 +57,7 @@ On macOS, the GUI app does not automatically inherit shell env vars. You can also enable Nix mode via defaults: ```bash -defaults write com.clawdbot.mac moltbot.nixMode -bool true +defaults write bot.molt.mac moltbot.nixMode -bool true ``` ### Config + state paths diff --git a/docs/install/uninstall.md b/docs/install/uninstall.md index 8d8be7a11a1..f3a180caa07 100644 --- a/docs/install/uninstall.md +++ b/docs/install/uninstall.md @@ -78,14 +78,14 @@ Use this if the gateway service keeps running but `moltbot` is missing. ### macOS (launchd) -Default label is `com.clawdbot.gateway` (or `com.clawdbot.`): +Default label is `bot.molt.gateway` (or `bot.molt.`; legacy `com.clawdbot.*` may still exist): ```bash -launchctl bootout gui/$UID/com.clawdbot.gateway -rm -f ~/Library/LaunchAgents/com.clawdbot.gateway.plist +launchctl bootout gui/$UID/bot.molt.gateway +rm -f ~/Library/LaunchAgents/bot.molt.gateway.plist ``` -If you used a profile, replace the label and plist name with `com.clawdbot.`. +If you used a profile, replace the label and plist name with `bot.molt.`. Remove any legacy `com.clawdbot.*` plists if present. ### Linux (systemd user unit) diff --git a/docs/install/updating.md b/docs/install/updating.md index 8ee27f7ad3a..634abfe9941 100644 --- a/docs/install/updating.md +++ b/docs/install/updating.md @@ -158,7 +158,7 @@ moltbot logs --follow ``` If you’re supervised: -- macOS launchd (app-bundled LaunchAgent): `launchctl kickstart -k gui/$UID/com.clawdbot.gateway` (use `com.clawdbot.` if set) +- macOS launchd (app-bundled LaunchAgent): `launchctl kickstart -k gui/$UID/bot.molt.gateway` (use `bot.molt.`; legacy `com.clawdbot.*` still works) - Linux systemd user service: `systemctl --user restart moltbot-gateway[-].service` - Windows (WSL2): `systemctl --user restart moltbot-gateway[-].service` - `launchctl`/`systemctl` only work if the service is installed; otherwise run `moltbot gateway install`. diff --git a/docs/platforms/index.md b/docs/platforms/index.md index a4a34b4ab80..65eeac2ed0d 100644 --- a/docs/platforms/index.md +++ b/docs/platforms/index.md @@ -46,5 +46,5 @@ Use one of these (all supported): - Repair/migrate: `moltbot doctor` (offers to install or fix the service) The service target depends on OS: -- macOS: LaunchAgent (`com.clawdbot.gateway` or `com.clawdbot.`) +- macOS: LaunchAgent (`bot.molt.gateway` or `bot.molt.`; legacy `com.clawdbot.*`) - Linux/WSL2: systemd user service (`moltbot-gateway[-].service`) diff --git a/docs/platforms/mac/bundled-gateway.md b/docs/platforms/mac/bundled-gateway.md index f8fb9179f88..909fddcfcb8 100644 --- a/docs/platforms/mac/bundled-gateway.md +++ b/docs/platforms/mac/bundled-gateway.md @@ -26,11 +26,11 @@ The macOS app’s **Install CLI** button runs the same flow via npm/pnpm (bun no ## Launchd (Gateway as LaunchAgent) Label: -- `com.clawdbot.gateway` (or `com.clawdbot.`) +- `bot.molt.gateway` (or `bot.molt.`; legacy `com.clawdbot.*` may remain) Plist location (per‑user): -- `~/Library/LaunchAgents/com.clawdbot.gateway.plist` - (or `~/Library/LaunchAgents/com.clawdbot..plist`) +- `~/Library/LaunchAgents/bot.molt.gateway.plist` + (or `~/Library/LaunchAgents/bot.molt..plist`) Manager: - The macOS app owns LaunchAgent install/update in Local mode. diff --git a/docs/platforms/mac/child-process.md b/docs/platforms/mac/child-process.md index 6483f1df58e..d8b2d8728c8 100644 --- a/docs/platforms/mac/child-process.md +++ b/docs/platforms/mac/child-process.md @@ -16,8 +16,8 @@ If you need tighter coupling to the UI, run the Gateway manually in a terminal. ## Default behavior (launchd) -- The app installs a per‑user LaunchAgent labeled `com.clawdbot.gateway` - (or `com.clawdbot.` when using `--profile`/`CLAWDBOT_PROFILE`). +- The app installs a per‑user LaunchAgent labeled `bot.molt.gateway` + (or `bot.molt.` when using `--profile`/`CLAWDBOT_PROFILE`; legacy `com.clawdbot.*` is supported). - When Local mode is enabled, the app ensures the LaunchAgent is loaded and starts the Gateway if needed. - Logs are written to the launchd gateway log path (visible in Debug Settings). @@ -25,11 +25,11 @@ If you need tighter coupling to the UI, run the Gateway manually in a terminal. Common commands: ```bash -launchctl kickstart -k gui/$UID/com.clawdbot.gateway -launchctl bootout gui/$UID/com.clawdbot.gateway +launchctl kickstart -k gui/$UID/bot.molt.gateway +launchctl bootout gui/$UID/bot.molt.gateway ``` -Replace the label with `com.clawdbot.` when running a named profile. +Replace the label with `bot.molt.` when running a named profile. ## Unsigned dev builds diff --git a/docs/platforms/mac/dev-setup.md b/docs/platforms/mac/dev-setup.md index 179589b263a..af0883e18dc 100644 --- a/docs/platforms/mac/dev-setup.md +++ b/docs/platforms/mac/dev-setup.md @@ -74,7 +74,7 @@ If the app crashes when you try to allow **Speech Recognition** or **Microphone* **Fix:** 1. Reset the TCC permissions: ```bash - tccutil reset All com.clawdbot.mac.debug + tccutil reset All bot.molt.mac.debug ``` 2. If that fails, change the `BUNDLE_ID` temporarily in [`scripts/package-mac-app.sh`](https://github.com/moltbot/moltbot/blob/main/scripts/package-mac-app.sh) to force a "clean slate" from macOS. diff --git a/docs/platforms/mac/logging.md b/docs/platforms/mac/logging.md index b7a9d9d330e..9a5594d7d0a 100644 --- a/docs/platforms/mac/logging.md +++ b/docs/platforms/mac/logging.md @@ -22,11 +22,11 @@ Notes: Unified logging redacts most payloads unless a subsystem opts into `privacy -off`. Per Peter's write-up on macOS [logging privacy shenanigans](https://steipete.me/posts/2025/logging-privacy-shenanigans) (2025) this is controlled by a plist in `/Library/Preferences/Logging/Subsystems/` keyed by the subsystem name. Only new log entries pick up the flag, so enable it before reproducing an issue. -## Enable for Moltbot (`com.clawdbot`) +## Enable for Moltbot (`bot.molt`) - Write the plist to a temp file first, then install it atomically as root: ```bash -cat <<'EOF' >/tmp/com.clawdbot.plist +cat <<'EOF' >/tmp/bot.molt.plist @@ -39,13 +39,13 @@ cat <<'EOF' >/tmp/com.clawdbot.plist EOF -sudo install -m 644 -o root -g wheel /tmp/com.clawdbot.plist /Library/Preferences/Logging/Subsystems/com.clawdbot.plist +sudo install -m 644 -o root -g wheel /tmp/bot.molt.plist /Library/Preferences/Logging/Subsystems/bot.molt.plist ``` - No reboot is required; logd notices the file quickly, but only new log lines will include private payloads. - View the richer output with the existing helper, e.g. `./scripts/clawlog.sh --category WebChat --last 5m`. ## Disable after debugging -- Remove the override: `sudo rm /Library/Preferences/Logging/Subsystems/com.clawdbot.plist`. +- Remove the override: `sudo rm /Library/Preferences/Logging/Subsystems/bot.molt.plist`. - Optionally run `sudo log config --reload` to force logd to drop the override immediately. - Remember this surface can include phone numbers and message bodies; keep the plist in place only while you actively need the extra detail. diff --git a/docs/platforms/mac/permissions.md b/docs/platforms/mac/permissions.md index bfc8f099ee8..d2570829c63 100644 --- a/docs/platforms/mac/permissions.md +++ b/docs/platforms/mac/permissions.md @@ -31,8 +31,8 @@ grants, and prompts can disappear entirely until the stale entries are cleared. Example resets (replace bundle ID as needed): ```bash -sudo tccutil reset Accessibility com.clawdbot.mac -sudo tccutil reset ScreenCapture com.clawdbot.mac +sudo tccutil reset Accessibility bot.molt.mac +sudo tccutil reset ScreenCapture bot.molt.mac sudo tccutil reset AppleEvents ``` diff --git a/docs/platforms/mac/release.md b/docs/platforms/mac/release.md index 4e6d428dacd..4be82c67a27 100644 --- a/docs/platforms/mac/release.md +++ b/docs/platforms/mac/release.md @@ -29,7 +29,7 @@ Notes: ```bash # From repo root; set release IDs so Sparkle feed is enabled. # APP_BUILD must be numeric + monotonic for Sparkle compare. -BUNDLE_ID=com.clawdbot.mac \ +BUNDLE_ID=bot.molt.mac \ APP_VERSION=2026.1.26 \ APP_BUILD="$(git rev-list --count HEAD)" \ BUILD_CONFIG=release \ @@ -47,7 +47,7 @@ scripts/create-dmg.sh dist/Moltbot.app dist/Moltbot-2026.1.26.dmg # xcrun notarytool store-credentials "moltbot-notary" \ # --apple-id "" --team-id "" --password "" NOTARIZE=1 NOTARYTOOL_PROFILE=moltbot-notary \ -BUNDLE_ID=com.clawdbot.mac \ +BUNDLE_ID=bot.molt.mac \ APP_VERSION=2026.1.26 \ APP_BUILD="$(git rev-list --count HEAD)" \ BUILD_CONFIG=release \ diff --git a/docs/platforms/mac/signing.md b/docs/platforms/mac/signing.md index cef1b359d29..71f1537657e 100644 --- a/docs/platforms/mac/signing.md +++ b/docs/platforms/mac/signing.md @@ -7,7 +7,7 @@ read_when: This app is usually built from [`scripts/package-mac-app.sh`](https://github.com/moltbot/moltbot/blob/main/scripts/package-mac-app.sh), which now: -- sets a stable debug bundle identifier: `com.clawdbot.mac.debug` +- sets a stable debug bundle identifier: `bot.molt.mac.debug` - writes the Info.plist with that bundle id (override via `BUNDLE_ID=...`) - calls [`scripts/codesign-mac-app.sh`](https://github.com/moltbot/moltbot/blob/main/scripts/codesign-mac-app.sh) to sign the main binary and app bundle so macOS treats each rebuild as the same signed bundle and keeps TCC permissions (notifications, accessibility, screen recording, mic, speech). For stable permissions, use a real signing identity; ad-hoc is opt-in and fragile (see [macOS permissions](/platforms/mac/permissions)). - uses `CODESIGN_TIMESTAMP=auto` by default; it enables trusted timestamps for Developer ID signatures. Set `CODESIGN_TIMESTAMP=off` to skip timestamping (offline debug builds). diff --git a/docs/platforms/mac/voice-overlay.md b/docs/platforms/mac/voice-overlay.md index 5d755fe6763..139445164f0 100644 --- a/docs/platforms/mac/voice-overlay.md +++ b/docs/platforms/mac/voice-overlay.md @@ -32,14 +32,14 @@ Audience: macOS app contributors. Goal: keep the voice overlay predictable when - Push-to-talk: no delay; wake-word: optional delay for auto-send. - Apply a short cooldown to the wake runtime after push-to-talk finishes so wake-word doesn’t immediately retrigger. 5. **Logging** - - Coordinator emits `.info` logs in subsystem `com.clawdbot`, categories `voicewake.overlay` and `voicewake.chime`. + - Coordinator emits `.info` logs in subsystem `bot.molt`, categories `voicewake.overlay` and `voicewake.chime`. - Key events: `session_started`, `adopted_by_push_to_talk`, `partial`, `finalized`, `send`, `dismiss`, `cancel`, `cooldown`. ### Debugging checklist - Stream logs while reproducing a sticky overlay: ```bash - sudo log stream --predicate 'subsystem == "com.clawdbot" AND category CONTAINS "voicewake"' --level info --style compact + sudo log stream --predicate 'subsystem == "bot.molt" AND category CONTAINS "voicewake"' --level info --style compact ``` - Verify only one active session token; stale callbacks should be dropped by the coordinator. - Ensure push-to-talk release always calls `endCapture` with the active token; if text is empty, expect `dismiss` without chime or send. diff --git a/docs/platforms/mac/webchat.md b/docs/platforms/mac/webchat.md index 80d5cfe2bf8..5f4e3230876 100644 --- a/docs/platforms/mac/webchat.md +++ b/docs/platforms/mac/webchat.md @@ -20,7 +20,7 @@ agent (with a session switcher for other sessions). ```bash dist/Moltbot.app/Contents/MacOS/Moltbot --webchat ``` -- Logs: `./scripts/clawlog.sh` (subsystem `com.clawdbot`, category `WebChatSwiftUI`). +- Logs: `./scripts/clawlog.sh` (subsystem `bot.molt`, category `WebChatSwiftUI`). ## How it’s wired diff --git a/docs/platforms/macos.md b/docs/platforms/macos.md index 8b00dc8c948..c98fe081766 100644 --- a/docs/platforms/macos.md +++ b/docs/platforms/macos.md @@ -32,15 +32,15 @@ The app does not spawn the Gateway as a child process. ## Launchd control -The app manages a per‑user LaunchAgent labeled `com.clawdbot.gateway` -(or `com.clawdbot.` when using `--profile`/`CLAWDBOT_PROFILE`). +The app manages a per‑user LaunchAgent labeled `bot.molt.gateway` +(or `bot.molt.` when using `--profile`/`CLAWDBOT_PROFILE`; legacy `com.clawdbot.*` still unloads). ```bash -launchctl kickstart -k gui/$UID/com.clawdbot.gateway -launchctl bootout gui/$UID/com.clawdbot.gateway +launchctl kickstart -k gui/$UID/bot.molt.gateway +launchctl bootout gui/$UID/bot.molt.gateway ``` -Replace the label with `com.clawdbot.` when running a named profile. +Replace the label with `bot.molt.` when running a named profile. If the LaunchAgent isn’t installed, enable it from the app or run `moltbot gateway install`. diff --git a/package.json b/package.json index 7c05bd9b41a..e1f1a8df7cf 100644 --- a/package.json +++ b/package.json @@ -102,10 +102,10 @@ "ios:gen": "cd apps/ios && xcodegen generate", "ios:open": "cd apps/ios && xcodegen generate && open Moltbot.xcodeproj", "ios:build": "bash -lc 'cd apps/ios && xcodegen generate && xcodebuild -project Moltbot.xcodeproj -scheme Moltbot -destination \"${IOS_DEST:-platform=iOS Simulator,name=iPhone 17}\" -configuration Debug build'", - "ios:run": "bash -lc 'cd apps/ios && xcodegen generate && xcodebuild -project Moltbot.xcodeproj -scheme Moltbot -destination \"${IOS_DEST:-platform=iOS Simulator,name=iPhone 17}\" -configuration Debug build && xcrun simctl boot \"${IOS_SIM:-iPhone 17}\" || true && xcrun simctl launch booted com.clawdbot.ios'", + "ios:run": "bash -lc 'cd apps/ios && xcodegen generate && xcodebuild -project Moltbot.xcodeproj -scheme Moltbot -destination \"${IOS_DEST:-platform=iOS Simulator,name=iPhone 17}\" -configuration Debug build && xcrun simctl boot \"${IOS_SIM:-iPhone 17}\" || true && xcrun simctl launch booted bot.molt.ios'", "android:assemble": "cd apps/android && ./gradlew :app:assembleDebug", "android:install": "cd apps/android && ./gradlew :app:installDebug", - "android:run": "cd apps/android && ./gradlew :app:installDebug && adb shell am start -n com.clawdbot.android/.MainActivity", + "android:run": "cd apps/android && ./gradlew :app:installDebug && adb shell am start -n bot.molt.android/.MainActivity", "android:test": "cd apps/android && ./gradlew :app:testDebugUnitTest", "mac:restart": "bash scripts/restart-mac.sh", "mac:package": "bash scripts/package-mac-app.sh", diff --git a/scripts/clawlog.sh b/scripts/clawlog.sh index 60e73498cd6..405887d8566 100755 --- a/scripts/clawlog.sh +++ b/scripts/clawlog.sh @@ -6,7 +6,7 @@ set -euo pipefail # Configuration -SUBSYSTEM="com.clawdbot" +SUBSYSTEM="bot.molt" DEFAULT_LEVEL="info" # Colors for output @@ -58,7 +58,7 @@ DESCRIPTION: Requires sudo access configured for /usr/bin/log command. LOG FLOW ARCHITECTURE: - Moltbot logs flow through the macOS unified log (subsystem: com.clawdbot). + Moltbot logs flow through the macOS unified log (subsystem: bot.molt). LOG CATEGORIES (examples): • voicewake - Voice wake detection/test harness diff --git a/scripts/package-mac-app.sh b/scripts/package-mac-app.sh index 62ea8048144..a63ecaf4de9 100755 --- a/scripts/package-mac-app.sh +++ b/scripts/package-mac-app.sh @@ -8,7 +8,7 @@ ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" APP_ROOT="$ROOT_DIR/dist/Moltbot.app" BUILD_ROOT="$ROOT_DIR/apps/macos/.build" PRODUCT="Moltbot" -BUNDLE_ID="${BUNDLE_ID:-com.clawdbot.mac.debug}" +BUNDLE_ID="${BUNDLE_ID:-bot.molt.mac.debug}" PKG_VERSION="$(cd "$ROOT_DIR" && node -p "require('./package.json').version" 2>/dev/null || echo "0.0.0")" BUILD_TS=$(date -u +"%Y-%m-%dT%H:%M:%SZ") GIT_COMMIT=$(cd "$ROOT_DIR" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") diff --git a/scripts/restart-mac.sh b/scripts/restart-mac.sh index 67ed81908ab..6dc81bb4e96 100755 --- a/scripts/restart-mac.sh +++ b/scripts/restart-mac.sh @@ -9,7 +9,7 @@ APP_PROCESS_PATTERN="Moltbot.app/Contents/MacOS/Moltbot" DEBUG_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build/debug/Moltbot" LOCAL_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build-local/debug/Moltbot" RELEASE_PROCESS_PATTERN="${ROOT_DIR}/apps/macos/.build/release/Moltbot" -LAUNCH_AGENT="${HOME}/Library/LaunchAgents/com.clawdbot.mac.plist" +LAUNCH_AGENT="${HOME}/Library/LaunchAgents/bot.molt.mac.plist" LOCK_KEY="$(printf '%s' "${ROOT_DIR}" | shasum -a 256 | cut -c1-8)" LOCK_DIR="${TMPDIR:-/tmp}/moltbot-restart-${LOCK_KEY}" LOCK_PID_FILE="${LOCK_DIR}/pid" @@ -145,7 +145,7 @@ kill_all_moltbot() { } stop_launch_agent() { - launchctl bootout gui/"$UID"/com.clawdbot.mac 2>/dev/null || true + launchctl bootout gui/"$UID"/bot.molt.mac 2>/dev/null || true } # 1) Kill all running instances first. @@ -265,5 +265,5 @@ else fi if [ "$NO_SIGN" -eq 1 ] && [ "$ATTACH_ONLY" -ne 1 ]; then - run_step "show gateway launch agent args (unsigned)" bash -lc "/usr/bin/plutil -p '${HOME}/Library/LaunchAgents/com.clawdbot.gateway.plist' | head -n 40 || true" + run_step "show gateway launch agent args (unsigned)" bash -lc "/usr/bin/plutil -p '${HOME}/Library/LaunchAgents/bot.molt.gateway.plist' | head -n 40 || true" fi diff --git a/src/cli/daemon-cli.coverage.test.ts b/src/cli/daemon-cli.coverage.test.ts index 6c8ff9cc97d..0a6d0739039 100644 --- a/src/cli/daemon-cli.coverage.test.ts +++ b/src/cli/daemon-cli.coverage.test.ts @@ -145,7 +145,7 @@ describe("daemon-cli coverage", () => { CLAWDBOT_CONFIG_PATH: "/tmp/moltbot-daemon-state/moltbot.json", CLAWDBOT_GATEWAY_PORT: "19001", }, - sourcePath: "/tmp/com.clawdbot.gateway.plist", + sourcePath: "/tmp/bot.molt.gateway.plist", }); const { registerDaemonCli } = await import("./daemon-cli.js"); diff --git a/src/commands/status.test.ts b/src/commands/status.test.ts index a425df6ad88..06d07d47681 100644 --- a/src/commands/status.test.ts +++ b/src/commands/status.test.ts @@ -255,7 +255,7 @@ vi.mock("../daemon/service.js", () => ({ readRuntime: async () => ({ status: "running", pid: 1234 }), readCommand: async () => ({ programArguments: ["node", "dist/entry.js", "gateway"], - sourcePath: "/tmp/Library/LaunchAgents/com.clawdbot.gateway.plist", + sourcePath: "/tmp/Library/LaunchAgents/bot.molt.gateway.plist", }), }), })); @@ -268,7 +268,7 @@ vi.mock("../daemon/node-service.js", () => ({ readRuntime: async () => ({ status: "running", pid: 4321 }), readCommand: async () => ({ programArguments: ["node", "dist/entry.js", "node-host"], - sourcePath: "/tmp/Library/LaunchAgents/com.clawdbot.node.plist", + sourcePath: "/tmp/Library/LaunchAgents/bot.molt.node.plist", }), }), })); diff --git a/src/daemon/constants.test.ts b/src/daemon/constants.test.ts index 4a82e4b7acd..854c527ea80 100644 --- a/src/daemon/constants.test.ts +++ b/src/daemon/constants.test.ts @@ -14,7 +14,7 @@ describe("resolveGatewayLaunchAgentLabel", () => { it("returns default label when no profile is set", () => { const result = resolveGatewayLaunchAgentLabel(); expect(result).toBe(GATEWAY_LAUNCH_AGENT_LABEL); - expect(result).toBe("com.clawdbot.gateway"); + expect(result).toBe("bot.molt.gateway"); }); it("returns default label when profile is undefined", () => { @@ -34,17 +34,17 @@ describe("resolveGatewayLaunchAgentLabel", () => { it("returns profile-specific label when profile is set", () => { const result = resolveGatewayLaunchAgentLabel("dev"); - expect(result).toBe("com.clawdbot.dev"); + expect(result).toBe("bot.molt.dev"); }); it("returns profile-specific label for custom profile", () => { const result = resolveGatewayLaunchAgentLabel("work"); - expect(result).toBe("com.clawdbot.work"); + expect(result).toBe("bot.molt.work"); }); it("trims whitespace from profile", () => { const result = resolveGatewayLaunchAgentLabel(" staging "); - expect(result).toBe("com.clawdbot.staging"); + expect(result).toBe("bot.molt.staging"); }); it("returns default label for empty string profile", () => { diff --git a/src/daemon/constants.ts b/src/daemon/constants.ts index 3a0325baa9d..b46a698177d 100644 --- a/src/daemon/constants.ts +++ b/src/daemon/constants.ts @@ -1,16 +1,19 @@ // Default service labels (for backward compatibility and when no profile specified) -export const GATEWAY_LAUNCH_AGENT_LABEL = "com.clawdbot.gateway"; +export const GATEWAY_LAUNCH_AGENT_LABEL = "bot.molt.gateway"; export const GATEWAY_SYSTEMD_SERVICE_NAME = "moltbot-gateway"; export const GATEWAY_WINDOWS_TASK_NAME = "Moltbot Gateway"; export const GATEWAY_SERVICE_MARKER = "moltbot"; export const GATEWAY_SERVICE_KIND = "gateway"; -export const NODE_LAUNCH_AGENT_LABEL = "com.clawdbot.node"; +export const NODE_LAUNCH_AGENT_LABEL = "bot.molt.node"; export const NODE_SYSTEMD_SERVICE_NAME = "moltbot-node"; export const NODE_WINDOWS_TASK_NAME = "Moltbot Node"; export const NODE_SERVICE_MARKER = "moltbot"; export const NODE_SERVICE_KIND = "node"; export const NODE_WINDOWS_TASK_SCRIPT_NAME = "node.cmd"; -export const LEGACY_GATEWAY_LAUNCH_AGENT_LABELS = ["com.steipete.clawdbot.gateway"]; +export const LEGACY_GATEWAY_LAUNCH_AGENT_LABELS = [ + "com.clawdbot.gateway", + "com.steipete.clawdbot.gateway", +]; export const LEGACY_GATEWAY_SYSTEMD_SERVICE_NAMES: string[] = []; export const LEGACY_GATEWAY_WINDOWS_TASK_NAMES: string[] = []; @@ -30,7 +33,15 @@ export function resolveGatewayLaunchAgentLabel(profile?: string): string { if (!normalized) { return GATEWAY_LAUNCH_AGENT_LABEL; } - return `com.clawdbot.${normalized}`; + return `bot.molt.${normalized}`; +} + +export function resolveLegacyGatewayLaunchAgentLabels(profile?: string): string[] { + const normalized = normalizeGatewayProfile(profile); + if (!normalized) { + return [...LEGACY_GATEWAY_LAUNCH_AGENT_LABELS]; + } + return [...LEGACY_GATEWAY_LAUNCH_AGENT_LABELS, `com.clawdbot.${normalized}`]; } export function resolveGatewaySystemdServiceName(profile?: string): string { diff --git a/src/daemon/inspect.ts b/src/daemon/inspect.ts index 1296a62d67b..46318956d7c 100644 --- a/src/daemon/inspect.ts +++ b/src/daemon/inspect.ts @@ -6,12 +6,12 @@ import { promisify } from "node:util"; import { GATEWAY_SERVICE_KIND, GATEWAY_SERVICE_MARKER, - LEGACY_GATEWAY_LAUNCH_AGENT_LABELS, LEGACY_GATEWAY_SYSTEMD_SERVICE_NAMES, LEGACY_GATEWAY_WINDOWS_TASK_NAMES, resolveGatewayLaunchAgentLabel, resolveGatewaySystemdServiceName, resolveGatewayWindowsTaskName, + resolveLegacyGatewayLaunchAgentLabels, } from "./constants.js"; export type ExtraGatewayService = { @@ -78,7 +78,7 @@ function isMoltbotGatewayLaunchdService(label: string, contents: string): boolea if (hasGatewayServiceMarker(contents)) return true; const lowerContents = contents.toLowerCase(); if (!lowerContents.includes("gateway")) return false; - return label.startsWith("com.clawdbot."); + return label.startsWith("bot.molt.") || label.startsWith("com.clawdbot."); } function isMoltbotGatewaySystemdService(name: string, contents: string): boolean { @@ -102,7 +102,8 @@ function tryExtractPlistLabel(contents: string): string | null { function isIgnoredLaunchdLabel(label: string): boolean { return ( - label === resolveGatewayLaunchAgentLabel() || LEGACY_GATEWAY_LAUNCH_AGENT_LABELS.includes(label) + label === resolveGatewayLaunchAgentLabel() || + resolveLegacyGatewayLaunchAgentLabels(process.env.CLAWDBOT_PROFILE).includes(label) ); } diff --git a/src/daemon/launchd.test.ts b/src/daemon/launchd.test.ts index 4954b6b15db..1052cb9b998 100644 --- a/src/daemon/launchd.test.ts +++ b/src/daemon/launchd.test.ts @@ -107,7 +107,7 @@ describe("launchd runtime parsing", () => { describe("launchctl list detection", () => { it("detects the resolved label in launchctl list", async () => { - await withLaunchctlStub({ listOutput: "123 0 com.clawdbot.gateway\n" }, async ({ env }) => { + await withLaunchctlStub({ listOutput: "123 0 bot.molt.gateway\n" }, async ({ env }) => { const listed = await isLaunchAgentListed({ env }); expect(listed).toBe(true); }); @@ -133,7 +133,7 @@ describe("launchd bootstrap repair", () => { .map((line) => JSON.parse(line) as string[]); const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501"; - const label = "com.clawdbot.gateway"; + const label = "bot.molt.gateway"; const plistPath = resolveLaunchAgentPlistPath(env); expect(calls).toContainEqual(["bootstrap", domain, plistPath]); @@ -201,7 +201,7 @@ describe("launchd install", () => { .map((line) => JSON.parse(line) as string[]); const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501"; - const label = "com.clawdbot.gateway"; + const label = "bot.molt.gateway"; const plistPath = resolveLaunchAgentPlistPath(env); const serviceId = `${domain}/${label}`; @@ -231,21 +231,21 @@ describe("resolveLaunchAgentPlistPath", () => { it("uses default label when CLAWDBOT_PROFILE is default", () => { const env = { HOME: "/Users/test", CLAWDBOT_PROFILE: "default" }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.gateway.plist", + "/Users/test/Library/LaunchAgents/bot.molt.gateway.plist", ); }); it("uses default label when CLAWDBOT_PROFILE is unset", () => { const env = { HOME: "/Users/test" }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.gateway.plist", + "/Users/test/Library/LaunchAgents/bot.molt.gateway.plist", ); }); it("uses profile-specific label when CLAWDBOT_PROFILE is set to a custom value", () => { const env = { HOME: "/Users/test", CLAWDBOT_PROFILE: "jbphoenix" }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.jbphoenix.plist", + "/Users/test/Library/LaunchAgents/bot.molt.jbphoenix.plist", ); }); @@ -277,28 +277,28 @@ describe("resolveLaunchAgentPlistPath", () => { CLAWDBOT_LAUNCHD_LABEL: " ", }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.myprofile.plist", + "/Users/test/Library/LaunchAgents/bot.molt.myprofile.plist", ); }); it("handles case-insensitive 'Default' profile", () => { const env = { HOME: "/Users/test", CLAWDBOT_PROFILE: "Default" }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.gateway.plist", + "/Users/test/Library/LaunchAgents/bot.molt.gateway.plist", ); }); it("handles case-insensitive 'DEFAULT' profile", () => { const env = { HOME: "/Users/test", CLAWDBOT_PROFILE: "DEFAULT" }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.gateway.plist", + "/Users/test/Library/LaunchAgents/bot.molt.gateway.plist", ); }); it("trims whitespace from CLAWDBOT_PROFILE", () => { const env = { HOME: "/Users/test", CLAWDBOT_PROFILE: " myprofile " }; expect(resolveLaunchAgentPlistPath(env)).toBe( - "/Users/test/Library/LaunchAgents/com.clawdbot.myprofile.plist", + "/Users/test/Library/LaunchAgents/bot.molt.myprofile.plist", ); }); }); diff --git a/src/daemon/launchd.ts b/src/daemon/launchd.ts index 529cfdc1a28..747494bf73e 100644 --- a/src/daemon/launchd.ts +++ b/src/daemon/launchd.ts @@ -7,8 +7,8 @@ import { colorize, isRich, theme } from "../terminal/theme.js"; import { formatGatewayServiceDescription, GATEWAY_LAUNCH_AGENT_LABEL, - LEGACY_GATEWAY_LAUNCH_AGENT_LABELS, resolveGatewayLaunchAgentLabel, + resolveLegacyGatewayLaunchAgentLabels, } from "./constants.js"; import { buildLaunchAgentPlist as buildLaunchAgentPlistImpl, @@ -248,7 +248,7 @@ export async function findLegacyLaunchAgents( ): Promise { const domain = resolveGuiDomain(); const results: LegacyLaunchAgent[] = []; - for (const label of LEGACY_GATEWAY_LAUNCH_AGENT_LABELS) { + for (const label of resolveLegacyGatewayLaunchAgentLabels(env.CLAWDBOT_PROFILE)) { const plistPath = resolveLaunchAgentPlistPathForLabel(env, label); const res = await execLaunchctl(["print", `${domain}/${label}`]); const loaded = res.code === 0; @@ -384,7 +384,7 @@ export async function installLaunchAgent({ const domain = resolveGuiDomain(); const label = resolveLaunchAgentLabel({ env }); - for (const legacyLabel of LEGACY_GATEWAY_LAUNCH_AGENT_LABELS) { + for (const legacyLabel of resolveLegacyGatewayLaunchAgentLabels(env.CLAWDBOT_PROFILE)) { const legacyPlistPath = resolveLaunchAgentPlistPathForLabel(env, legacyLabel); await execLaunchctl(["bootout", domain, legacyPlistPath]); await execLaunchctl(["unload", legacyPlistPath]); diff --git a/src/daemon/service-env.test.ts b/src/daemon/service-env.test.ts index 73e2fc5640d..8a5cc6072bf 100644 --- a/src/daemon/service-env.test.ts +++ b/src/daemon/service-env.test.ts @@ -230,7 +230,7 @@ describe("buildServiceEnvironment", () => { expect(typeof env.CLAWDBOT_SERVICE_VERSION).toBe("string"); expect(env.CLAWDBOT_SYSTEMD_UNIT).toBe("moltbot-gateway.service"); if (process.platform === "darwin") { - expect(env.CLAWDBOT_LAUNCHD_LABEL).toBe("com.clawdbot.gateway"); + expect(env.CLAWDBOT_LAUNCHD_LABEL).toBe("bot.molt.gateway"); } }); @@ -241,7 +241,7 @@ describe("buildServiceEnvironment", () => { }); expect(env.CLAWDBOT_SYSTEMD_UNIT).toBe("moltbot-gateway-work.service"); if (process.platform === "darwin") { - expect(env.CLAWDBOT_LAUNCHD_LABEL).toBe("com.clawdbot.work"); + expect(env.CLAWDBOT_LAUNCHD_LABEL).toBe("bot.molt.work"); } }); });