mirror of
https://github.com/moltbot/moltbot.git
synced 2026-03-07 22:44:16 +00:00
feat(android): add screen-tab canvas restore flow
This commit is contained in:
@@ -14,6 +14,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
|
||||
private val runtime: NodeRuntime = (app as NodeApp).runtime
|
||||
|
||||
val canvas: CanvasController = runtime.canvas
|
||||
val canvasCurrentUrl: StateFlow<String?> = runtime.canvas.currentUrl
|
||||
val canvasA2uiHydrated: StateFlow<Boolean> = runtime.canvasA2uiHydrated
|
||||
val canvasRehydratePending: StateFlow<Boolean> = runtime.canvasRehydratePending
|
||||
val canvasRehydrateErrorText: StateFlow<String?> = runtime.canvasRehydrateErrorText
|
||||
val camera: CameraCaptureManager = runtime.camera
|
||||
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
|
||||
val sms: SmsManager = runtime.sms
|
||||
@@ -171,6 +175,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
|
||||
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
|
||||
}
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "screen_tab") {
|
||||
runtime.requestCanvasRehydrate(source = source, force = true)
|
||||
}
|
||||
|
||||
fun loadChat(sessionKey: String) {
|
||||
runtime.loadChat(sessionKey)
|
||||
}
|
||||
|
||||
@@ -164,6 +164,12 @@ class NodeRuntime(context: Context) {
|
||||
isForeground = { _isForeground.value },
|
||||
cameraEnabled = { cameraEnabled.value },
|
||||
locationEnabled = { locationMode.value != LocationMode.Off },
|
||||
onCanvasA2uiPush = {
|
||||
_canvasA2uiHydrated.value = true
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
},
|
||||
onCanvasA2uiReset = { _canvasA2uiHydrated.value = false },
|
||||
)
|
||||
|
||||
private lateinit var gatewayEventHandler: GatewayEventHandler
|
||||
@@ -195,6 +201,13 @@ class NodeRuntime(context: Context) {
|
||||
private val _screenRecordActive = MutableStateFlow(false)
|
||||
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
|
||||
|
||||
private val _canvasA2uiHydrated = MutableStateFlow(false)
|
||||
val canvasA2uiHydrated: StateFlow<Boolean> = _canvasA2uiHydrated.asStateFlow()
|
||||
private val _canvasRehydratePending = MutableStateFlow(false)
|
||||
val canvasRehydratePending: StateFlow<Boolean> = _canvasRehydratePending.asStateFlow()
|
||||
private val _canvasRehydrateErrorText = MutableStateFlow<String?>(null)
|
||||
val canvasRehydrateErrorText: StateFlow<String?> = _canvasRehydrateErrorText.asStateFlow()
|
||||
|
||||
private val _serverName = MutableStateFlow<String?>(null)
|
||||
val serverName: StateFlow<String?> = _serverName.asStateFlow()
|
||||
|
||||
@@ -208,6 +221,8 @@ class NodeRuntime(context: Context) {
|
||||
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
|
||||
|
||||
private var lastAutoA2uiUrl: String? = null
|
||||
private var didAutoRequestCanvasRehydrate = false
|
||||
private val canvasRehydrateSeq = AtomicLong(0)
|
||||
private var operatorConnected = false
|
||||
private var nodeConnected = false
|
||||
private var operatorStatusText: String = "Offline"
|
||||
@@ -257,12 +272,21 @@ class NodeRuntime(context: Context) {
|
||||
onConnected = { _, _, _ ->
|
||||
nodeConnected = true
|
||||
nodeStatusText = "Connected"
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
updateStatus()
|
||||
maybeNavigateToA2uiOnConnect()
|
||||
requestCanvasRehydrate(source = "node_connect", force = false)
|
||||
},
|
||||
onDisconnected = { message ->
|
||||
nodeConnected = false
|
||||
nodeStatusText = message
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
updateStatus()
|
||||
showLocalCanvasOnDisconnect()
|
||||
},
|
||||
@@ -329,9 +353,58 @@ class NodeRuntime(context: Context) {
|
||||
|
||||
private fun showLocalCanvasOnDisconnect() {
|
||||
lastAutoA2uiUrl = null
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
canvas.navigate("")
|
||||
}
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "manual", force: Boolean = true) {
|
||||
scope.launch {
|
||||
if (!nodeConnected) return@launch
|
||||
if (!force && didAutoRequestCanvasRehydrate) return@launch
|
||||
didAutoRequestCanvasRehydrate = true
|
||||
val requestId = canvasRehydrateSeq.incrementAndGet()
|
||||
_canvasRehydratePending.value = true
|
||||
_canvasRehydrateErrorText.value = null
|
||||
|
||||
val sessionKey = resolveMainSessionKey()
|
||||
val prompt =
|
||||
"Restore canvas now for session=$sessionKey source=$source. " +
|
||||
"If existing A2UI state exists, replay it immediately. " +
|
||||
"If not, create and render a compact mobile-friendly dashboard in Canvas."
|
||||
try {
|
||||
nodeSession.sendNodeEvent(
|
||||
event = "agent.request",
|
||||
payloadJson =
|
||||
buildJsonObject {
|
||||
put("message", JsonPrimitive(prompt))
|
||||
put("sessionKey", JsonPrimitive(sessionKey))
|
||||
put("thinking", JsonPrimitive("low"))
|
||||
put("deliver", JsonPrimitive(false))
|
||||
}.toString(),
|
||||
)
|
||||
scope.launch {
|
||||
delay(20_000)
|
||||
if (canvasRehydrateSeq.get() != requestId) return@launch
|
||||
if (!_canvasRehydratePending.value) return@launch
|
||||
if (_canvasA2uiHydrated.value) return@launch
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = "No canvas update yet. Tap to retry."
|
||||
}
|
||||
} catch (err: Throwable) {
|
||||
if (!force) {
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
}
|
||||
if (canvasRehydrateSeq.get() == requestId) {
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = "Failed to request restore. Tap to retry."
|
||||
}
|
||||
Log.w("OpenClawCanvas", "canvas rehydrate request failed (${source}): ${err.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val instanceId: StateFlow<String> = prefs.instanceId
|
||||
val displayName: StateFlow<String> = prefs.displayName
|
||||
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
|
||||
|
||||
@@ -10,6 +10,9 @@ import androidx.core.graphics.scale
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import java.io.ByteArrayOutputStream
|
||||
import android.util.Base64
|
||||
import org.json.JSONObject
|
||||
@@ -31,6 +34,8 @@ class CanvasController {
|
||||
@Volatile private var debugStatusEnabled: Boolean = false
|
||||
@Volatile private var debugStatusTitle: String? = null
|
||||
@Volatile private var debugStatusSubtitle: String? = null
|
||||
private val _currentUrl = MutableStateFlow<String?>(null)
|
||||
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
|
||||
|
||||
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
|
||||
|
||||
@@ -45,9 +50,16 @@ class CanvasController {
|
||||
applyDebugStatus()
|
||||
}
|
||||
|
||||
fun detach(webView: WebView) {
|
||||
if (this.webView === webView) {
|
||||
this.webView = null
|
||||
}
|
||||
}
|
||||
|
||||
fun navigate(url: String) {
|
||||
val trimmed = url.trim()
|
||||
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
|
||||
_currentUrl.value = this.url
|
||||
reload()
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,8 @@ class InvokeDispatcher(
|
||||
private val isForeground: () -> Boolean,
|
||||
private val cameraEnabled: () -> Boolean,
|
||||
private val locationEnabled: () -> Boolean,
|
||||
private val onCanvasA2uiPush: () -> Unit,
|
||||
private val onCanvasA2uiReset: () -> Unit,
|
||||
) {
|
||||
suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
|
||||
// Check foreground requirement for canvas/camera/screen commands
|
||||
@@ -117,6 +119,7 @@ class InvokeDispatcher(
|
||||
)
|
||||
}
|
||||
val res = canvas.eval(A2UIHandler.a2uiResetJS)
|
||||
onCanvasA2uiReset()
|
||||
GatewaySession.InvokeResult.ok(res)
|
||||
}
|
||||
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
|
||||
@@ -143,6 +146,7 @@ class InvokeDispatcher(
|
||||
}
|
||||
val js = A2UIHandler.a2uiApplyMessagesJS(messages)
|
||||
val res = canvas.eval(js)
|
||||
onCanvasA2uiPush()
|
||||
GatewaySession.InvokeResult.ok(res)
|
||||
}
|
||||
|
||||
|
||||
@@ -115,13 +115,55 @@ fun PostOnboardingTabs(viewModel: MainViewModel, modifier: Modifier = Modifier)
|
||||
HomeTab.Connect -> ConnectTabScreen(viewModel = viewModel)
|
||||
HomeTab.Chat -> ChatSheet(viewModel = viewModel)
|
||||
HomeTab.Voice -> ComingSoonTabScreen(label = "VOICE", title = "Coming soon", description = "Voice mode is coming soon.")
|
||||
HomeTab.Screen -> ComingSoonTabScreen(label = "SCREEN", title = "Coming soon", description = "Screen mode is coming soon.")
|
||||
HomeTab.Screen -> ScreenTabScreen(viewModel = viewModel)
|
||||
HomeTab.Settings -> SettingsSheet(viewModel = viewModel)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ScreenTabScreen(viewModel: MainViewModel) {
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
val canvasUrl by viewModel.canvasCurrentUrl.collectAsState()
|
||||
val canvasA2uiHydrated by viewModel.canvasA2uiHydrated.collectAsState()
|
||||
val canvasRehydratePending by viewModel.canvasRehydratePending.collectAsState()
|
||||
val canvasRehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState()
|
||||
val isA2uiUrl = canvasUrl?.contains("/__openclaw__/a2ui/") == true
|
||||
val showRestoreCta = isConnected && (canvasUrl.isNullOrBlank() || (isA2uiUrl && !canvasA2uiHydrated))
|
||||
val restoreCtaText =
|
||||
when {
|
||||
canvasRehydratePending -> "Restore requested. Waiting for agent…"
|
||||
!canvasRehydrateErrorText.isNullOrBlank() -> canvasRehydrateErrorText!!
|
||||
else -> "Canvas reset. Tap to restore dashboard."
|
||||
}
|
||||
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
CanvasScreen(viewModel = viewModel, modifier = Modifier.fillMaxSize())
|
||||
|
||||
if (showRestoreCta) {
|
||||
Surface(
|
||||
onClick = {
|
||||
if (canvasRehydratePending) return@Surface
|
||||
viewModel.requestCanvasRehydrate(source = "screen_tab_cta")
|
||||
},
|
||||
modifier = Modifier.align(Alignment.TopCenter).padding(horizontal = 16.dp, vertical = 16.dp),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
color = mobileSurface.copy(alpha = 0.9f),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
shadowElevation = 4.dp,
|
||||
) {
|
||||
Text(
|
||||
text = restoreCtaText,
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.Medium),
|
||||
color = mobileText,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun TopStatusBar(
|
||||
statusText: String,
|
||||
|
||||
Reference in New Issue
Block a user