feat(android): add screen-tab canvas restore flow

This commit is contained in:
Ayaan Zaidi
2026-02-25 11:13:42 +05:30
committed by Ayaan Zaidi
parent e5399835b2
commit 1c0c58e30d
5 changed files with 140 additions and 1 deletions

View File

@@ -14,6 +14,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
private val runtime: NodeRuntime = (app as NodeApp).runtime
val canvas: CanvasController = runtime.canvas
val canvasCurrentUrl: StateFlow<String?> = runtime.canvas.currentUrl
val canvasA2uiHydrated: StateFlow<Boolean> = runtime.canvasA2uiHydrated
val canvasRehydratePending: StateFlow<Boolean> = runtime.canvasRehydratePending
val canvasRehydrateErrorText: StateFlow<String?> = runtime.canvasRehydrateErrorText
val camera: CameraCaptureManager = runtime.camera
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
val sms: SmsManager = runtime.sms
@@ -171,6 +175,10 @@ class MainViewModel(app: Application) : AndroidViewModel(app) {
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
}
fun requestCanvasRehydrate(source: String = "screen_tab") {
runtime.requestCanvasRehydrate(source = source, force = true)
}
fun loadChat(sessionKey: String) {
runtime.loadChat(sessionKey)
}

View File

@@ -164,6 +164,12 @@ class NodeRuntime(context: Context) {
isForeground = { _isForeground.value },
cameraEnabled = { cameraEnabled.value },
locationEnabled = { locationMode.value != LocationMode.Off },
onCanvasA2uiPush = {
_canvasA2uiHydrated.value = true
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
},
onCanvasA2uiReset = { _canvasA2uiHydrated.value = false },
)
private lateinit var gatewayEventHandler: GatewayEventHandler
@@ -195,6 +201,13 @@ class NodeRuntime(context: Context) {
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _canvasA2uiHydrated = MutableStateFlow(false)
val canvasA2uiHydrated: StateFlow<Boolean> = _canvasA2uiHydrated.asStateFlow()
private val _canvasRehydratePending = MutableStateFlow(false)
val canvasRehydratePending: StateFlow<Boolean> = _canvasRehydratePending.asStateFlow()
private val _canvasRehydrateErrorText = MutableStateFlow<String?>(null)
val canvasRehydrateErrorText: StateFlow<String?> = _canvasRehydrateErrorText.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
@@ -208,6 +221,8 @@ class NodeRuntime(context: Context) {
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
private var lastAutoA2uiUrl: String? = null
private var didAutoRequestCanvasRehydrate = false
private val canvasRehydrateSeq = AtomicLong(0)
private var operatorConnected = false
private var nodeConnected = false
private var operatorStatusText: String = "Offline"
@@ -257,12 +272,21 @@ class NodeRuntime(context: Context) {
onConnected = { _, _, _ ->
nodeConnected = true
nodeStatusText = "Connected"
didAutoRequestCanvasRehydrate = false
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
updateStatus()
maybeNavigateToA2uiOnConnect()
requestCanvasRehydrate(source = "node_connect", force = false)
},
onDisconnected = { message ->
nodeConnected = false
nodeStatusText = message
didAutoRequestCanvasRehydrate = false
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
updateStatus()
showLocalCanvasOnDisconnect()
},
@@ -329,9 +353,58 @@ class NodeRuntime(context: Context) {
private fun showLocalCanvasOnDisconnect() {
lastAutoA2uiUrl = null
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
canvas.navigate("")
}
fun requestCanvasRehydrate(source: String = "manual", force: Boolean = true) {
scope.launch {
if (!nodeConnected) return@launch
if (!force && didAutoRequestCanvasRehydrate) return@launch
didAutoRequestCanvasRehydrate = true
val requestId = canvasRehydrateSeq.incrementAndGet()
_canvasRehydratePending.value = true
_canvasRehydrateErrorText.value = null
val sessionKey = resolveMainSessionKey()
val prompt =
"Restore canvas now for session=$sessionKey source=$source. " +
"If existing A2UI state exists, replay it immediately. " +
"If not, create and render a compact mobile-friendly dashboard in Canvas."
try {
nodeSession.sendNodeEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(prompt))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
}.toString(),
)
scope.launch {
delay(20_000)
if (canvasRehydrateSeq.get() != requestId) return@launch
if (!_canvasRehydratePending.value) return@launch
if (_canvasA2uiHydrated.value) return@launch
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = "No canvas update yet. Tap to retry."
}
} catch (err: Throwable) {
if (!force) {
didAutoRequestCanvasRehydrate = false
}
if (canvasRehydrateSeq.get() == requestId) {
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = "Failed to request restore. Tap to retry."
}
Log.w("OpenClawCanvas", "canvas rehydrate request failed (${source}): ${err.message}")
}
}
}
val instanceId: StateFlow<String> = prefs.instanceId
val displayName: StateFlow<String> = prefs.displayName
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled

View File

@@ -10,6 +10,9 @@ import androidx.core.graphics.scale
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import java.io.ByteArrayOutputStream
import android.util.Base64
import org.json.JSONObject
@@ -31,6 +34,8 @@ class CanvasController {
@Volatile private var debugStatusEnabled: Boolean = false
@Volatile private var debugStatusTitle: String? = null
@Volatile private var debugStatusSubtitle: String? = null
private val _currentUrl = MutableStateFlow<String?>(null)
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
@@ -45,9 +50,16 @@ class CanvasController {
applyDebugStatus()
}
fun detach(webView: WebView) {
if (this.webView === webView) {
this.webView = null
}
}
fun navigate(url: String) {
val trimmed = url.trim()
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
_currentUrl.value = this.url
reload()
}

View File

@@ -20,6 +20,8 @@ class InvokeDispatcher(
private val isForeground: () -> Boolean,
private val cameraEnabled: () -> Boolean,
private val locationEnabled: () -> Boolean,
private val onCanvasA2uiPush: () -> Unit,
private val onCanvasA2uiReset: () -> Unit,
) {
suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
// Check foreground requirement for canvas/camera/screen commands
@@ -117,6 +119,7 @@ class InvokeDispatcher(
)
}
val res = canvas.eval(A2UIHandler.a2uiResetJS)
onCanvasA2uiReset()
GatewaySession.InvokeResult.ok(res)
}
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
@@ -143,6 +146,7 @@ class InvokeDispatcher(
}
val js = A2UIHandler.a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
onCanvasA2uiPush()
GatewaySession.InvokeResult.ok(res)
}

View File

@@ -115,13 +115,55 @@ fun PostOnboardingTabs(viewModel: MainViewModel, modifier: Modifier = Modifier)
HomeTab.Connect -> ConnectTabScreen(viewModel = viewModel)
HomeTab.Chat -> ChatSheet(viewModel = viewModel)
HomeTab.Voice -> ComingSoonTabScreen(label = "VOICE", title = "Coming soon", description = "Voice mode is coming soon.")
HomeTab.Screen -> ComingSoonTabScreen(label = "SCREEN", title = "Coming soon", description = "Screen mode is coming soon.")
HomeTab.Screen -> ScreenTabScreen(viewModel = viewModel)
HomeTab.Settings -> SettingsSheet(viewModel = viewModel)
}
}
}
}
@Composable
private fun ScreenTabScreen(viewModel: MainViewModel) {
val isConnected by viewModel.isConnected.collectAsState()
val canvasUrl by viewModel.canvasCurrentUrl.collectAsState()
val canvasA2uiHydrated by viewModel.canvasA2uiHydrated.collectAsState()
val canvasRehydratePending by viewModel.canvasRehydratePending.collectAsState()
val canvasRehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState()
val isA2uiUrl = canvasUrl?.contains("/__openclaw__/a2ui/") == true
val showRestoreCta = isConnected && (canvasUrl.isNullOrBlank() || (isA2uiUrl && !canvasA2uiHydrated))
val restoreCtaText =
when {
canvasRehydratePending -> "Restore requested. Waiting for agent…"
!canvasRehydrateErrorText.isNullOrBlank() -> canvasRehydrateErrorText!!
else -> "Canvas reset. Tap to restore dashboard."
}
Box(modifier = Modifier.fillMaxSize()) {
CanvasScreen(viewModel = viewModel, modifier = Modifier.fillMaxSize())
if (showRestoreCta) {
Surface(
onClick = {
if (canvasRehydratePending) return@Surface
viewModel.requestCanvasRehydrate(source = "screen_tab_cta")
},
modifier = Modifier.align(Alignment.TopCenter).padding(horizontal = 16.dp, vertical = 16.dp),
shape = RoundedCornerShape(12.dp),
color = mobileSurface.copy(alpha = 0.9f),
border = BorderStroke(1.dp, mobileBorder),
shadowElevation = 4.dp,
) {
Text(
text = restoreCtaText,
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
style = mobileCallout.copy(fontWeight = FontWeight.Medium),
color = mobileText,
)
}
}
}
}
@Composable
private fun TopStatusBar(
statusText: String,