feat(android): wire new device capabilities into runtime

This commit is contained in:
Ayaan Zaidi
2026-02-28 09:14:42 +05:30
committed by Ayaan Zaidi
parent b9e474deb4
commit 943dce37be
10 changed files with 294 additions and 0 deletions

View File

@@ -15,6 +15,15 @@
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.SEND_SMS" />
<uses-permission android:name="android.permission.READ_MEDIA_IMAGES" />
<uses-permission
android:name="android.permission.READ_EXTERNAL_STORAGE"
android:maxSdkVersion="32" />
<uses-permission android:name="android.permission.READ_CONTACTS" />
<uses-permission android:name="android.permission.WRITE_CONTACTS" />
<uses-permission android:name="android.permission.READ_CALENDAR" />
<uses-permission android:name="android.permission.WRITE_CALENDAR" />
<uses-permission android:name="android.permission.ACTIVITY_RECOGNITION" />
<uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
<uses-feature
android:name="android.hardware.camera"

View File

@@ -98,6 +98,26 @@ class NodeRuntime(context: Context) {
appContext = appContext,
)
private val systemHandler: SystemHandler = SystemHandler(
appContext = appContext,
)
private val photosHandler: PhotosHandler = PhotosHandler(
appContext = appContext,
)
private val contactsHandler: ContactsHandler = ContactsHandler(
appContext = appContext,
)
private val calendarHandler: CalendarHandler = CalendarHandler(
appContext = appContext,
)
private val motionHandler: MotionHandler = MotionHandler(
appContext = appContext,
)
private val screenHandler: ScreenHandler = ScreenHandler(
screenRecorder = screenRecorder,
setScreenRecordActive = { _screenRecordActive.value = it },
@@ -120,6 +140,7 @@ class NodeRuntime(context: Context) {
cameraEnabled = { cameraEnabled.value },
locationMode = { locationMode.value },
voiceWakeMode = { VoiceWakeMode.Off },
motionAvailable = { motionHandler.isAvailable() },
smsAvailable = { sms.canSendSms() },
hasRecordAudioPermission = { hasRecordAudioPermission() },
manualTls = { manualTls.value },
@@ -131,6 +152,11 @@ class NodeRuntime(context: Context) {
locationHandler = locationHandler,
deviceHandler = deviceHandler,
notificationsHandler = notificationsHandler,
systemHandler = systemHandler,
photosHandler = photosHandler,
contactsHandler = contactsHandler,
calendarHandler = calendarHandler,
motionHandler = motionHandler,
screenHandler = screenHandler,
smsHandler = smsHandlerImpl,
a2uiHandler = a2uiHandler,
@@ -139,6 +165,7 @@ class NodeRuntime(context: Context) {
isForeground = { _isForeground.value },
cameraEnabled = { cameraEnabled.value },
locationEnabled = { locationMode.value != LocationMode.Off },
motionAvailable = { motionHandler.isAvailable() },
smsAvailable = { sms.canSendSms() },
debugBuild = { BuildConfig.DEBUG },
refreshNodeCanvasCapability = { nodeSession.refreshNodeCanvasCapability() },

View File

@@ -15,6 +15,7 @@ class ConnectionManager(
private val cameraEnabled: () -> Boolean,
private val locationMode: () -> LocationMode,
private val voiceWakeMode: () -> VoiceWakeMode,
private val motionAvailable: () -> Boolean,
private val smsAvailable: () -> Boolean,
private val hasRecordAudioPermission: () -> Boolean,
private val manualTls: () -> Boolean,
@@ -78,6 +79,7 @@ class ConnectionManager(
locationEnabled = locationMode() != LocationMode.Off,
smsAvailable = smsAvailable(),
voiceWakeEnabled = voiceWakeMode() != VoiceWakeMode.Off && hasRecordAudioPermission(),
motionAvailable = motionAvailable(),
debugBuild = BuildConfig.DEBUG,
)

View File

@@ -130,6 +130,24 @@ class DeviceHandler(
private fun permissionsPayloadJson(): String {
val canSendSms = appContext.packageManager.hasSystemFeature(PackageManager.FEATURE_TELEPHONY)
val notificationAccess = DeviceNotificationListenerService.isAccessEnabled(appContext)
val photosGranted =
if (Build.VERSION.SDK_INT >= 33) {
hasPermission(Manifest.permission.READ_MEDIA_IMAGES)
} else {
hasPermission(Manifest.permission.READ_EXTERNAL_STORAGE)
}
val motionGranted =
if (Build.VERSION.SDK_INT >= 29) {
hasPermission(Manifest.permission.ACTIVITY_RECOGNITION)
} else {
true
}
val notificationsGranted =
if (Build.VERSION.SDK_INT >= 33) {
hasPermission(Manifest.permission.POST_NOTIFICATIONS)
} else {
true
}
return buildJsonObject {
put(
"permissions",
@@ -178,6 +196,41 @@ class DeviceHandler(
promptableWhenDenied = true,
),
)
put(
"notifications",
permissionStateJson(
granted = notificationsGranted,
promptableWhenDenied = true,
),
)
put(
"photos",
permissionStateJson(
granted = photosGranted,
promptableWhenDenied = true,
),
)
put(
"contacts",
permissionStateJson(
granted = hasPermission(Manifest.permission.READ_CONTACTS),
promptableWhenDenied = true,
),
)
put(
"calendar",
permissionStateJson(
granted = hasPermission(Manifest.permission.READ_CALENDAR),
promptableWhenDenied = true,
),
)
put(
"motion",
permissionStateJson(
granted = motionGranted,
promptableWhenDenied = Build.VERSION.SDK_INT >= 29,
),
)
// Screen capture on Android is interactive per-capture consent, not a sticky app permission.
put(
"screenCapture",

View File

@@ -1,20 +1,26 @@
package ai.openclaw.android.node
import ai.openclaw.android.protocol.OpenClawCalendarCommand
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawCapability
import ai.openclaw.android.protocol.OpenClawContactsCommand
import ai.openclaw.android.protocol.OpenClawDeviceCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawMotionCommand
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
import ai.openclaw.android.protocol.OpenClawPhotosCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
import ai.openclaw.android.protocol.OpenClawSystemCommand
data class NodeRuntimeFlags(
val cameraEnabled: Boolean,
val locationEnabled: Boolean,
val smsAvailable: Boolean,
val voiceWakeEnabled: Boolean,
val motionAvailable: Boolean,
val debugBuild: Boolean,
)
@@ -23,6 +29,7 @@ enum class InvokeCommandAvailability {
CameraEnabled,
LocationEnabled,
SmsAvailable,
MotionAvailable,
DebugBuild,
}
@@ -32,6 +39,7 @@ enum class NodeCapabilityAvailability {
LocationEnabled,
SmsAvailable,
VoiceWakeEnabled,
MotionAvailable,
}
data class NodeCapabilitySpec(
@@ -67,6 +75,13 @@ object InvokeCommandRegistry {
name = OpenClawCapability.Location.rawValue,
availability = NodeCapabilityAvailability.LocationEnabled,
),
NodeCapabilitySpec(name = OpenClawCapability.Photos.rawValue),
NodeCapabilitySpec(name = OpenClawCapability.Contacts.rawValue),
NodeCapabilitySpec(name = OpenClawCapability.Calendar.rawValue),
NodeCapabilitySpec(
name = OpenClawCapability.Motion.rawValue,
availability = NodeCapabilityAvailability.MotionAvailable,
),
)
val all: List<InvokeCommandSpec> =
@@ -107,6 +122,9 @@ object InvokeCommandRegistry {
name = OpenClawScreenCommand.Record.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawSystemCommand.Notify.rawValue,
),
InvokeCommandSpec(
name = OpenClawCameraCommand.List.rawValue,
requiresForeground = true,
@@ -144,6 +162,29 @@ object InvokeCommandRegistry {
InvokeCommandSpec(
name = OpenClawNotificationsCommand.Actions.rawValue,
),
InvokeCommandSpec(
name = OpenClawPhotosCommand.Latest.rawValue,
),
InvokeCommandSpec(
name = OpenClawContactsCommand.Search.rawValue,
),
InvokeCommandSpec(
name = OpenClawContactsCommand.Add.rawValue,
),
InvokeCommandSpec(
name = OpenClawCalendarCommand.Events.rawValue,
),
InvokeCommandSpec(
name = OpenClawCalendarCommand.Add.rawValue,
),
InvokeCommandSpec(
name = OpenClawMotionCommand.Activity.rawValue,
availability = InvokeCommandAvailability.MotionAvailable,
),
InvokeCommandSpec(
name = OpenClawMotionCommand.Pedometer.rawValue,
availability = InvokeCommandAvailability.MotionAvailable,
),
InvokeCommandSpec(
name = OpenClawSmsCommand.Send.rawValue,
availability = InvokeCommandAvailability.SmsAvailable,
@@ -172,6 +213,7 @@ object InvokeCommandRegistry {
NodeCapabilityAvailability.LocationEnabled -> flags.locationEnabled
NodeCapabilityAvailability.SmsAvailable -> flags.smsAvailable
NodeCapabilityAvailability.VoiceWakeEnabled -> flags.voiceWakeEnabled
NodeCapabilityAvailability.MotionAvailable -> flags.motionAvailable
}
}
.map { it.name }
@@ -185,6 +227,7 @@ object InvokeCommandRegistry {
InvokeCommandAvailability.CameraEnabled -> flags.cameraEnabled
InvokeCommandAvailability.LocationEnabled -> flags.locationEnabled
InvokeCommandAvailability.SmsAvailable -> flags.smsAvailable
InvokeCommandAvailability.MotionAvailable -> flags.motionAvailable
InvokeCommandAvailability.DebugBuild -> flags.debugBuild
}
}

View File

@@ -1,14 +1,19 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
import ai.openclaw.android.protocol.OpenClawCalendarCommand
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawContactsCommand
import ai.openclaw.android.protocol.OpenClawDeviceCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawMotionCommand
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
import ai.openclaw.android.protocol.OpenClawPhotosCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
import ai.openclaw.android.protocol.OpenClawSystemCommand
class InvokeDispatcher(
private val canvas: CanvasController,
@@ -16,6 +21,11 @@ class InvokeDispatcher(
private val locationHandler: LocationHandler,
private val deviceHandler: DeviceHandler,
private val notificationsHandler: NotificationsHandler,
private val systemHandler: SystemHandler,
private val photosHandler: PhotosHandler,
private val contactsHandler: ContactsHandler,
private val calendarHandler: CalendarHandler,
private val motionHandler: MotionHandler,
private val screenHandler: ScreenHandler,
private val smsHandler: SmsHandler,
private val a2uiHandler: A2UIHandler,
@@ -24,6 +34,7 @@ class InvokeDispatcher(
private val isForeground: () -> Boolean,
private val cameraEnabled: () -> Boolean,
private val locationEnabled: () -> Boolean,
private val motionAvailable: () -> Boolean,
private val smsAvailable: () -> Boolean,
private val debugBuild: () -> Boolean,
private val refreshNodeCanvasCapability: suspend () -> Boolean,
@@ -130,6 +141,24 @@ class InvokeDispatcher(
OpenClawNotificationsCommand.List.rawValue -> notificationsHandler.handleNotificationsList(paramsJson)
OpenClawNotificationsCommand.Actions.rawValue -> notificationsHandler.handleNotificationsActions(paramsJson)
// System command
OpenClawSystemCommand.Notify.rawValue -> systemHandler.handleSystemNotify(paramsJson)
// Photos command
OpenClawPhotosCommand.Latest.rawValue -> photosHandler.handlePhotosLatest(paramsJson)
// Contacts command
OpenClawContactsCommand.Search.rawValue -> contactsHandler.handleContactsSearch(paramsJson)
OpenClawContactsCommand.Add.rawValue -> contactsHandler.handleContactsAdd(paramsJson)
// Calendar command
OpenClawCalendarCommand.Events.rawValue -> calendarHandler.handleCalendarEvents(paramsJson)
OpenClawCalendarCommand.Add.rawValue -> calendarHandler.handleCalendarAdd(paramsJson)
// Motion command
OpenClawMotionCommand.Activity.rawValue -> motionHandler.handleMotionActivity(paramsJson)
OpenClawMotionCommand.Pedometer.rawValue -> motionHandler.handleMotionPedometer(paramsJson)
// Screen command
OpenClawScreenCommand.Record.rawValue -> screenHandler.handleScreenRecord(paramsJson)
@@ -212,6 +241,15 @@ class InvokeDispatcher(
message = "LOCATION_DISABLED: enable Location in Settings",
)
}
InvokeCommandAvailability.MotionAvailable ->
if (motionAvailable()) {
null
} else {
GatewaySession.InvokeResult.error(
code = "MOTION_UNAVAILABLE",
message = "MOTION_UNAVAILABLE: motion sensors not available",
)
}
InvokeCommandAvailability.SmsAvailable ->
if (smsAvailable()) {
null

View File

@@ -8,6 +8,10 @@ enum class OpenClawCapability(val rawValue: String) {
VoiceWake("voiceWake"),
Location("location"),
Device("device"),
Photos("photos"),
Contacts("contacts"),
Calendar("calendar"),
Motion("motion"),
}
enum class OpenClawCanvasCommand(val rawValue: String) {
@@ -93,3 +97,51 @@ enum class OpenClawNotificationsCommand(val rawValue: String) {
const val NamespacePrefix: String = "notifications."
}
}
enum class OpenClawSystemCommand(val rawValue: String) {
Notify("system.notify"),
;
companion object {
const val NamespacePrefix: String = "system."
}
}
enum class OpenClawPhotosCommand(val rawValue: String) {
Latest("photos.latest"),
;
companion object {
const val NamespacePrefix: String = "photos."
}
}
enum class OpenClawContactsCommand(val rawValue: String) {
Search("contacts.search"),
Add("contacts.add"),
;
companion object {
const val NamespacePrefix: String = "contacts."
}
}
enum class OpenClawCalendarCommand(val rawValue: String) {
Events("calendar.events"),
Add("calendar.add"),
;
companion object {
const val NamespacePrefix: String = "calendar."
}
}
enum class OpenClawMotionCommand(val rawValue: String) {
Activity("motion.activity"),
Pedometer("motion.pedometer"),
;
companion object {
const val NamespacePrefix: String = "motion."
}
}

View File

@@ -90,6 +90,11 @@ class DeviceHandlerTest {
"backgroundLocation",
"sms",
"notificationListener",
"notifications",
"photos",
"contacts",
"calendar",
"motion",
"screenCapture",
)
for (key in expected) {

View File

@@ -1,11 +1,16 @@
package ai.openclaw.android.node
import ai.openclaw.android.protocol.OpenClawCalendarCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawCapability
import ai.openclaw.android.protocol.OpenClawContactsCommand
import ai.openclaw.android.protocol.OpenClawDeviceCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawMotionCommand
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
import ai.openclaw.android.protocol.OpenClawPhotosCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
import ai.openclaw.android.protocol.OpenClawSystemCommand
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
@@ -20,6 +25,7 @@ class InvokeCommandRegistryTest {
locationEnabled = false,
smsAvailable = false,
voiceWakeEnabled = false,
motionAvailable = false,
debugBuild = false,
),
)
@@ -31,6 +37,10 @@ class InvokeCommandRegistryTest {
assertFalse(capabilities.contains(OpenClawCapability.Location.rawValue))
assertFalse(capabilities.contains(OpenClawCapability.Sms.rawValue))
assertFalse(capabilities.contains(OpenClawCapability.VoiceWake.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Photos.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Contacts.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Calendar.rawValue))
assertFalse(capabilities.contains(OpenClawCapability.Motion.rawValue))
}
@Test
@@ -42,6 +52,7 @@ class InvokeCommandRegistryTest {
locationEnabled = true,
smsAvailable = true,
voiceWakeEnabled = true,
motionAvailable = true,
debugBuild = false,
),
)
@@ -53,6 +64,10 @@ class InvokeCommandRegistryTest {
assertTrue(capabilities.contains(OpenClawCapability.Location.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Sms.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.VoiceWake.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Photos.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Contacts.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Calendar.rawValue))
assertTrue(capabilities.contains(OpenClawCapability.Motion.rawValue))
}
@Test
@@ -64,6 +79,7 @@ class InvokeCommandRegistryTest {
locationEnabled = false,
smsAvailable = false,
voiceWakeEnabled = false,
motionAvailable = false,
debugBuild = false,
),
)
@@ -78,6 +94,14 @@ class InvokeCommandRegistryTest {
assertTrue(commands.contains(OpenClawDeviceCommand.Health.rawValue))
assertTrue(commands.contains(OpenClawNotificationsCommand.List.rawValue))
assertTrue(commands.contains(OpenClawNotificationsCommand.Actions.rawValue))
assertTrue(commands.contains(OpenClawSystemCommand.Notify.rawValue))
assertTrue(commands.contains(OpenClawPhotosCommand.Latest.rawValue))
assertTrue(commands.contains(OpenClawContactsCommand.Search.rawValue))
assertTrue(commands.contains(OpenClawContactsCommand.Add.rawValue))
assertTrue(commands.contains(OpenClawCalendarCommand.Events.rawValue))
assertTrue(commands.contains(OpenClawCalendarCommand.Add.rawValue))
assertFalse(commands.contains(OpenClawMotionCommand.Activity.rawValue))
assertFalse(commands.contains(OpenClawMotionCommand.Pedometer.rawValue))
assertFalse(commands.contains(OpenClawSmsCommand.Send.rawValue))
assertFalse(commands.contains("debug.logs"))
assertFalse(commands.contains("debug.ed25519"))
@@ -93,6 +117,7 @@ class InvokeCommandRegistryTest {
locationEnabled = true,
smsAvailable = true,
voiceWakeEnabled = false,
motionAvailable = true,
debugBuild = true,
),
)
@@ -107,6 +132,14 @@ class InvokeCommandRegistryTest {
assertTrue(commands.contains(OpenClawDeviceCommand.Health.rawValue))
assertTrue(commands.contains(OpenClawNotificationsCommand.List.rawValue))
assertTrue(commands.contains(OpenClawNotificationsCommand.Actions.rawValue))
assertTrue(commands.contains(OpenClawSystemCommand.Notify.rawValue))
assertTrue(commands.contains(OpenClawPhotosCommand.Latest.rawValue))
assertTrue(commands.contains(OpenClawContactsCommand.Search.rawValue))
assertTrue(commands.contains(OpenClawContactsCommand.Add.rawValue))
assertTrue(commands.contains(OpenClawCalendarCommand.Events.rawValue))
assertTrue(commands.contains(OpenClawCalendarCommand.Add.rawValue))
assertTrue(commands.contains(OpenClawMotionCommand.Activity.rawValue))
assertTrue(commands.contains(OpenClawMotionCommand.Pedometer.rawValue))
assertTrue(commands.contains(OpenClawSmsCommand.Send.rawValue))
assertTrue(commands.contains("debug.logs"))
assertTrue(commands.contains("debug.ed25519"))

View File

@@ -29,6 +29,10 @@ class OpenClawProtocolConstantsTest {
assertEquals("location", OpenClawCapability.Location.rawValue)
assertEquals("sms", OpenClawCapability.Sms.rawValue)
assertEquals("device", OpenClawCapability.Device.rawValue)
assertEquals("photos", OpenClawCapability.Photos.rawValue)
assertEquals("contacts", OpenClawCapability.Contacts.rawValue)
assertEquals("calendar", OpenClawCapability.Calendar.rawValue)
assertEquals("motion", OpenClawCapability.Motion.rawValue)
}
@Test
@@ -56,4 +60,32 @@ class OpenClawProtocolConstantsTest {
assertEquals("device.permissions", OpenClawDeviceCommand.Permissions.rawValue)
assertEquals("device.health", OpenClawDeviceCommand.Health.rawValue)
}
@Test
fun systemCommandsUseStableStrings() {
assertEquals("system.notify", OpenClawSystemCommand.Notify.rawValue)
}
@Test
fun photosCommandsUseStableStrings() {
assertEquals("photos.latest", OpenClawPhotosCommand.Latest.rawValue)
}
@Test
fun contactsCommandsUseStableStrings() {
assertEquals("contacts.search", OpenClawContactsCommand.Search.rawValue)
assertEquals("contacts.add", OpenClawContactsCommand.Add.rawValue)
}
@Test
fun calendarCommandsUseStableStrings() {
assertEquals("calendar.events", OpenClawCalendarCommand.Events.rawValue)
assertEquals("calendar.add", OpenClawCalendarCommand.Add.rawValue)
}
@Test
fun motionCommandsUseStableStrings() {
assertEquals("motion.activity", OpenClawMotionCommand.Activity.rawValue)
assertEquals("motion.pedometer", OpenClawMotionCommand.Pedometer.rawValue)
}
}