diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts index 15d4d15249d..6466474af2b 100644 --- a/apps/android/app/build.gradle.kts +++ b/apps/android/app/build.gradle.kts @@ -126,6 +126,11 @@ dependencies { implementation("androidx.exifinterface:exifinterface:1.4.2") implementation("com.squareup.okhttp3:okhttp:5.3.2") implementation("org.bouncycastle:bcprov-jdk18on:1.83") + implementation("org.commonmark:commonmark:0.27.1") + implementation("org.commonmark:commonmark-ext-autolink:0.27.1") + implementation("org.commonmark:commonmark-ext-gfm-strikethrough:0.27.1") + implementation("org.commonmark:commonmark-ext-gfm-tables:0.27.1") + implementation("org.commonmark:commonmark-ext-task-list-items:0.27.1") // CameraX (for node.invoke camera.* parity) implementation("androidx.camera:camera-core:1.5.2") diff --git a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt index e5d4def3fd9..e121212529a 100644 --- a/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt +++ b/apps/android/app/src/main/java/ai/openclaw/android/ui/chat/ChatMarkdown.kt @@ -3,10 +3,20 @@ package ai.openclaw.android.ui.chat import android.graphics.BitmapFactory import android.util.Base64 import androidx.compose.foundation.Image +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.IntrinsicSize +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.text.selection.SelectionContainer import androidx.compose.material3.Text import androidx.compose.runtime.Composable @@ -15,18 +25,23 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.text.AnnotatedString import androidx.compose.ui.text.SpanStyle +import androidx.compose.ui.text.TextStyle import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.font.FontStyle import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextDecoration import androidx.compose.ui.text.withStyle import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import ai.openclaw.android.ui.mobileAccent import ai.openclaw.android.ui.mobileCallout import ai.openclaw.android.ui.mobileCaption1 import ai.openclaw.android.ui.mobileCodeBg @@ -34,159 +49,510 @@ import ai.openclaw.android.ui.mobileCodeText import ai.openclaw.android.ui.mobileTextSecondary import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.commonmark.Extension +import org.commonmark.ext.autolink.AutolinkExtension +import org.commonmark.ext.gfm.strikethrough.Strikethrough +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension +import org.commonmark.ext.gfm.tables.TableBlock +import org.commonmark.ext.gfm.tables.TableBody +import org.commonmark.ext.gfm.tables.TableCell +import org.commonmark.ext.gfm.tables.TableHead +import org.commonmark.ext.gfm.tables.TableRow +import org.commonmark.ext.gfm.tables.TablesExtension +import org.commonmark.ext.task.list.items.TaskListItemMarker +import org.commonmark.ext.task.list.items.TaskListItemsExtension +import org.commonmark.node.BlockQuote +import org.commonmark.node.BulletList +import org.commonmark.node.Code +import org.commonmark.node.Document +import org.commonmark.node.Emphasis +import org.commonmark.node.FencedCodeBlock +import org.commonmark.node.Heading +import org.commonmark.node.HardLineBreak +import org.commonmark.node.HtmlBlock +import org.commonmark.node.HtmlInline +import org.commonmark.node.Image as MarkdownImage +import org.commonmark.node.IndentedCodeBlock +import org.commonmark.node.Link +import org.commonmark.node.ListItem +import org.commonmark.node.Node +import org.commonmark.node.OrderedList +import org.commonmark.node.Paragraph +import org.commonmark.node.SoftLineBreak +import org.commonmark.node.StrongEmphasis +import org.commonmark.node.Text as MarkdownTextNode +import org.commonmark.node.ThematicBreak +import org.commonmark.parser.Parser + +private const val LIST_INDENT_DP = 14 +private val dataImageRegex = Regex("^data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)$") + +private val markdownParser: Parser by lazy { + val extensions: List = + listOf( + AutolinkExtension.create(), + StrikethroughExtension.create(), + TablesExtension.create(), + TaskListItemsExtension.create(), + ) + Parser.builder() + .extensions(extensions) + .build() +} @Composable fun ChatMarkdown(text: String, textColor: Color) { - val blocks = remember(text) { splitMarkdown(text) } - val inlineCodeBg = mobileCodeBg - val inlineCodeColor = mobileCodeText + val document = remember(text) { markdownParser.parse(text) as Document } + val inlineStyles = InlineStyles(inlineCodeBg = mobileCodeBg, inlineCodeColor = mobileCodeText) Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { - for (b in blocks) { - when (b) { - is ChatMarkdownBlock.Text -> { - val trimmed = b.text.trimEnd() - if (trimmed.isEmpty()) continue + RenderMarkdownBlocks( + start = document.firstChild, + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = 0, + ) + } +} + +@Composable +private fun RenderMarkdownBlocks( + start: Node?, + textColor: Color, + inlineStyles: InlineStyles, + listDepth: Int, +) { + var node = start + while (node != null) { + val current = node + when (current) { + is Paragraph -> { + RenderParagraph(current, textColor = textColor, inlineStyles = inlineStyles) + } + is Heading -> { + val headingText = remember(current) { buildInlineMarkdown(current.firstChild, inlineStyles) } + Text( + text = headingText, + style = headingStyle(current.level), + color = textColor, + ) + } + is FencedCodeBlock -> { + SelectionContainer(modifier = Modifier.fillMaxWidth()) { + ChatCodeBlock(code = current.literal.orEmpty(), language = current.info?.trim()?.ifEmpty { null }) + } + } + is IndentedCodeBlock -> { + SelectionContainer(modifier = Modifier.fillMaxWidth()) { + ChatCodeBlock(code = current.literal.orEmpty(), language = null) + } + } + is BlockQuote -> { + Row( + modifier = Modifier + .fillMaxWidth() + .height(IntrinsicSize.Min) + .padding(vertical = 2.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.Top, + ) { + Box( + modifier = Modifier + .width(2.dp) + .fillMaxHeight() + .background(mobileTextSecondary.copy(alpha = 0.35f)), + ) + Column( + modifier = Modifier.weight(1f), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + RenderMarkdownBlocks( + start = current.firstChild, + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth, + ) + } + } + } + is BulletList -> { + RenderBulletList( + list = current, + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth, + ) + } + is OrderedList -> { + RenderOrderedList( + list = current, + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth, + ) + } + is TableBlock -> { + RenderTableBlock( + table = current, + textColor = textColor, + inlineStyles = inlineStyles, + ) + } + is ThematicBreak -> { + Box( + modifier = Modifier + .fillMaxWidth() + .height(1.dp) + .background(mobileTextSecondary.copy(alpha = 0.25f)), + ) + } + is HtmlBlock -> { + val literal = current.literal.orEmpty().trim() + if (literal.isNotEmpty()) { Text( - text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor), - style = mobileCallout, + text = literal, + style = mobileCallout.copy(fontFamily = FontFamily.Monospace), color = textColor, ) } - is ChatMarkdownBlock.Code -> { - SelectionContainer(modifier = Modifier.fillMaxWidth()) { - ChatCodeBlock(code = b.code, language = b.language) - } - } - is ChatMarkdownBlock.InlineImage -> { - InlineBase64Image(base64 = b.base64, mimeType = b.mimeType) + } + } + node = current.next + } +} + +@Composable +private fun RenderParagraph( + paragraph: Paragraph, + textColor: Color, + inlineStyles: InlineStyles, +) { + val standaloneImage = remember(paragraph) { standaloneDataImage(paragraph) } + if (standaloneImage != null) { + InlineBase64Image(base64 = standaloneImage.base64, mimeType = standaloneImage.mimeType) + return + } + + val annotated = remember(paragraph) { buildInlineMarkdown(paragraph.firstChild, inlineStyles) } + if (annotated.text.trimEnd().isEmpty()) { + return + } + + Text( + text = annotated, + style = mobileCallout, + color = textColor, + ) +} + +@Composable +private fun RenderBulletList( + list: BulletList, + textColor: Color, + inlineStyles: InlineStyles, + listDepth: Int, +) { + Column( + modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + var item = list.firstChild + while (item != null) { + if (item is ListItem) { + RenderListItem( + item = item, + markerText = "•", + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth, + ) + } + item = item.next + } + } +} + +@Composable +private fun RenderOrderedList( + list: OrderedList, + textColor: Color, + inlineStyles: InlineStyles, + listDepth: Int, +) { + Column( + modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + var index = list.markerStartNumber ?: 1 + var item = list.firstChild + while (item != null) { + if (item is ListItem) { + RenderListItem( + item = item, + markerText = "$index.", + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth, + ) + index += 1 + } + item = item.next + } + } +} + +@Composable +private fun RenderListItem( + item: ListItem, + markerText: String, + textColor: Color, + inlineStyles: InlineStyles, + listDepth: Int, +) { + var contentStart = item.firstChild + var marker = markerText + val task = contentStart as? TaskListItemMarker + if (task != null) { + marker = if (task.isChecked) "☑" else "☐" + contentStart = task.next + } + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.Top, + ) { + Text( + text = marker, + style = mobileCallout.copy(fontWeight = FontWeight.SemiBold), + color = textColor, + modifier = Modifier.width(24.dp), + ) + + Column( + modifier = Modifier.weight(1f), + verticalArrangement = Arrangement.spacedBy(4.dp), + ) { + RenderMarkdownBlocks( + start = contentStart, + textColor = textColor, + inlineStyles = inlineStyles, + listDepth = listDepth + 1, + ) + } + } +} + +@Composable +private fun RenderTableBlock( + table: TableBlock, + textColor: Color, + inlineStyles: InlineStyles, +) { + val rows = remember(table) { buildTableRows(table, inlineStyles) } + if (rows.isEmpty()) return + + val maxCols = rows.maxOf { row -> row.cells.size }.coerceAtLeast(1) + val scrollState = rememberScrollState() + + Column( + modifier = Modifier + .fillMaxWidth() + .horizontalScroll(scrollState) + .border(1.dp, mobileTextSecondary.copy(alpha = 0.25f)), + ) { + for (row in rows) { + Row( + modifier = Modifier.fillMaxWidth(), + ) { + for (index in 0 until maxCols) { + val cell = row.cells.getOrNull(index) ?: AnnotatedString("") + Text( + text = cell, + style = if (row.isHeader) mobileCaption1.copy(fontWeight = FontWeight.SemiBold) else mobileCallout, + color = textColor, + modifier = Modifier + .border(1.dp, mobileTextSecondary.copy(alpha = 0.22f)) + .padding(horizontal = 8.dp, vertical = 6.dp) + .width(160.dp), + ) } } } } } -private sealed interface ChatMarkdownBlock { - data class Text(val text: String) : ChatMarkdownBlock - data class Code(val code: String, val language: String?) : ChatMarkdownBlock - data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock -} - -private fun splitMarkdown(raw: String): List { - if (raw.isEmpty()) return emptyList() - - val out = ArrayList() - var idx = 0 - while (idx < raw.length) { - val fenceStart = raw.indexOf("```", startIndex = idx) - if (fenceStart < 0) { - out.addAll(splitInlineImages(raw.substring(idx))) - break +private fun buildTableRows(table: TableBlock, inlineStyles: InlineStyles): List { + val rows = mutableListOf() + var child = table.firstChild + while (child != null) { + when (child) { + is TableHead -> rows.addAll(readTableSection(child, isHeader = true, inlineStyles = inlineStyles)) + is TableBody -> rows.addAll(readTableSection(child, isHeader = false, inlineStyles = inlineStyles)) + is TableRow -> rows.add(readTableRow(child, isHeader = false, inlineStyles = inlineStyles)) } - - if (fenceStart > idx) { - out.addAll(splitInlineImages(raw.substring(idx, fenceStart))) - } - - val langLineStart = fenceStart + 3 - val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it } - val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null } - - val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd - val fenceEnd = raw.indexOf("```", startIndex = codeStart) - if (fenceEnd < 0) { - out.addAll(splitInlineImages(raw.substring(fenceStart))) - break - } - val code = raw.substring(codeStart, fenceEnd) - out.add(ChatMarkdownBlock.Code(code = code, language = language)) - - idx = fenceEnd + 3 + child = child.next } - - return out + return rows } -private fun splitInlineImages(text: String): List { - if (text.isEmpty()) return emptyList() - val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)") - val out = ArrayList() - - var idx = 0 - while (idx < text.length) { - val m = regex.find(text, startIndex = idx) ?: break - val start = m.range.first - val end = m.range.last + 1 - if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start))) - - val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png") - val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty() - if (b64.isNotEmpty()) { - out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64)) +private fun readTableSection(section: Node, isHeader: Boolean, inlineStyles: InlineStyles): List { + val rows = mutableListOf() + var row = section.firstChild + while (row != null) { + if (row is TableRow) { + rows.add(readTableRow(row, isHeader = isHeader, inlineStyles = inlineStyles)) } - idx = end + row = row.next } - - if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx))) - return out + return rows } -private fun parseInlineMarkdown( - text: String, - inlineCodeBg: androidx.compose.ui.graphics.Color, - inlineCodeColor: androidx.compose.ui.graphics.Color, -): AnnotatedString { - if (text.isEmpty()) return AnnotatedString("") +private fun readTableRow(row: TableRow, isHeader: Boolean, inlineStyles: InlineStyles): TableRenderRow { + val cells = mutableListOf() + var cellNode = row.firstChild + while (cellNode != null) { + if (cellNode is TableCell) { + cells.add(buildInlineMarkdown(cellNode.firstChild, inlineStyles)) + } + cellNode = cellNode.next + } + return TableRenderRow(isHeader = isHeader, cells = cells) +} - val out = buildAnnotatedString { - var i = 0 - while (i < text.length) { - if (text.startsWith("**", startIndex = i)) { - val end = text.indexOf("**", startIndex = i + 2) - if (end > i + 2) { - withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { - append(text.substring(i + 2, end)) - } - i = end + 2 - continue +private fun buildInlineMarkdown(start: Node?, inlineStyles: InlineStyles): AnnotatedString { + return buildAnnotatedString { + appendInlineNode( + node = start, + inlineCodeBg = inlineStyles.inlineCodeBg, + inlineCodeColor = inlineStyles.inlineCodeColor, + ) + } +} + +private fun AnnotatedString.Builder.appendInlineNode( + node: Node?, + inlineCodeBg: Color, + inlineCodeColor: Color, +) { + var current = node + while (current != null) { + when (current) { + is MarkdownTextNode -> append(current.literal) + is SoftLineBreak -> append('\n') + is HardLineBreak -> append('\n') + is Code -> { + withStyle( + SpanStyle( + fontFamily = FontFamily.Monospace, + background = inlineCodeBg, + color = inlineCodeColor, + ), + ) { + append(current.literal) } } - - if (text[i] == '`') { - val end = text.indexOf('`', startIndex = i + 1) - if (end > i + 1) { - withStyle( - SpanStyle( - fontFamily = FontFamily.Monospace, - background = inlineCodeBg, - color = inlineCodeColor, - ), - ) { - append(text.substring(i + 1, end)) - } - i = end + 1 - continue + is Emphasis -> { + withStyle(SpanStyle(fontStyle = FontStyle.Italic)) { + appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor) } } - - if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) { - val end = text.indexOf('*', startIndex = i + 1) - if (end > i + 1) { - withStyle(SpanStyle(fontStyle = FontStyle.Italic)) { - append(text.substring(i + 1, end)) - } - i = end + 1 - continue + is StrongEmphasis -> { + withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { + appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor) } } - - append(text[i]) - i += 1 + is Strikethrough -> { + withStyle(SpanStyle(textDecoration = TextDecoration.LineThrough)) { + appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor) + } + } + is Link -> { + withStyle( + SpanStyle( + color = mobileAccent, + textDecoration = TextDecoration.Underline, + ), + ) { + appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor) + } + } + is MarkdownImage -> { + val alt = buildPlainText(current.firstChild) + if (alt.isNotBlank()) { + append(alt) + } else { + append("image") + } + } + is HtmlInline -> { + if (!current.literal.isNullOrBlank()) { + append(current.literal) + } + } + else -> { + appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor) + } } + current = current.next } - return out } +private fun buildPlainText(start: Node?): String { + val sb = StringBuilder() + var node = start + while (node != null) { + when (node) { + is MarkdownTextNode -> sb.append(node.literal) + is SoftLineBreak, is HardLineBreak -> sb.append('\n') + else -> sb.append(buildPlainText(node.firstChild)) + } + node = node.next + } + return sb.toString() +} + +private fun standaloneDataImage(paragraph: Paragraph): ParsedDataImage? { + val only = paragraph.firstChild as? MarkdownImage ?: return null + if (only.next != null) return null + return parseDataImageDestination(only.destination) +} + +private fun parseDataImageDestination(destination: String?): ParsedDataImage? { + val raw = destination?.trim().orEmpty() + if (raw.isEmpty()) return null + val match = dataImageRegex.matchEntire(raw) ?: return null + val subtype = match.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png" + val base64 = match.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty() + if (base64.isEmpty()) return null + return ParsedDataImage(mimeType = "image/$subtype", base64 = base64) +} + +private fun headingStyle(level: Int): TextStyle { + return when (level.coerceIn(1, 6)) { + 1 -> mobileCallout.copy(fontSize = 22.sp, lineHeight = 28.sp, fontWeight = FontWeight.Bold) + 2 -> mobileCallout.copy(fontSize = 20.sp, lineHeight = 26.sp, fontWeight = FontWeight.Bold) + 3 -> mobileCallout.copy(fontSize = 18.sp, lineHeight = 24.sp, fontWeight = FontWeight.SemiBold) + 4 -> mobileCallout.copy(fontSize = 16.sp, lineHeight = 22.sp, fontWeight = FontWeight.SemiBold) + else -> mobileCallout.copy(fontWeight = FontWeight.SemiBold) + } +} + +private data class InlineStyles( + val inlineCodeBg: Color, + val inlineCodeColor: Color, +) + +private data class TableRenderRow( + val isHeader: Boolean, + val cells: List, +) + +private data class ParsedDataImage( + val mimeType: String, + val base64: String, +) + @Composable private fun InlineBase64Image(base64: String, mimeType: String?) { var image by remember(base64) { mutableStateOf(null) }