Open upstream PR: voice notes and audio in chat

apps/android/WhatsNew.md

@@ -17,6 +17,8 @@ else a generic "Bug fixes and improvements" line. Delete old sections.
 - bot 要你選擇時，聊天中會出現可點選的選項卡。選項會送回正在進行的工作，已回答或過期的卡片不會重複送出。
 - 更新 App 或電腦上的 Codync 之後，手機上保存的聊天記錄會保留，不再清空重新下載。
 - 聊天裡的 routine 紀錄改為置中顯示。
+- bot 可以傳語音訊息了：語音會在那則訊息的泡泡最上方直接播放，文字接在下面。
+- bot 傳來的音訊檔（MP3、M4A、WAV、AAC，20 MB 以內）可以直接在聊天裡播放：點播放鍵，進度列會顯示播放到哪，拖動或點一下進度列就能跳到別的位置。
 
 ### en-US
 - New "Move active bots to the top": turn it on in a computer's settings and a bot with a new message (yours, its reply, or a routine run) moves to the top of the list.
@@ -27,6 +29,8 @@ else a generic "Bug fixes and improvements" line. Delete old sections.
 - When your bot needs a choice, tap an option in its chat card. Your answer continues the current work; answered or expired cards cannot send another choice.
 - After an update of the app or of Codync on your computer, the conversations saved on the phone are kept, not emptied and downloaded again.
 - A routine's line in the chat is centered.
+- Bots can now send voice notes: the note plays at the top of its message's bubble, with the text under it.
+- Sound files a bot sends (MP3, M4A, WAV or AAC, up to 20 MB) play right in the chat: tap play, and drag or tap the progress bar to jump to another point.
 
 ## 2.44.1
 ### zh-Hant

apps/android/app/src/main/kotlin/com/codync/android/AnalyticsEvent.kt

@@ -138,6 +138,11 @@ internal sealed class AnalyticsEvent(val name: String) {
     data class DevServerAction(val use: DevServerUse) : AnalyticsEvent("dev_server_action") {
         override val properties get() = mapOf("action" to use.json)
     }
+    /** A voice note started from its beginning, or which step broke ([failure]); never which note. */
+    data class VoiceNotePlayed(val failure: VoiceNoteFailure?) : AnalyticsEvent("voice_note_played") {
+        override val properties get() =
+            mapOf("outcome" to JsonPrimitive(if (failure == null) "played" else "failed")) + listOfNotNull(failure?.let { "error" to it.json })
+    }
     /** Find in a conversation opened (iOS `conversationFindOpened`). */
     data object ConversationFindOpened : AnalyticsEvent("conversation_find_opened")
     /** A search in the find bar settled: whether it found anything, never the text (iOS `conversationFindSearched`). */
@@ -292,6 +297,7 @@ internal enum class OutboundLink(override val wire: String) : CatalogValue {
 }
 
 internal enum class DevServerUse(override val wire: String) : CatalogValue { Open("open"), Copy("copy") }
+internal enum class VoiceNoteFailure(override val wire: String) : CatalogValue { Load("load"), Playback("playback") }
 
 internal enum class VoiceEngineKind(override val wire: String) : CatalogValue {
     OnDevice("on_device"), OpenAI("openai"), Gemini("gemini"),

apps/android/app/src/main/kotlin/com/codync/android/AppStoreDrafts.kt

@@ -113,8 +113,8 @@ fun AppStore.removeFile(file: OutgoingFile) = action("remove:" + file.path) { wi
 suspend fun AppStore.attachment(botId: String, file: Attachment): File =
     cachedFile(file.id, file.size) { files, destination -> files.download(botId, file, destination) }
 
-/** A picture a bot shared (iOS `sharedFileData`): checked against its SHA-256 on its way into the same cache. */
-suspend fun AppStore.sharedImage(entryId: String, file: SharedFile): File = cachedFile(file.id, file.size) { files, destination ->
+/** A picture or sound a bot shared (iOS `sharedFileData`): checked against its SHA-256 on its way into the same cache. */
+suspend fun AppStore.sharedFile(entryId: String, file: SharedFile): File = cachedFile(file.id, file.size) { files, destination ->
     val folder = withContext(Dispatchers.IO) { attachments.shareFolder() }
     val saved = files.downloadShared(entryId, file, folder)
     withContext(Dispatchers.IO) { try { saved.copyTo(destination, overwrite = true) } finally { folder.deleteRecursively() } }

apps/android/app/src/main/kotlin/com/codync/android/AppStoreFiles.kt

@@ -52,6 +52,10 @@ internal fun AppStore.saveFile(entryId: String, botId: String, file: SharedFile,
     }
 }
 
+/** A file's one download control (iOS `FileDownloads.toggle`): downloads it, or cancels the download under way. */
+fun AppStore.toggleFile(entryId: String, botId: String, file: SharedFile) =
+    if (state.value.fileDownloads[file.id] is FileDownloadState.Downloading) cancelFile(file.id) else saveFile(entryId, botId, file)
+
 fun AppStore.cancelFile(fileId: String) {
     actions[downloadKey(fileId)]?.cancel()
     setDownload(fileId, null)

apps/android/app/src/main/kotlin/com/codync/android/AttachmentList.kt

@@ -66,7 +66,7 @@ internal class ImageSource(val id: String, val name: String, val load: suspend (
 
 internal fun Attachment.imageSource(botId: String, store: AppStore) = ImageSource(id, name) { store.attachment(botId, this) }
 
-internal fun SharedFile.imageSource(entryId: String, store: AppStore) = ImageSource(id, name) { store.sharedImage(entryId, this) }
+internal fun SharedFile.imageSource(entryId: String, store: AppStore) = ImageSource(id, name) { store.sharedFile(entryId, this) }
 
 /** A sent image, fetched from the computer once and then read from the cache; a placeholder until then. */
 @Composable internal fun AttachmentImage(file: Attachment, botId: String, maxPixels: Int, state: AppState, store: AppStore, modifier: Modifier,
@@ -106,12 +106,41 @@ internal fun SharedFile.imageSource(entryId: String, store: AppStore) = ImageSou
 
 /** The symbol for a file by its extension. */
 internal object AttachmentIcon {
-    fun symbol(name: String): String = when (name.substringAfterLast('.', "").lowercase()) {
-        "png", "jpg", "jpeg", "heic", "gif", "webp", "tiff", "bmp" -> "photo"
-        "pdf" -> "doc.richtext"
-        "zip", "gz", "tar", "7z" -> "doc.zipper"
-        "mov", "mp4", "m4v" -> "film"
-        "mp3", "m4a", "wav", "aac" -> "waveform"
-        else -> "doc"
+    fun symbol(name: String): String {
+        val ext = name.substringAfterLast('.', "").lowercase()
+        return when {
+            isImageName(name) -> "photo"
+            ext == "pdf" -> "doc.richtext"
+            ext in listOf("zip", "gz", "tar", "7z") -> "doc.zipper"
+            ext in listOf("mov", "mp4", "m4v") -> "film"
+            isAudioName(name) -> "waveform"
+            else -> "doc"
+        }
+    }
+}
+
+/**
+ * A file size the way `ByteCountFormatter(.file)` writes it: powers of 1000, no decimals for KB, one for MB, two for GB,
+ * the trailing `.0` dropped, and a size that rounds up to 1000 of a unit shown in the next one.
+ */
+fun formatBytes(size: Long): String {
+    if (size == 0L) return "Zero KB"
+    if (size < 1000) return if (size == 1L) "1 byte" else "$size bytes"
+    val units = listOf("KB", "MB", "GB", "TB")
+    var value = size / 1000.0
+    var unit = 0
+    while (unit < units.lastIndex && round(value, if (unit == 0) 0 else unit.coerceAtMost(2)) >= 1000) {
+        value /= 1000
+        unit++
     }
+    val shown = round(value, if (unit == 0) 0 else unit.coerceAtMost(2))
+    return "${trim(shown)} ${units[unit]}"
 }
+
+private fun round(value: Double, decimals: Int): Double {
+    val scale = Math.pow(10.0, decimals.toDouble())
+    return Math.floor(value * scale + 0.5) / scale
+}
+
+private fun trim(value: Double): String =
+    if (value == value.toLong().toDouble()) value.toLong().toString() else value.toString()

apps/android/app/src/main/kotlin/com/codync/android/ChatRows.kt

@@ -32,6 +32,7 @@ import androidx.compose.runtime.key
 import androidx.compose.runtime.mutableLongStateOf
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
 import androidx.compose.runtime.setValue
 import androidx.compose.runtime.withFrameNanos
 import androidx.compose.ui.Alignment
@@ -41,9 +42,12 @@ import androidx.compose.ui.draw.rotate
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.platform.LocalFocusManager
+import androidx.compose.ui.semantics.CustomAccessibilityAction
 import androidx.compose.ui.semantics.Role
 import androidx.compose.ui.semantics.clearAndSetSemantics
 import androidx.compose.ui.semantics.contentDescription
+import androidx.compose.ui.semantics.customActions
+import androidx.compose.ui.semantics.onClick
 import androidx.compose.ui.semantics.semantics
 import androidx.compose.ui.semantics.stateDescription
 import androidx.compose.ui.text.style.TextAlign
@@ -54,6 +58,7 @@ import com.codync.android.design.*
 import com.kyant.shapes.Capsule
 import com.kyant.shapes.RoundedRectangle
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
 import kotlinx.serialization.json.JsonArray
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.booleanOrNull
@@ -248,12 +253,28 @@ internal val Entry.reactions: List<String> get() = (data["reactions"] as? JsonAr
     val focus = LocalFocusManager.current
     val files = entry.files
     val artifact = entry.artifact
+    val voiceNote = entry.voiceNote
     val menu = MessageMenu.of(entry, store, reply, openTrace)
     val textMenu = { menu.items(listOf(textCopyItem(context, entry.text))) }
     Column(Modifier.fillMaxWidth().padding(end = 40.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
-        // An artifact card or a message with files is that: the text is only the computer's fallback.
+        // An artifact card or a message with files is that: the text is only the computer's fallback. A voice note
+        // instead plays inside its bubble, over the message it captions.
         if (artifact != null) ArtifactCardView(artifact, entry.botId, state, store, Modifier.contextActions(reactions = menu.reactions,
             onClick = { focus.clearFocus(); openArtifact(OpenArtifact(entry.botId, artifact.id, artifact.title, entry.id)) }, items = textMenu))
+        else if (voiceNote != null) {
+            val player = rememberSharedAudio(entry, voiceNote, store)
+            val download = state.fileDownloads[voiceNote.id]
+            val busy = download is FileDownloadState.Downloading
+            val save = { store.toggleFile(entry.id, entry.botId, voiceNote) }
+            // Its Download follows Copy, as on desktop and iPhone.
+            val downloadItem = MenuItem(if (busy) "Cancel download" else "Download", icon = if (busy) "xmark" else "arrow.down.to.line", action = save)
+            Box(Modifier.clip(RoundedRectangle(18.dp)).background(Palette.bubbleAgent)
+                .contextActions(reactions = menu.reactions, onClick = { focus.clearFocus() },
+                    preview = { VoiceNoteMessage(entry.text, voiceNote, player, download, save, selectable = true) },
+                    items = { menu.items(listOf(textCopyItem(context, entry.text), downloadItem)) })) {
+                VoiceNoteMessage(entry.text, voiceNote, player, download, save, selectable = false)
+            }
+        }
         // Each card holds its own menu: a picture's copies the picture.
         else if (files.isNotEmpty()) Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
             for (file in files) FileDownloadCard(entry, file, menu, state, store)
@@ -264,6 +285,38 @@ internal val Entry.reactions: List<String> get() = (data["reactions"] as? JsonAr
     }
 }
 
+/**
+ * A reply carrying a voice note, as a messaging app shows one with a caption: the player at the top of its bubble,
+ * the message under it. The player row is one element — "Voice note, <caption>" — with the card's play and download
+ * actions; download has no button here, the bubble's menu and those actions carry it.
+ */
+@Composable private fun VoiceNoteMessage(text: String, file: SharedFile, audio: SharedAudio, download: FileDownloadState?,
+    save: () -> Unit, selectable: Boolean) {
+    val scope = rememberCoroutineScope()
+    val translate = translator()
+    val busy = download is FileDownloadState.Downloading
+    val caption = detail(file, download, audio, translate)
+    val play = { scope.launch { audio.toggle() }; Unit }
+    Column(Modifier.background(Palette.bubbleAgent).padding(horizontal = 16.dp, vertical = 10.dp),
+        verticalArrangement = Arrangement.spacedBy(8.dp)) {
+        Row(Modifier.fillMaxWidth().clearAndSetSemantics {
+            contentDescription = "${translate("Voice note")}, $caption"
+            audioSeek(audio)
+            customActions = listOf(
+                CustomAccessibilityAction(translate(if (audio.phase == AudioPhase.Playing) "Pause" else "Play")) { play(); true },
+                CustomAccessibilityAction(translate(if (busy) "Cancel download" else "Download")) { save(); true })
+            onClick(label = translate(if (audio.phase == AudioPhase.Playing) "Pause" else "Play")) { play(); true }
+        }, verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp)) {
+            SharedAudioButton(audio, play)
+            if (audio.duration > 0.0) AudioProgress(audio, Modifier.weight(1f))
+            // Steady digits, so the bar's end doesn't move as the clock runs.
+            Text(caption, style = Type.caption2.copy(fontFeatureSettings = "tnum"),
+                color = if (audio.phase == AudioPhase.Failed) Palette.danger else Palette.secondary)
+        }
+        if (text.isNotEmpty()) MarkdownText(text, selectable)
+    }
+}
+
 /**
  * A reply. Holding it opens its menu over a lifted copy, as on iOS, so only that copy is `selectable`:
  * text selection takes the same long press.