feat: singleton audio, transcript reflection, voice persistence

Matthias Nott

2026-03-07 0e888d62af1434fef231e11a5c307a5b48a8deb1

feat: singleton audio, transcript reflection, voice persistence

- audio.ts: true singleton player — only one audio at a time, with
  URI-based tracking so each bubble knows its play state
- audio.ts: playSingle() for manual playback, playAudio() for
  autoplay queue, explicit pause before remove to stop native audio
- MessageBubble: show transcript text below voice player, track
  playing state via singleton URI
- VoiceButton: pass recording duration from recorder.currentTime
- ChatContext: handle incoming 'transcript' type to update voice
  bubbles with transcribed text, send messageId with voice messages
- ChatContext: voice messages persist transcript text, empty chunks
  are filtered on reload, transcribed voices become text bubbles
- ConnectionContext: pass messageId with voice messages to gateway
- wol.ts: add 5s timeout with settled guard to prevent hanging
- types: add WsIncomingTranscript, messageId on WsVoiceMessage

 app/chat.tsx
..
..
@@ -11,7 +11,7 @@
11
11
 import { ImageCaptionModal } from "../components/chat/ImageCaptionModal";
12
12
 import { StatusDot } from "../components/ui/StatusDot";
13
13
 import { SessionDrawer } from "../components/SessionDrawer";
14
-import { playAudio, stopPlayback, isPlaying, onPlayingChange } from "../services/audio";
14
+import { playSingle, stopPlayback, isPlaying, onPlayingChange } from "../services/audio";
15
15
 
16
16
 interface StagedImage {
17
17
   base64: string;
..
..
@@ -32,7 +32,7 @@
32
32
   const [stagedImage, setStagedImage] = useState<StagedImage | null>(null);
33
33
 
34
34
   useEffect(() => {
35
-    return onPlayingChange(setAudioPlaying);
35
+    return onPlayingChange((uri) => setAudioPlaying(uri !== null));
36
36
   }, []);
37
37
 
38
38
   const handleScreenshot = useCallback(() => {
..
..
@@ -137,10 +137,8 @@
137
137
     }
138
138
     for (let i = messages.length - 1; i >= 0; i--) {
139
139
       const msg = messages[i];
140
-      if (msg.role === "assistant") {
141
-        if (msg.audioUri) {
142
-          playAudio(msg.audioUri).catch(() => {});
143
-        }
140
+      if (msg.role === "assistant" && msg.audioUri) {
141
+        playSingle(msg.audioUri).catch(() => {});
144
142
         return;
145
143
       }
146
144
     }
 components/chat/InputBar.tsx
..
..
@@ -12,7 +12,7 @@
12
12
 
13
13
 interface InputBarProps {
14
14
   onSendText: (text: string) => void;
15
-  onVoiceRecorded: (uri: string) => void;
15
+  onVoiceRecorded: (uri: string, durationMs?: number) => void;
16
16
   onReplay: () => void;
17
17
   isTextMode: boolean;
18
18
   onToggleMode: () => void;
 components/chat/MessageBubble.tsx
..
..
@@ -1,7 +1,7 @@
1
1
 import React, { useCallback, useEffect, useState } from "react";
2
2
 import { Image, Pressable, Text, View } from "react-native";
3
3
 import { Message } from "../../types";
4
-import { playAudio, stopPlayback, onPlayingChange } from "../../services/audio";
4
+import { playSingle, stopPlayback, onPlayingChange } from "../../services/audio";
5
5
 import { ImageViewer } from "./ImageViewer";
6
6
 import { useTheme } from "../../contexts/ThemeContext";
7
7
 
..
..
@@ -27,11 +27,12 @@
27
27
   const [showViewer, setShowViewer] = useState(false);
28
28
   const { colors, isDark } = useTheme();
29
29
 
30
+  // Track whether THIS bubble's audio is playing via the singleton URI
30
31
   useEffect(() => {
31
-    return onPlayingChange((playing) => {
32
-      if (!playing) setIsPlaying(false);
32
+    return onPlayingChange((uri) => {
33
+      setIsPlaying(uri !== null && uri === message.audioUri);
33
34
     });
34
-  }, []);
35
+  }, [message.audioUri]);
35
36
 
36
37
   const isUser = message.role === "user";
37
38
   const isSystem = message.role === "system";
..
..
@@ -40,11 +41,11 @@
40
41
     if (!message.audioUri) return;
41
42
 
42
43
     if (isPlaying) {
44
+      // This bubble is playing — stop it
43
45
       await stopPlayback();
44
-      setIsPlaying(false);
45
46
     } else {
46
-      setIsPlaying(true);
47
-      await playAudio(message.audioUri, () => setIsPlaying(false));
47
+      // Play this bubble (stops anything else automatically)
48
+      await playSingle(message.audioUri, () => {});
48
49
     }
49
50
   }, [isPlaying, message.audioUri]);
50
51
 
..
..
@@ -114,56 +115,70 @@
114
115
             />
115
116
           </View>
116
117
         ) : message.type === "voice" ? (
117
-          <Pressable
118
-            onPress={handleVoicePress}
119
-            style={{ flexDirection: "row", alignItems: "center", gap: 12 }}
120
-          >
121
-            <View
122
-              style={{
123
-                width: 36,
124
-                height: 36,
125
-                borderRadius: 18,
126
-                alignItems: "center",
127
-                justifyContent: "center",
128
-                backgroundColor: isPlaying
129
-                  ? "#FF9F43"
130
-                  : isUser
131
-                  ? "rgba(255,255,255,0.2)"
132
-                  : colors.border,
133
-              }}
118
+          <View>
119
+            <Pressable
120
+              onPress={handleVoicePress}
121
+              style={{ flexDirection: "row", alignItems: "center", gap: 12 }}
134
122
             >
135
-              <Text style={{ fontSize: 14, color: isUser ? "#FFF" : colors.text }}>
136
-                {isPlaying ? "\u23F8" : "\u25B6"}
123
+              <View
124
+                style={{
125
+                  width: 36,
126
+                  height: 36,
127
+                  borderRadius: 18,
128
+                  alignItems: "center",
129
+                  justifyContent: "center",
130
+                  backgroundColor: isPlaying
131
+                    ? "#FF9F43"
132
+                    : isUser
133
+                    ? "rgba(255,255,255,0.2)"
134
+                    : colors.border,
135
+                }}
136
+              >
137
+                <Text style={{ fontSize: 14, color: isUser ? "#FFF" : colors.text }}>
138
+                  {isPlaying ? "\u23F8" : "\u25B6"}
139
+                </Text>
140
+              </View>
141
+
142
+              <View style={{ flex: 1, flexDirection: "row", alignItems: "center", gap: 1, height: 32 }}>
143
+                {Array.from({ length: 20 }).map((_, i) => (
144
+                  <View
145
+                    key={i}
146
+                    style={{
147
+                      flex: 1,
148
+                      borderRadius: 2,
149
+                      backgroundColor: isPlaying && i < 10
150
+                        ? "#FF9F43"
151
+                        : isUser
152
+                        ? "rgba(255,255,255,0.5)"
153
+                        : colors.textMuted,
154
+                      height: `${20 + Math.sin(i * 0.8) * 60}%`,
155
+                    }}
156
+                  />
157
+                ))}
158
+              </View>
159
+
160
+              <Text
161
+                style={{
162
+                  fontSize: 11,
163
+                  color: isUser ? "rgba(255,255,255,0.8)" : colors.textSecondary,
164
+                }}
165
+              >
166
+                {formatDuration(message.duration)}
137
167
               </Text>
138
-            </View>
139
-
140
-            <View style={{ flex: 1, flexDirection: "row", alignItems: "center", gap: 1, height: 32 }}>
141
-              {Array.from({ length: 20 }).map((_, i) => (
142
-                <View
143
-                  key={i}
144
-                  style={{
145
-                    flex: 1,
146
-                    borderRadius: 2,
147
-                    backgroundColor: isPlaying && i < 10
148
-                      ? "#FF9F43"
149
-                      : isUser
150
-                      ? "rgba(255,255,255,0.5)"
151
-                      : colors.textMuted,
152
-                    height: `${20 + Math.sin(i * 0.8) * 60}%`,
153
-                  }}
154
-                />
155
-              ))}
156
-            </View>
157
-
158
-            <Text
159
-              style={{
160
-                fontSize: 11,
161
-                color: isUser ? "rgba(255,255,255,0.8)" : colors.textSecondary,
162
-              }}
163
-            >
164
-              {formatDuration(message.duration)}
165
-            </Text>
166
-          </Pressable>
168
+            </Pressable>
169
+            {message.content ? (
170
+              <Text
171
+                style={{
172
+                  fontSize: 14,
173
+                  lineHeight: 20,
174
+                  marginTop: 8,
175
+                  color: isUser ? "rgba(255,255,255,0.9)" : colors.textSecondary,
176
+                }}
177
+              >
178
+                {message.content}
179
+              </Text>
180
+            ) : null}
181
+          </View>
167
182
         ) : (
168
183
           <Text
169
184
             style={{
 components/chat/VoiceButton.tsx
..
..
@@ -10,7 +10,7 @@
10
10
 import { stopPlayback } from "../../services/audio";
11
11
 
12
12
 interface VoiceButtonProps {
13
-  onVoiceRecorded: (uri: string) => void;
13
+  onVoiceRecorded: (uri: string, durationMs?: number) => void;
14
14
 }
15
15
 
16
16
 const VOICE_BUTTON_SIZE = 72;
..
..
@@ -98,7 +98,11 @@
98
98
       });
99
99
       const uri = recorder.uri;
100
100
       if (uri) {
101
-        onVoiceRecorded(uri);
101
+        // currentTime is in seconds after stop
102
+        const durationMs = recorder.currentTime > 0
103
+          ? Math.round(recorder.currentTime * 1000)
104
+          : undefined;
105
+        onVoiceRecorded(uri, durationMs);
102
106
       }
103
107
     } catch (err) {
104
108
       console.error("Failed to stop recording:", err);
 contexts/ChatContext.tsx
..
..
@@ -26,7 +26,9 @@
26
26
 
27
27
 const MESSAGES_DIR = "pailot-messages";
28
28
 
29
-/** Strip heavy fields (base64 images, audio URIs) before persisting. */
29
+/** Strip heavy fields (base64 images, audio URIs) before persisting.
30
+ *  Voice messages keep their content (transcript) but lose audioUri
31
+ *  since cache files won't survive app restarts. */
30
32
 function lightMessage(m: Message): Message {
31
33
   const light = { ...m };
32
34
   if (light.imageBase64) light.imageBase64 = undefined;
..
..
@@ -63,7 +65,16 @@
63
65
       if (!file.endsWith(".json")) continue;
64
66
       const sessionId = file.replace(".json", "");
65
67
       const content = await fs.readAsStringAsync(`${dir}${file}`);
66
-      result[sessionId] = JSON.parse(content) as Message[];
68
+      result[sessionId] = (JSON.parse(content) as Message[])
69
+        // Drop voice messages with no audio and no content (empty chunks)
70
+        .filter((m) => !(m.type === "voice" && !m.audioUri && !m.content))
71
+        .map((m) => {
72
+          // Voice messages without audio but with transcript → show as text
73
+          if (m.type === "voice" && !m.audioUri && m.content) {
74
+            return { ...m, type: "text" };
75
+          }
76
+          return m;
77
+        });
67
78
     }
68
79
     return result;
69
80
   } catch {
..
..
@@ -179,12 +190,15 @@
179
190
     }
180
191
   }, [messages]);
181
192
 
182
-  // On connect: ask gateway to detect the focused iTerm2 session and sync
193
+  // On connect: ask gateway to sync sessions. If we already had a session
194
+  // selected, tell the gateway so it preserves our selection instead of
195
+  // jumping to whatever iTerm has focused on the Mac.
183
196
   useEffect(() => {
184
197
     if (status === "connected") {
185
198
       needsSync.current = true;
186
-      sendCommand("sync");
199
+      sendCommand("sync", activeSessionId ? { activeSessionId } : undefined);
187
200
     }
201
+    // eslint-disable-next-line react-hooks/exhaustive-deps — only fire on status change
188
202
   }, [status, sendCommand]);
189
203
 
190
204
   // Helper: add a message to the active session
..
..
@@ -233,6 +247,23 @@
233
247
     },
234
248
     []
235
249
   );
250
+
251
+  // Update a message's content (e.g., voice transcript reflection)
252
+  const updateMessageContent = useCallback((id: string, content: string) => {
253
+    setMessages((prev) => {
254
+      const next = prev.map((m) =>
255
+        m.id === id ? { ...m, content } : m
256
+      );
257
+      setActiveSessionId((sessId) => {
258
+        if (sessId) {
259
+          messagesMapRef.current[sessId] = next;
260
+          debouncedSave(messagesMapRef.current);
261
+        }
262
+        return sessId;
263
+      });
264
+      return next;
265
+    });
266
+  }, []);
236
267
 
237
268
   // Handle incoming WebSocket messages
238
269
   useEffect(() => {
..
..
@@ -322,6 +353,11 @@
322
353
           sendCommand("sessions");
323
354
           break;
324
355
         }
356
+        case "transcript": {
357
+          // Voice → text reflection: replace voice bubble with transcribed text
358
+          updateMessageContent(data.messageId, data.content);
359
+          break;
360
+        }
325
361
         case "error": {
326
362
           const msg: Message = {
327
363
             id: generateId(),
..
..
@@ -339,7 +375,7 @@
339
375
     return () => {
340
376
       onMessageReceived.current = null;
341
377
     };
342
-  }, [onMessageReceived, sendCommand, addMessageToActive, syncActiveFromSessions]);
378
+  }, [onMessageReceived, sendCommand, addMessageToActive, updateMessageContent, syncActiveFromSessions]);
343
379
 
344
380
   const sendTextMessage = useCallback(
345
381
     (text: string) => {
..
..
@@ -375,7 +411,7 @@
375
411
       addMessageToActive(msg);
376
412
       try {
377
413
         const base64 = await encodeAudioToBase64(audioUri);
378
-        const sent = wsVoice(base64);
414
+        const sent = wsVoice(base64, "", id);
379
415
         updateMessageStatus(id, sent ? "sent" : "error");
380
416
       } catch (err) {
381
417
         console.error("Failed to encode audio:", err);
 contexts/ConnectionContext.tsx
..
..
@@ -24,7 +24,7 @@
24
24
   connect: (config?: ServerConfig) => void;
25
25
   disconnect: () => void;
26
26
   sendTextMessage: (text: string) => boolean;
27
-  sendVoiceMessage: (audioBase64: string, transcript?: string) => boolean;
27
+  sendVoiceMessage: (audioBase64: string, transcript?: string, messageId?: string) => boolean;
28
28
   sendImageMessage: (imageBase64: string, caption: string, mimeType: string) => boolean;
29
29
   sendCommand: (command: string, args?: Record<string, unknown>) => boolean;
30
30
   saveServerConfig: (config: ServerConfig) => Promise<void>;
..
..
@@ -120,11 +120,12 @@
120
120
   }, []);
121
121
 
122
122
   const sendVoiceMessage = useCallback(
123
-    (audioBase64: string, transcript: string = ""): boolean => {
123
+    (audioBase64: string, transcript: string = "", messageId?: string): boolean => {
124
124
       return wsClient.send({
125
125
         type: "voice",
126
126
         content: transcript,
127
127
         audioBase64,
128
+        messageId,
128
129
       });
129
130
     },
130
131
     []
 services/audio.ts
..
..
@@ -10,20 +10,34 @@
10
10
   durationMs: number;
11
11
 }
12
12
 
13
+// --- Singleton audio player ---
14
+// Only ONE audio can play at a time. Any new play request stops the current one.
15
+
13
16
 let currentPlayer: ReturnType<typeof createAudioPlayer> | null = null;
14
-const playingListeners = new Set<(playing: boolean) => void>();
17
+let currentUri: string | null = null;
18
+let cancelCurrent: (() => void) | null = null;
15
19
 
16
-// Audio queue for chaining sequential voice notes
17
-const audioQueue: Array<{ uri: string; onFinish?: () => void }> = [];
18
-let processingQueue = false;
20
+// Listeners get the URI of what's playing (or null when stopped)
21
+const playingListeners = new Set<(uri: string | null) => void>();
19
22
 
20
-function notifyListeners(playing: boolean): void {
21
-  for (const cb of playingListeners) cb(playing);
23
+function notifyListeners(uri: string | null): void {
24
+  currentUri = uri;
25
+  for (const cb of playingListeners) cb(uri);
22
26
 }
23
27
 
24
-export function onPlayingChange(cb: (playing: boolean) => void): () => void {
28
+/** Subscribe to playing state changes. Returns unsubscribe function. */
29
+export function onPlayingChange(cb: (uri: string | null) => void): () => void {
25
30
   playingListeners.add(cb);
26
31
   return () => { playingListeners.delete(cb); };
32
+}
33
+
34
+/** Get the URI currently playing, or null. */
35
+export function playingUri(): string | null {
36
+  return currentUri;
37
+}
38
+
39
+export function isPlaying(): boolean {
40
+  return currentPlayer !== null;
27
41
 }
28
42
 
29
43
 export async function requestPermissions(): Promise<boolean> {
..
..
@@ -44,9 +58,13 @@
44
58
   return tmpPath;
45
59
 }
46
60
 
61
+// --- Audio queue for chaining sequential voice notes (autoplay) ---
62
+const audioQueue: Array<{ uri: string; onFinish?: () => void }> = [];
63
+let processingQueue = false;
64
+
47
65
 /**
48
- * Queue audio for playback. Multiple calls chain sequentially —
49
- * the next voice note plays only after the current one finishes.
66
+ * Play audio. Stops any current playback first (singleton).
67
+ * Multiple calls chain sequentially via queue (for chunked voice notes).
50
68
  */
51
69
 export async function playAudio(
52
70
   uri: string,
..
..
@@ -56,6 +74,18 @@
56
74
   if (!processingQueue) {
57
75
     processAudioQueue();
58
76
   }
77
+}
78
+
79
+/**
80
+ * Play a single audio file, stopping any current playback first.
81
+ * Does NOT queue — immediately replaces whatever is playing.
82
+ */
83
+export async function playSingle(
84
+  uri: string,
85
+  onFinish?: () => void
86
+): Promise<void> {
87
+  await stopPlayback();
88
+  await playOneAudio(uri, onFinish);
59
89
 }
60
90
 
61
91
 async function processAudioQueue(): Promise<void> {
..
..
@@ -72,35 +102,57 @@
72
102
 
73
103
 function playOneAudio(uri: string, onFinish?: () => void): Promise<void> {
74
104
   return new Promise<void>(async (resolve) => {
105
+    let settled = false;
106
+    const finish = () => {
107
+      if (settled) return;
108
+      settled = true;
109
+      cancelCurrent = null;
110
+      clearTimeout(timer);
111
+      onFinish?.();
112
+      try { player?.pause(); } catch { /* ignore */ }
113
+      try { player?.remove(); } catch { /* ignore */ }
114
+      if (currentPlayer === player) {
115
+        currentPlayer = null;
116
+        notifyListeners(null);
117
+      }
118
+      resolve();
119
+    };
120
+
121
+    // Stop any currently playing audio first
122
+    if (cancelCurrent) {
123
+      cancelCurrent();
124
+    }
125
+
126
+    // Register cancel callback so stopPlayback can abort us
127
+    cancelCurrent = finish;
128
+
129
+    // Safety timeout
130
+    const timer = setTimeout(finish, 5 * 60 * 1000);
131
+    let player: ReturnType<typeof createAudioPlayer> | null = null;
132
+
75
133
     try {
76
134
       await setAudioModeAsync({ playsInSilentMode: true });
77
135
 
78
-      const player = createAudioPlayer(uri);
136
+      player = createAudioPlayer(uri);
79
137
       currentPlayer = player;
80
-      notifyListeners(true);
138
+      notifyListeners(uri);
81
139
 
82
140
       player.addListener("playbackStatusUpdate", (status) => {
83
-        if (!status.playing && status.currentTime >= status.duration && status.duration > 0) {
84
-          onFinish?.();
85
-          player.remove();
86
-          if (currentPlayer === player) {
87
-            currentPlayer = null;
88
-            if (audioQueue.length === 0) notifyListeners(false);
89
-          }
90
-          resolve();
141
+        if (!status.playing && status.currentTime > 0 &&
142
+            (status.duration <= 0 || status.currentTime >= status.duration)) {
143
+          finish();
91
144
         }
92
145
       });
93
146
 
94
147
       player.play();
95
148
     } catch (error) {
96
149
       console.error("Failed to play audio:", error);
150
+      settled = true;
151
+      cancelCurrent = null;
152
+      clearTimeout(timer);
97
153
       resolve();
98
154
     }
99
155
   });
100
-}
101
-
102
-export function isPlaying(): boolean {
103
-  return currentPlayer !== null;
104
156
 }
105
157
 
106
158
 /**
..
..
@@ -108,7 +160,9 @@
108
160
  */
109
161
 export async function stopPlayback(): Promise<void> {
110
162
   audioQueue.length = 0;
111
-  if (currentPlayer) {
163
+  if (cancelCurrent) {
164
+    cancelCurrent();
165
+  } else if (currentPlayer) {
112
166
     try {
113
167
       currentPlayer.pause();
114
168
       currentPlayer.remove();
..
..
@@ -116,7 +170,7 @@
116
170
       // Ignore cleanup errors
117
171
     }
118
172
     currentPlayer = null;
119
-    notifyListeners(false);
173
+    notifyListeners(null);
120
174
   }
121
175
 }
122
176
 
 services/wol.ts
..
..
@@ -56,12 +56,31 @@
56
56
     }
57
57
   }
58
58
 
59
+  const TIMEOUT_MS = 5000;
60
+
59
61
   return new Promise<void>((resolve, reject) => {
62
+    let settled = false;
63
+    const settle = (fn: () => void) => {
64
+      if (settled) return;
65
+      settled = true;
66
+      clearTimeout(timer);
67
+      fn();
68
+    };
69
+
70
+    const timer = setTimeout(() => {
71
+      settle(() => {
72
+        try { socket.close(); } catch { /* ignore */ }
73
+        reject(new Error("WoL timed out — magic packet may not have been sent"));
74
+      });
75
+    }, TIMEOUT_MS);
76
+
60
77
     const socket = dgram.createSocket({ type: "udp4" });
61
78
 
62
79
     socket.once("error", (err: Error) => {
63
-      try { socket.close(); } catch { /* ignore */ }
64
-      reject(err);
80
+      settle(() => {
81
+        try { socket.close(); } catch { /* ignore */ }
82
+        reject(err);
83
+      });
65
84
     });
66
85
 
67
86
     socket.bind(0, () => {
..
..
@@ -72,20 +91,22 @@
72
91
       }
73
92
 
74
93
       let pending = broadcastAddresses.length;
75
-      let failed = false;
76
94
 
77
95
       for (const addr of broadcastAddresses) {
78
96
         socket.send(packet, 0, packet.length, 9, addr, (err?: Error) => {
79
-          if (err && !failed) {
80
-            failed = true;
81
-            try { socket.close(); } catch { /* ignore */ }
82
-            reject(err);
97
+          if (err) {
98
+            settle(() => {
99
+              try { socket.close(); } catch { /* ignore */ }
100
+              reject(err);
101
+            });
83
102
             return;
84
103
           }
85
104
           pending--;
86
105
           if (pending === 0) {
87
-            try { socket.close(); } catch { /* ignore */ }
88
-            resolve();
106
+            settle(() => {
107
+              try { socket.close(); } catch { /* ignore */ }
108
+              resolve();
109
+            });
89
110
           }
90
111
         });
91
112
       }
 types/index.ts
..
..
@@ -34,6 +34,7 @@
34
34
   type: "voice";
35
35
   audioBase64: string;
36
36
   content: string;
37
+  messageId?: string;
37
38
 }
38
39
 
39
40
 export interface WsImageMessage {
..
..
@@ -95,6 +96,12 @@
95
96
   name: string;
96
97
 }
97
98
 
99
+export interface WsIncomingTranscript {
100
+  type: "transcript";
101
+  messageId: string;
102
+  content: string;
103
+}
104
+
98
105
 export interface WsIncomingError {
99
106
   type: "error";
100
107
   message: string;
..
..
@@ -112,5 +119,6 @@
112
119
   | WsIncomingSessions
113
120
   | WsIncomingSessionSwitched
114
121
   | WsIncomingSessionRenamed
122
+  | WsIncomingTranscript
115
123
   | WsIncomingError
116
124
   | WsIncomingStatus;