feat: on-device speech recognition, navigation screen, session picker

Matthias Nott

2026-03-02 a0f39302919fbacf7a0d407f01b1a50413ea6f70

 contexts/ChatContext.tsx
..
..
@@ -6,9 +6,9 @@
6
6
   useRef,
7
7
   useState,
8
8
 } from "react";
9
-import { Message, WebSocketMessage } from "../types";
9
+import { Message, WsIncoming, WsSession } from "../types";
10
10
 import { useConnection } from "./ConnectionContext";
11
-import { playAudio } from "../services/audio";
11
+import { playAudio, encodeAudioToBase64 } from "../services/audio";
12
12
 
13
13
 function generateId(): string {
14
14
   return Date.now().toString(36) + Math.random().toString(36).slice(2);
..
..
@@ -19,13 +19,29 @@
19
19
   sendTextMessage: (text: string) => void;
20
20
   sendVoiceMessage: (audioUri: string, durationMs?: number) => void;
21
21
   clearMessages: () => void;
22
+  // Session management
23
+  sessions: WsSession[];
24
+  requestSessions: () => void;
25
+  switchSession: (sessionId: string) => void;
26
+  renameSession: (sessionId: string, name: string) => void;
27
+  // Screenshot / navigation
28
+  latestScreenshot: string | null;
29
+  requestScreenshot: () => void;
30
+  sendNavKey: (key: string) => void;
22
31
 }
23
32
 
24
33
 const ChatContext = createContext<ChatContextValue | null>(null);
25
34
 
26
35
 export function ChatProvider({ children }: { children: React.ReactNode }) {
27
36
   const [messages, setMessages] = useState<Message[]>([]);
28
-  const { sendTextMessage: wsSend, sendVoiceMessage: wsVoice, onMessageReceived } = useConnection();
37
+  const [sessions, setSessions] = useState<WsSession[]>([]);
38
+  const [latestScreenshot, setLatestScreenshot] = useState<string | null>(null);
39
+  const {
40
+    sendTextMessage: wsSend,
41
+    sendVoiceMessage: wsVoice,
42
+    sendCommand,
43
+    onMessageReceived,
44
+  } = useConnection();
29
45
 
30
46
   const addMessage = useCallback((msg: Message) => {
31
47
     setMessages((prev) => [...prev, msg]);
..
..
@@ -42,34 +58,92 @@
42
58
 
43
59
   // Handle incoming WebSocket messages
44
60
   useEffect(() => {
45
-    onMessageReceived.current = (data: WebSocketMessage) => {
46
-      if (data.type === "text") {
47
-        const msg: Message = {
48
-          id: generateId(),
49
-          role: "assistant",
50
-          type: "text",
51
-          content: data.content,
52
-          timestamp: Date.now(),
53
-          status: "sent",
54
-        };
55
-        setMessages((prev) => [...prev, msg]);
56
-      } else if (data.type === "voice") {
57
-        const msg: Message = {
58
-          id: generateId(),
59
-          role: "assistant",
60
-          type: "voice",
61
-          content: data.content ?? "",
62
-          audioUri: data.audioBase64
63
-            ? `data:audio/mp4;base64,${data.audioBase64}`
64
-            : undefined,
65
-          timestamp: Date.now(),
66
-          status: "sent",
67
-        };
68
-        setMessages((prev) => [...prev, msg]);
69
-
70
-        // Auto-play incoming voice messages
71
-        if (msg.audioUri) {
72
-          playAudio(msg.audioUri).catch(() => {});
61
+    onMessageReceived.current = (data: WsIncoming) => {
62
+      switch (data.type) {
63
+        case "text": {
64
+          const msg: Message = {
65
+            id: generateId(),
66
+            role: "assistant",
67
+            type: "text",
68
+            content: data.content,
69
+            timestamp: Date.now(),
70
+            status: "sent",
71
+          };
72
+          setMessages((prev) => [...prev, msg]);
73
+          break;
74
+        }
75
+        case "voice": {
76
+          const msg: Message = {
77
+            id: generateId(),
78
+            role: "assistant",
79
+            type: "voice",
80
+            content: data.content ?? "",
81
+            audioUri: data.audioBase64
82
+              ? `data:audio/mp4;base64,${data.audioBase64}`
83
+              : undefined,
84
+            timestamp: Date.now(),
85
+            status: "sent",
86
+          };
87
+          setMessages((prev) => [...prev, msg]);
88
+          if (msg.audioUri) {
89
+            playAudio(msg.audioUri).catch(() => {});
90
+          }
91
+          break;
92
+        }
93
+        case "image": {
94
+          // Store as latest screenshot for navigation mode
95
+          setLatestScreenshot(data.imageBase64);
96
+          // Also add to chat as an image message
97
+          const msg: Message = {
98
+            id: generateId(),
99
+            role: "assistant",
100
+            type: "image",
101
+            content: data.caption ?? "Screenshot",
102
+            imageBase64: data.imageBase64,
103
+            timestamp: Date.now(),
104
+            status: "sent",
105
+          };
106
+          setMessages((prev) => [...prev, msg]);
107
+          break;
108
+        }
109
+        case "sessions": {
110
+          setSessions(data.sessions);
111
+          break;
112
+        }
113
+        case "session_switched": {
114
+          const msg: Message = {
115
+            id: generateId(),
116
+            role: "system",
117
+            type: "text",
118
+            content: `Switched to ${data.name}`,
119
+            timestamp: Date.now(),
120
+          };
121
+          setMessages((prev) => [...prev, msg]);
122
+          break;
123
+        }
124
+        case "session_renamed": {
125
+          const msg: Message = {
126
+            id: generateId(),
127
+            role: "system",
128
+            type: "text",
129
+            content: `Renamed to ${data.name}`,
130
+            timestamp: Date.now(),
131
+          };
132
+          setMessages((prev) => [...prev, msg]);
133
+          // Refresh sessions to show updated name
134
+          sendCommand("sessions");
135
+          break;
136
+        }
137
+        case "error": {
138
+          const msg: Message = {
139
+            id: generateId(),
140
+            role: "system",
141
+            type: "text",
142
+            content: data.message,
143
+            timestamp: Date.now(),
144
+          };
145
+          setMessages((prev) => [...prev, msg]);
146
+          break;
73
147
         }
74
148
       }
75
149
     };
..
..
@@ -77,7 +151,7 @@
77
151
     return () => {
78
152
       onMessageReceived.current = null;
79
153
     };
80
-  }, [onMessageReceived]);
154
+  }, [onMessageReceived, sendCommand]);
81
155
 
82
156
   const sendTextMessage = useCallback(
83
157
     (text: string) => {
..
..
@@ -91,7 +165,6 @@
91
165
         status: "sending",
92
166
       };
93
167
       addMessage(msg);
94
-
95
168
       const sent = wsSend(text);
96
169
       updateMessageStatus(id, sent ? "sent" : "error");
97
170
     },
..
..
@@ -99,7 +172,7 @@
99
172
   );
100
173
 
101
174
   const sendVoiceMessage = useCallback(
102
-    (audioUri: string, durationMs?: number) => {
175
+    async (audioUri: string, durationMs?: number) => {
103
176
       const id = generateId();
104
177
       const msg: Message = {
105
178
         id,
..
..
@@ -112,10 +185,14 @@
112
185
         duration: durationMs,
113
186
       };
114
187
       addMessage(msg);
115
-
116
-      // For now, send with empty base64 since we'd need expo-file-system to encode
117
-      const sent = wsVoice("", "Voice message");
118
-      updateMessageStatus(id, sent ? "sent" : "error");
188
+      try {
189
+        const base64 = await encodeAudioToBase64(audioUri);
190
+        const sent = wsVoice(base64);
191
+        updateMessageStatus(id, sent ? "sent" : "error");
192
+      } catch (err) {
193
+        console.error("Failed to encode audio:", err);
194
+        updateMessageStatus(id, "error");
195
+      }
119
196
     },
120
197
     [wsVoice, addMessage, updateMessageStatus]
121
198
   );
..
..
@@ -124,9 +201,52 @@
124
201
     setMessages([]);
125
202
   }, []);
126
203
 
204
+  // --- Session management ---
205
+  const requestSessions = useCallback(() => {
206
+    sendCommand("sessions");
207
+  }, [sendCommand]);
208
+
209
+  const switchSession = useCallback(
210
+    (sessionId: string) => {
211
+      sendCommand("switch", { sessionId });
212
+    },
213
+    [sendCommand]
214
+  );
215
+
216
+  const renameSession = useCallback(
217
+    (sessionId: string, name: string) => {
218
+      sendCommand("rename", { sessionId, name });
219
+    },
220
+    [sendCommand]
221
+  );
222
+
223
+  // --- Screenshot / navigation ---
224
+  const requestScreenshot = useCallback(() => {
225
+    sendCommand("screenshot");
226
+  }, [sendCommand]);
227
+
228
+  const sendNavKey = useCallback(
229
+    (key: string) => {
230
+      sendCommand("nav", { key });
231
+    },
232
+    [sendCommand]
233
+  );
234
+
127
235
   return (
128
236
     <ChatContext.Provider
129
-      value={{ messages, sendTextMessage, sendVoiceMessage, clearMessages }}
237
+      value={{
238
+        messages,
239
+        sendTextMessage,
240
+        sendVoiceMessage,
241
+        clearMessages,
242
+        sessions,
243
+        requestSessions,
244
+        switchSession,
245
+        renameSession,
246
+        latestScreenshot,
247
+        requestScreenshot,
248
+        sendNavKey,
249
+      }}
130
250
     >
131
251
       {children}
132
252
     </ChatContext.Provider>