feat: on-device speech recognition, navigation screen, session picker

Matthias Nott

2026-03-02 a0f39302919fbacf7a0d407f01b1a50413ea6f70

 types/index.ts
..
..
@@ -1,5 +1,5 @@
1
1
 export type MessageRole = "user" | "assistant" | "system";
2
-export type MessageType = "text" | "voice";
2
+export type MessageType = "text" | "voice" | "image";
3
3
 
4
4
 export interface Message {
5
5
   id: string;
..
..
@@ -7,6 +7,7 @@
7
7
   type: MessageType;
8
8
   content: string;
9
9
   audioUri?: string;
10
+  imageBase64?: string;
10
11
   timestamp: number;
11
12
   status?: "sending" | "sent" | "error";
12
13
   duration?: number;
..
..
@@ -19,8 +20,81 @@
19
20
 
20
21
 export type ConnectionStatus = "disconnected" | "connecting" | "connected";
21
22
 
22
-export interface WebSocketMessage {
23
-  type: "text" | "voice";
23
+// --- WebSocket protocol ---
24
+
25
+/** Outgoing from app to watcher */
26
+export interface WsTextMessage {
27
+  type: "text";
28
+  content: string;
29
+}
30
+
31
+export interface WsVoiceMessage {
32
+  type: "voice";
33
+  audioBase64: string;
34
+  content: string;
35
+}
36
+
37
+export interface WsCommandMessage {
38
+  type: "command";
39
+  command: string;
40
+  args?: Record<string, unknown>;
41
+}
42
+
43
+export type WsOutgoing = WsTextMessage | WsVoiceMessage | WsCommandMessage;
44
+
45
+/** Incoming from watcher to app */
46
+export interface WsIncomingText {
47
+  type: "text";
48
+  content: string;
49
+}
50
+
51
+export interface WsIncomingVoice {
52
+  type: "voice";
24
53
   content: string;
25
54
   audioBase64?: string;
26
55
 }
56
+
57
+export interface WsIncomingImage {
58
+  type: "image";
59
+  imageBase64: string;
60
+  caption?: string;
61
+}
62
+
63
+export interface WsSession {
64
+  index: number;
65
+  name: string;
66
+  type: "claude" | "terminal";
67
+  isActive: boolean;
68
+  id: string;
69
+}
70
+
71
+export interface WsIncomingSessions {
72
+  type: "sessions";
73
+  sessions: WsSession[];
74
+}
75
+
76
+export interface WsIncomingSessionSwitched {
77
+  type: "session_switched";
78
+  name: string;
79
+  sessionId: string;
80
+}
81
+
82
+export interface WsIncomingSessionRenamed {
83
+  type: "session_renamed";
84
+  sessionId: string;
85
+  name: string;
86
+}
87
+
88
+export interface WsIncomingError {
89
+  type: "error";
90
+  message: string;
91
+}
92
+
93
+export type WsIncoming =
94
+  | WsIncomingText
95
+  | WsIncomingVoice
96
+  | WsIncomingImage
97
+  | WsIncomingSessions
98
+  | WsIncomingSessionSwitched
99
+  | WsIncomingSessionRenamed
100
+  | WsIncomingError;