feat: multi-image upload and catch_up message delivery

Matthias Nott

2026-03-15 e25bdba29f49b1b55a8a8cccdc4583aea3c101ed

 contexts/ChatContext.tsx
..
..
@@ -6,6 +6,7 @@
6
6
   useRef,
7
7
   useState,
8
8
 } from "react";
9
+import { AppState, AppStateStatus } from "react-native";
9
10
 import { Message, WsIncoming, WsSession, PaiProject } from "../types";
10
11
 import { useConnection } from "./ConnectionContext";
11
12
 import { playAudio, encodeAudioToBase64, saveBase64Audio, canAutoplay } from "../services/audio";
..
..
@@ -142,6 +143,7 @@
142
143
   loadMoreMessages: () => void;
143
144
   hasMoreMessages: boolean;
144
145
   unreadCounts: Record<string, number>;
146
+  unreadSessions: Set<string>;
145
147
   incomingToast: IncomingToast | null;
146
148
   dismissToast: () => void;
147
149
   latestScreenshot: string | null;
..
..
@@ -158,12 +160,18 @@
158
160
   const [latestScreenshot, setLatestScreenshot] = useState<string | null>(null);
159
161
   const needsSync = useRef(true);
160
162
 
163
+  // Sequence tracking for catch_up protocol
164
+  const lastSeqRef = useRef(0);
165
+  const seenSeqsRef = useRef(new Set<number>());
166
+
161
167
   // Per-session message storage
162
168
   const messagesMapRef = useRef<Record<string, Message[]>>({});
163
169
   // Messages for the active session (drives re-renders)
164
170
   const [messages, setMessages] = useState<Message[]>([]);
165
171
   // Unread counts for non-active sessions
166
172
   const [unreadCounts, setUnreadCounts] = useState<Record<string, number>>({});
173
+  // Server-pushed unread indicators (sessions with new activity since last viewed)
174
+  const [unreadSessions, setUnreadSessions] = useState<Set<string>>(new Set());
167
175
   // Per-session typing indicator (sessionId → boolean)
168
176
   const typingMapRef = useRef<Record<string, boolean>>({});
169
177
   const [isTyping, setIsTyping] = useState(false);
..
..
@@ -211,6 +219,12 @@
211
219
             delete next[active.id];
212
220
             return next;
213
221
           });
222
+          setUnreadSessions((prev) => {
223
+            if (!prev.has(active.id)) return prev;
224
+            const next = new Set(prev);
225
+            next.delete(active.id);
226
+            return next;
227
+          });
214
228
           // Sync typing indicator for the new active session
215
229
           const activeTyping = typingMapRef.current[active.id] ?? false;
216
230
           setIsTyping(activeTyping);
..
..
@@ -221,18 +235,34 @@
221
235
     }
222
236
   }, []);
223
237
 
224
-  // On connect: ask gateway to sync sessions. If we already had a session
225
-  // selected, tell the gateway so it preserves our selection instead of
226
-  // jumping to whatever iTerm has focused on the Mac.
238
+  // On connect: ask gateway to sync sessions, then request catch_up for missed messages.
227
239
   useEffect(() => {
228
240
     if (status === "connected") {
229
241
       needsSync.current = true;
230
242
       const id = activeSessionIdRef.current;
231
243
       sendCommand("sync", id ? { activeSessionId: id } : undefined);
244
+      // Request any messages we missed while disconnected/backgrounded
245
+      sendCommand("catch_up", { lastSeq: lastSeqRef.current });
232
246
     } else if (status === "disconnected") {
233
247
       setIsTyping(false);
234
248
     }
235
249
     // eslint-disable-next-line react-hooks/exhaustive-deps — only fire on status change
250
+  }, [status, sendCommand]);
251
+
252
+  // On foreground resume: request catch_up for any messages missed while backgrounded.
253
+  // iOS keeps the WebSocket "open" at TCP level but suspends the app — messages sent
254
+  // during that time are lost. catch_up replays them from the server's message log.
255
+  useEffect(() => {
256
+    let lastState: AppStateStatus = AppState.currentState;
257
+    const sub = AppState.addEventListener("change", (nextState) => {
258
+      if (lastState.match(/inactive|background/) && nextState === "active") {
259
+        if (status === "connected") {
260
+          sendCommand("catch_up", { lastSeq: lastSeqRef.current });
261
+        }
262
+      }
263
+      lastState = nextState;
264
+    });
265
+    return () => sub.remove();
236
266
   }, [status, sendCommand]);
237
267
 
238
268
   // Helper: add a message to the active session
..
..
@@ -309,135 +339,173 @@
309
339
     });
310
340
   }, []);
311
341
 
342
+  // Process a single incoming message (used by both live delivery and catch_up replay)
343
+  const processIncoming = useCallback(async (data: WsIncoming, isCatchUp = false) => {
344
+    // Dedup by seq: if we've seen this seq before, skip it
345
+    const seq = (data as any).seq as number | undefined;
346
+    if (seq) {
347
+      if (seenSeqsRef.current.has(seq)) return;
348
+      seenSeqsRef.current.add(seq);
349
+      lastSeqRef.current = Math.max(lastSeqRef.current, seq);
350
+      // Keep seen set bounded (last 500 seqs)
351
+      if (seenSeqsRef.current.size > 500) {
352
+        const arr = Array.from(seenSeqsRef.current).sort((a, b) => a - b);
353
+        seenSeqsRef.current = new Set(arr.slice(-300));
354
+      }
355
+    }
356
+
357
+    switch (data.type) {
358
+      case "text": {
359
+        if (!isCatchUp) setIsTyping(false);
360
+        const msg: Message = {
361
+          id: generateId(),
362
+          role: "assistant",
363
+          type: "text",
364
+          content: data.content,
365
+          timestamp: Date.now(),
366
+          status: "sent",
367
+        };
368
+        if (data.sessionId) {
369
+          addMessageToSession(data.sessionId, msg);
370
+        } else {
371
+          addMessageToActive(msg);
372
+        }
373
+        if (!isCatchUp) notifyIncomingMessage("PAILot", data.content ?? "New message");
374
+        break;
375
+      }
376
+      case "voice": {
377
+        if (!isCatchUp) setIsTyping(false);
378
+        let audioUri: string | undefined;
379
+        if (data.audioBase64) {
380
+          try {
381
+            audioUri = await saveBase64Audio(data.audioBase64);
382
+          } catch {
383
+            // fallback: no playable audio
384
+          }
385
+        }
386
+        const msg: Message = {
387
+          id: generateId(),
388
+          role: "assistant",
389
+          type: "voice",
390
+          content: data.content ?? "",
391
+          audioUri,
392
+          timestamp: Date.now(),
393
+          status: "sent",
394
+        };
395
+        const isForActive = !data.sessionId || data.sessionId === activeSessionIdRef.current;
396
+        if (data.sessionId) {
397
+          addMessageToSession(data.sessionId, msg);
398
+        } else {
399
+          addMessageToActive(msg);
400
+        }
401
+        if (!isCatchUp) notifyIncomingMessage("PAILot", data.content ?? "Voice message");
402
+        // Only autoplay if live (not catch_up) and for the currently viewed session
403
+        if (!isCatchUp && msg.audioUri && canAutoplay() && isForActive) {
404
+          playAudio(msg.audioUri).catch(() => {});
405
+        }
406
+        break;
407
+      }
408
+      case "image": {
409
+        setLatestScreenshot(data.imageBase64);
410
+        const msg: Message = {
411
+          id: generateId(),
412
+          role: "assistant",
413
+          type: "image",
414
+          content: data.caption ?? "Screenshot",
415
+          imageBase64: data.imageBase64,
416
+          timestamp: Date.now(),
417
+          status: "sent",
418
+        };
419
+        if (data.sessionId) {
420
+          addMessageToSession(data.sessionId, msg);
421
+        } else {
422
+          addMessageToActive(msg);
423
+        }
424
+        if (!isCatchUp) notifyIncomingMessage("PAILot", data.caption ?? "New image");
425
+        break;
426
+      }
427
+      case "sessions": {
428
+        const incoming = data.sessions as WsSession[];
429
+        setSessions(incoming);
430
+        syncActiveFromSessions(incoming);
431
+        needsSync.current = false;
432
+        break;
433
+      }
434
+      case "session_switched": {
435
+        sendCommand("sessions");
436
+        break;
437
+      }
438
+      case "session_renamed": {
439
+        sendCommand("sessions");
440
+        break;
441
+      }
442
+      case "transcript": {
443
+        updateMessageContent(data.messageId, data.content);
444
+        break;
445
+      }
446
+      case "typing": {
447
+        const typingSession = (data.sessionId as string) || activeSessionIdRef.current || "_global";
448
+        typingMapRef.current[typingSession] = !!data.typing;
449
+        const activeTyping = typingMapRef.current[activeSessionIdRef.current ?? ""] ?? false;
450
+        setIsTyping(activeTyping);
451
+        break;
452
+      }
453
+      case "status": {
454
+        break;
455
+      }
456
+      case "projects": {
457
+        setProjects(data.projects ?? []);
458
+        break;
459
+      }
460
+      case "unread": {
461
+        const targetId = data.sessionId as string;
462
+        if (targetId && targetId !== activeSessionIdRef.current) {
463
+          setUnreadSessions((prev) => {
464
+            if (prev.has(targetId)) return prev;
465
+            const next = new Set(prev);
466
+            next.add(targetId);
467
+            return next;
468
+          });
469
+        }
470
+        break;
471
+      }
472
+      case "error": {
473
+        const errMsg: Message = {
474
+          id: generateId(),
475
+          role: "system",
476
+          type: "text",
477
+          content: data.message,
478
+          timestamp: Date.now(),
479
+        };
480
+        addMessageToActive(errMsg);
481
+        break;
482
+      }
483
+    }
484
+  }, [addMessageToActive, addMessageToSession, sendCommand, syncActiveFromSessions, updateMessageContent]);
485
+
312
486
   // Handle incoming WebSocket messages
313
487
   useEffect(() => {
314
488
     onMessageReceived.current = async (data: WsIncoming) => {
315
-      switch (data.type) {
316
-        case "text": {
317
-          setIsTyping(false);
318
-          const msg: Message = {
319
-            id: generateId(),
320
-            role: "assistant",
321
-            type: "text",
322
-            content: data.content,
323
-            timestamp: Date.now(),
324
-            status: "sent",
325
-          };
326
-          if (data.sessionId) {
327
-            addMessageToSession(data.sessionId, msg);
328
-          } else {
329
-            addMessageToActive(msg);
489
+      // Handle catch_up response: replay all missed messages
490
+      if (data.type === "catch_up") {
491
+        const messages = (data as any).messages as WsIncoming[];
492
+        const serverSeq = (data as any).serverSeq as number | undefined;
493
+        if (serverSeq) lastSeqRef.current = Math.max(lastSeqRef.current, serverSeq);
494
+        if (messages && messages.length > 0) {
495
+          for (const msg of messages) {
496
+            await processIncoming(msg, true);
330
497
           }
331
-          notifyIncomingMessage("PAILot", data.content ?? "New message");
332
-          break;
333
498
         }
334
-        case "voice": {
335
-          setIsTyping(false);
336
-          let audioUri: string | undefined;
337
-          if (data.audioBase64) {
338
-            try {
339
-              audioUri = await saveBase64Audio(data.audioBase64);
340
-            } catch {
341
-              // fallback: no playable audio
342
-            }
343
-          }
344
-          const msg: Message = {
345
-            id: generateId(),
346
-            role: "assistant",
347
-            type: "voice",
348
-            content: data.content ?? "",
349
-            audioUri,
350
-            timestamp: Date.now(),
351
-            status: "sent",
352
-          };
353
-          const isForActive = !data.sessionId || data.sessionId === activeSessionIdRef.current;
354
-          if (data.sessionId) {
355
-            addMessageToSession(data.sessionId, msg);
356
-          } else {
357
-            addMessageToActive(msg);
358
-          }
359
-          notifyIncomingMessage("PAILot", data.content ?? "Voice message");
360
-          // Only autoplay if this voice note is for the currently viewed session
361
-          if (msg.audioUri && canAutoplay() && isForActive) {
362
-            playAudio(msg.audioUri).catch(() => {});
363
-          }
364
-          break;
365
-        }
366
-        case "image": {
367
-          setLatestScreenshot(data.imageBase64);
368
-          const msg: Message = {
369
-            id: generateId(),
370
-            role: "assistant",
371
-            type: "image",
372
-            content: data.caption ?? "Screenshot",
373
-            imageBase64: data.imageBase64,
374
-            timestamp: Date.now(),
375
-            status: "sent",
376
-          };
377
-          if (data.sessionId) {
378
-            addMessageToSession(data.sessionId, msg);
379
-          } else {
380
-            addMessageToActive(msg);
381
-          }
382
-          notifyIncomingMessage("PAILot", data.caption ?? "New image");
383
-          break;
384
-        }
385
-        case "sessions": {
386
-          const incoming = data.sessions as WsSession[];
387
-          setSessions(incoming);
388
-          syncActiveFromSessions(incoming);
389
-          needsSync.current = false;
390
-          break;
391
-        }
392
-        case "session_switched": {
393
-          // Just refresh session list — no system message needed
394
-          sendCommand("sessions");
395
-          break;
396
-        }
397
-        case "session_renamed": {
398
-          // Just refresh session list — no system message needed
399
-          sendCommand("sessions");
400
-          break;
401
-        }
402
-        case "transcript": {
403
-          // Voice → text reflection: replace voice bubble with transcribed text
404
-          updateMessageContent(data.messageId, data.content);
405
-          break;
406
-        }
407
-        case "typing": {
408
-          const typingSession = (data.sessionId as string) || activeSessionIdRef.current || "_global";
409
-          typingMapRef.current[typingSession] = !!data.typing;
410
-          // Only show typing indicator if it's for the active session
411
-          const activeTyping = typingMapRef.current[activeSessionIdRef.current ?? ""] ?? false;
412
-          setIsTyping(activeTyping);
413
-          break;
414
-        }
415
-        case "status": {
416
-          // Connection status update — ignore for now
417
-          break;
418
-        }
419
-        case "projects": {
420
-          setProjects(data.projects ?? []);
421
-          break;
422
-        }
423
-        case "error": {
424
-          const msg: Message = {
425
-            id: generateId(),
426
-            role: "system",
427
-            type: "text",
428
-            content: data.message,
429
-            timestamp: Date.now(),
430
-          };
431
-          addMessageToActive(msg);
432
-          break;
433
-        }
499
+        return;
434
500
       }
501
+      // Live message — process normally
502
+      await processIncoming(data);
435
503
     };
436
504
 
437
505
     return () => {
438
506
       onMessageReceived.current = null;
439
507
     };
440
-  }, [onMessageReceived, sendCommand, addMessageToActive, updateMessageContent, syncActiveFromSessions]);
508
+  }, [onMessageReceived, processIncoming]);
441
509
 
442
510
   const sendTextMessage = useCallback(
443
511
     (text: string) => {
..
..
@@ -532,6 +600,13 @@
532
600
     (sessionId: string) => {
533
601
       // messagesMapRef is already kept in sync by all mutators — no need to save here
534
602
       sendCommand("switch", { sessionId });
603
+      // Clear the server-pushed unread indicator immediately on user intent
604
+      setUnreadSessions((prev) => {
605
+        if (!prev.has(sessionId)) return prev;
606
+        const next = new Set(prev);
607
+        next.delete(sessionId);
608
+        return next;
609
+      });
535
610
     },
536
611
     [sendCommand]
537
612
   );
..
..
@@ -552,6 +627,12 @@
552
627
         if (!u[sessionId]) return u;
553
628
         const next = { ...u };
554
629
         delete next[sessionId];
630
+        return next;
631
+      });
632
+      setUnreadSessions((prev) => {
633
+        if (!prev.has(sessionId)) return prev;
634
+        const next = new Set(prev);
635
+        next.delete(sessionId);
555
636
         return next;
556
637
       });
557
638
     },
..
..
@@ -622,6 +703,7 @@
622
703
         loadMoreMessages,
623
704
         hasMoreMessages,
624
705
         unreadCounts,
706
+        unreadSessions,
625
707
         incomingToast,
626
708
         dismissToast,
627
709
         latestScreenshot,