From e25bdba29f49b1b55a8a8cccdc4583aea3c101ed Mon Sep 17 00:00:00 2001
From: Matthias Nott <mnott@mnsoft.org>
Date: Sun, 15 Mar 2026 13:41:09 +0100
Subject: [PATCH] feat: multi-image upload and catch_up message delivery
---
contexts/ChatContext.tsx | 324 +++++++++++++++++++++++++++++++++--------------------
1 files changed, 203 insertions(+), 121 deletions(-)
diff --git a/contexts/ChatContext.tsx b/contexts/ChatContext.tsx
index 144f375..3297da9 100644
--- a/contexts/ChatContext.tsx
+++ b/contexts/ChatContext.tsx
@@ -6,6 +6,7 @@
useRef,
useState,
} from "react";
+import { AppState, AppStateStatus } from "react-native";
import { Message, WsIncoming, WsSession, PaiProject } from "../types";
import { useConnection } from "./ConnectionContext";
import { playAudio, encodeAudioToBase64, saveBase64Audio, canAutoplay } from "../services/audio";
@@ -142,6 +143,7 @@
loadMoreMessages: () => void;
hasMoreMessages: boolean;
unreadCounts: Record<string, number>;
+ unreadSessions: Set<string>;
incomingToast: IncomingToast | null;
dismissToast: () => void;
latestScreenshot: string | null;
@@ -158,12 +160,18 @@
const [latestScreenshot, setLatestScreenshot] = useState<string | null>(null);
const needsSync = useRef(true);
+ // Sequence tracking for catch_up protocol
+ const lastSeqRef = useRef(0);
+ const seenSeqsRef = useRef(new Set<number>());
+
// Per-session message storage
const messagesMapRef = useRef<Record<string, Message[]>>({});
// Messages for the active session (drives re-renders)
const [messages, setMessages] = useState<Message[]>([]);
// Unread counts for non-active sessions
const [unreadCounts, setUnreadCounts] = useState<Record<string, number>>({});
+ // Server-pushed unread indicators (sessions with new activity since last viewed)
+ const [unreadSessions, setUnreadSessions] = useState<Set<string>>(new Set());
// Per-session typing indicator (sessionId → boolean)
const typingMapRef = useRef<Record<string, boolean>>({});
const [isTyping, setIsTyping] = useState(false);
@@ -211,6 +219,12 @@
delete next[active.id];
return next;
});
+ setUnreadSessions((prev) => {
+ if (!prev.has(active.id)) return prev;
+ const next = new Set(prev);
+ next.delete(active.id);
+ return next;
+ });
// Sync typing indicator for the new active session
const activeTyping = typingMapRef.current[active.id] ?? false;
setIsTyping(activeTyping);
@@ -221,18 +235,34 @@
}
}, []);
- // On connect: ask gateway to sync sessions. If we already had a session
- // selected, tell the gateway so it preserves our selection instead of
- // jumping to whatever iTerm has focused on the Mac.
+ // On connect: ask gateway to sync sessions, then request catch_up for missed messages.
useEffect(() => {
if (status === "connected") {
needsSync.current = true;
const id = activeSessionIdRef.current;
sendCommand("sync", id ? { activeSessionId: id } : undefined);
+ // Request any messages we missed while disconnected/backgrounded
+ sendCommand("catch_up", { lastSeq: lastSeqRef.current });
} else if (status === "disconnected") {
setIsTyping(false);
}
// eslint-disable-next-line react-hooks/exhaustive-deps — only fire on status change
+ }, [status, sendCommand]);
+
+ // On foreground resume: request catch_up for any messages missed while backgrounded.
+ // iOS keeps the WebSocket "open" at TCP level but suspends the app — messages sent
+ // during that time are lost. catch_up replays them from the server's message log.
+ useEffect(() => {
+ let lastState: AppStateStatus = AppState.currentState;
+ const sub = AppState.addEventListener("change", (nextState) => {
+ if (lastState.match(/inactive|background/) && nextState === "active") {
+ if (status === "connected") {
+ sendCommand("catch_up", { lastSeq: lastSeqRef.current });
+ }
+ }
+ lastState = nextState;
+ });
+ return () => sub.remove();
}, [status, sendCommand]);
// Helper: add a message to the active session
@@ -309,135 +339,173 @@
});
}, []);
+ // Process a single incoming message (used by both live delivery and catch_up replay)
+ const processIncoming = useCallback(async (data: WsIncoming, isCatchUp = false) => {
+ // Dedup by seq: if we've seen this seq before, skip it
+ const seq = (data as any).seq as number | undefined;
+ if (seq) {
+ if (seenSeqsRef.current.has(seq)) return;
+ seenSeqsRef.current.add(seq);
+ lastSeqRef.current = Math.max(lastSeqRef.current, seq);
+ // Keep seen set bounded (last 500 seqs)
+ if (seenSeqsRef.current.size > 500) {
+ const arr = Array.from(seenSeqsRef.current).sort((a, b) => a - b);
+ seenSeqsRef.current = new Set(arr.slice(-300));
+ }
+ }
+
+ switch (data.type) {
+ case "text": {
+ if (!isCatchUp) setIsTyping(false);
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "text",
+ content: data.content,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ if (data.sessionId) {
+ addMessageToSession(data.sessionId, msg);
+ } else {
+ addMessageToActive(msg);
+ }
+ if (!isCatchUp) notifyIncomingMessage("PAILot", data.content ?? "New message");
+ break;
+ }
+ case "voice": {
+ if (!isCatchUp) setIsTyping(false);
+ let audioUri: string | undefined;
+ if (data.audioBase64) {
+ try {
+ audioUri = await saveBase64Audio(data.audioBase64);
+ } catch {
+ // fallback: no playable audio
+ }
+ }
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "voice",
+ content: data.content ?? "",
+ audioUri,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ const isForActive = !data.sessionId || data.sessionId === activeSessionIdRef.current;
+ if (data.sessionId) {
+ addMessageToSession(data.sessionId, msg);
+ } else {
+ addMessageToActive(msg);
+ }
+ if (!isCatchUp) notifyIncomingMessage("PAILot", data.content ?? "Voice message");
+ // Only autoplay if live (not catch_up) and for the currently viewed session
+ if (!isCatchUp && msg.audioUri && canAutoplay() && isForActive) {
+ playAudio(msg.audioUri).catch(() => {});
+ }
+ break;
+ }
+ case "image": {
+ setLatestScreenshot(data.imageBase64);
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "image",
+ content: data.caption ?? "Screenshot",
+ imageBase64: data.imageBase64,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ if (data.sessionId) {
+ addMessageToSession(data.sessionId, msg);
+ } else {
+ addMessageToActive(msg);
+ }
+ if (!isCatchUp) notifyIncomingMessage("PAILot", data.caption ?? "New image");
+ break;
+ }
+ case "sessions": {
+ const incoming = data.sessions as WsSession[];
+ setSessions(incoming);
+ syncActiveFromSessions(incoming);
+ needsSync.current = false;
+ break;
+ }
+ case "session_switched": {
+ sendCommand("sessions");
+ break;
+ }
+ case "session_renamed": {
+ sendCommand("sessions");
+ break;
+ }
+ case "transcript": {
+ updateMessageContent(data.messageId, data.content);
+ break;
+ }
+ case "typing": {
+ const typingSession = (data.sessionId as string) || activeSessionIdRef.current || "_global";
+ typingMapRef.current[typingSession] = !!data.typing;
+ const activeTyping = typingMapRef.current[activeSessionIdRef.current ?? ""] ?? false;
+ setIsTyping(activeTyping);
+ break;
+ }
+ case "status": {
+ break;
+ }
+ case "projects": {
+ setProjects(data.projects ?? []);
+ break;
+ }
+ case "unread": {
+ const targetId = data.sessionId as string;
+ if (targetId && targetId !== activeSessionIdRef.current) {
+ setUnreadSessions((prev) => {
+ if (prev.has(targetId)) return prev;
+ const next = new Set(prev);
+ next.add(targetId);
+ return next;
+ });
+ }
+ break;
+ }
+ case "error": {
+ const errMsg: Message = {
+ id: generateId(),
+ role: "system",
+ type: "text",
+ content: data.message,
+ timestamp: Date.now(),
+ };
+ addMessageToActive(errMsg);
+ break;
+ }
+ }
+ }, [addMessageToActive, addMessageToSession, sendCommand, syncActiveFromSessions, updateMessageContent]);
+
// Handle incoming WebSocket messages
useEffect(() => {
onMessageReceived.current = async (data: WsIncoming) => {
- switch (data.type) {
- case "text": {
- setIsTyping(false);
- const msg: Message = {
- id: generateId(),
- role: "assistant",
- type: "text",
- content: data.content,
- timestamp: Date.now(),
- status: "sent",
- };
- if (data.sessionId) {
- addMessageToSession(data.sessionId, msg);
- } else {
- addMessageToActive(msg);
+ // Handle catch_up response: replay all missed messages
+ if (data.type === "catch_up") {
+ const messages = (data as any).messages as WsIncoming[];
+ const serverSeq = (data as any).serverSeq as number | undefined;
+ if (serverSeq) lastSeqRef.current = Math.max(lastSeqRef.current, serverSeq);
+ if (messages && messages.length > 0) {
+ for (const msg of messages) {
+ await processIncoming(msg, true);
}
- notifyIncomingMessage("PAILot", data.content ?? "New message");
- break;
}
- case "voice": {
- setIsTyping(false);
- let audioUri: string | undefined;
- if (data.audioBase64) {
- try {
- audioUri = await saveBase64Audio(data.audioBase64);
- } catch {
- // fallback: no playable audio
- }
- }
- const msg: Message = {
- id: generateId(),
- role: "assistant",
- type: "voice",
- content: data.content ?? "",
- audioUri,
- timestamp: Date.now(),
- status: "sent",
- };
- const isForActive = !data.sessionId || data.sessionId === activeSessionIdRef.current;
- if (data.sessionId) {
- addMessageToSession(data.sessionId, msg);
- } else {
- addMessageToActive(msg);
- }
- notifyIncomingMessage("PAILot", data.content ?? "Voice message");
- // Only autoplay if this voice note is for the currently viewed session
- if (msg.audioUri && canAutoplay() && isForActive) {
- playAudio(msg.audioUri).catch(() => {});
- }
- break;
- }
- case "image": {
- setLatestScreenshot(data.imageBase64);
- const msg: Message = {
- id: generateId(),
- role: "assistant",
- type: "image",
- content: data.caption ?? "Screenshot",
- imageBase64: data.imageBase64,
- timestamp: Date.now(),
- status: "sent",
- };
- if (data.sessionId) {
- addMessageToSession(data.sessionId, msg);
- } else {
- addMessageToActive(msg);
- }
- notifyIncomingMessage("PAILot", data.caption ?? "New image");
- break;
- }
- case "sessions": {
- const incoming = data.sessions as WsSession[];
- setSessions(incoming);
- syncActiveFromSessions(incoming);
- needsSync.current = false;
- break;
- }
- case "session_switched": {
- // Just refresh session list — no system message needed
- sendCommand("sessions");
- break;
- }
- case "session_renamed": {
- // Just refresh session list — no system message needed
- sendCommand("sessions");
- break;
- }
- case "transcript": {
- // Voice → text reflection: replace voice bubble with transcribed text
- updateMessageContent(data.messageId, data.content);
- break;
- }
- case "typing": {
- const typingSession = (data.sessionId as string) || activeSessionIdRef.current || "_global";
- typingMapRef.current[typingSession] = !!data.typing;
- // Only show typing indicator if it's for the active session
- const activeTyping = typingMapRef.current[activeSessionIdRef.current ?? ""] ?? false;
- setIsTyping(activeTyping);
- break;
- }
- case "status": {
- // Connection status update — ignore for now
- break;
- }
- case "projects": {
- setProjects(data.projects ?? []);
- break;
- }
- case "error": {
- const msg: Message = {
- id: generateId(),
- role: "system",
- type: "text",
- content: data.message,
- timestamp: Date.now(),
- };
- addMessageToActive(msg);
- break;
- }
+ return;
}
+ // Live message — process normally
+ await processIncoming(data);
};
return () => {
onMessageReceived.current = null;
};
- }, [onMessageReceived, sendCommand, addMessageToActive, updateMessageContent, syncActiveFromSessions]);
+ }, [onMessageReceived, processIncoming]);
const sendTextMessage = useCallback(
(text: string) => {
@@ -532,6 +600,13 @@
(sessionId: string) => {
// messagesMapRef is already kept in sync by all mutators — no need to save here
sendCommand("switch", { sessionId });
+ // Clear the server-pushed unread indicator immediately on user intent
+ setUnreadSessions((prev) => {
+ if (!prev.has(sessionId)) return prev;
+ const next = new Set(prev);
+ next.delete(sessionId);
+ return next;
+ });
},
[sendCommand]
);
@@ -552,6 +627,12 @@
if (!u[sessionId]) return u;
const next = { ...u };
delete next[sessionId];
+ return next;
+ });
+ setUnreadSessions((prev) => {
+ if (!prev.has(sessionId)) return prev;
+ const next = new Set(prev);
+ next.delete(sessionId);
return next;
});
},
@@ -622,6 +703,7 @@
loadMoreMessages,
hasMoreMessages,
unreadCounts,
+ unreadSessions,
incomingToast,
dismissToast,
latestScreenshot,
--
Gitblit v1.3.1