From a0f39302919fbacf7a0d407f01b1a50413ea6f70 Mon Sep 17 00:00:00 2001
From: Matthias Nott <mnott@mnsoft.org>
Date: Mon, 02 Mar 2026 23:15:13 +0100
Subject: [PATCH] feat: on-device speech recognition, navigation screen, session picker
---
package-lock.json | 585 ++++++++++++--
services/audio.ts | 105 --
components/chat/VoiceButton.tsx | 179 +++-
app.json | 14
types/index.ts | 80 ++
components/chat/InputBar.tsx | 157 +++
components/chat/CommandBar.tsx | 146 ++-
contexts/ConnectionContext.tsx | 33
package.json | 21
app/chat.tsx | 182 +++-
app/navigate.tsx | 167 ++++
components/SessionPicker.tsx | 289 +++++++
components/chat/MessageBubble.tsx | 29
contexts/ChatContext.tsx | 198 ++++-
app/settings.tsx | 36
15 files changed, 1,791 insertions(+), 430 deletions(-)
diff --git a/app.json b/app.json
index 05c5664..1d0fbe4 100644
--- a/app.json
+++ b/app.json
@@ -18,7 +18,8 @@
"bundleIdentifier": "org.mnsoft.pailot",
"appleTeamId": "7KU642K5ZL",
"infoPlist": {
- "NSMicrophoneUsageDescription": "PAILot needs microphone access to record voice messages.",
+ "NSMicrophoneUsageDescription": "PAILot needs microphone access for voice input.",
+ "NSSpeechRecognitionUsageDescription": "PAILot uses speech recognition to convert your voice to text.",
"UIBackgroundModes": [
"audio"
]
@@ -43,9 +44,16 @@
"plugins": [
"expo-router",
[
- "expo-av",
+ "expo-audio",
{
- "microphonePermission": "PAILot needs microphone access to record voice messages."
+ "microphonePermission": "PAILot needs microphone access for voice input."
+ }
+ ],
+ [
+ "expo-speech-recognition",
+ {
+ "microphonePermission": "PAILot needs microphone access for voice input.",
+ "speechRecognitionPermission": "PAILot uses speech recognition to convert your voice to text."
}
],
"expo-secure-store"
diff --git a/app/chat.tsx b/app/chat.tsx
index cd0e2af..d773e98 100644
--- a/app/chat.tsx
+++ b/app/chat.tsx
@@ -1,4 +1,4 @@
-import React, { useCallback } from "react";
+import React, { useCallback, useState } from "react";
import { Pressable, Text, View } from "react-native";
import { SafeAreaView } from "react-native-safe-area-context";
import { router } from "expo-router";
@@ -6,69 +6,129 @@
import { useConnection } from "../contexts/ConnectionContext";
import { MessageList } from "../components/chat/MessageList";
import { InputBar } from "../components/chat/InputBar";
-import { CommandBar } from "../components/chat/CommandBar";
+import { CommandBar, TextModeCommandBar } from "../components/chat/CommandBar";
import { StatusDot } from "../components/ui/StatusDot";
+import { SessionPicker } from "../components/SessionPicker";
+import { playAudio } from "../services/audio";
export default function ChatScreen() {
- const { messages, sendTextMessage, sendVoiceMessage, clearMessages } =
+ const { messages, sendTextMessage, clearMessages, requestScreenshot } =
useChat();
const { status } = useConnection();
+ const [isTextMode, setIsTextMode] = useState(false);
+ const [showSessions, setShowSessions] = useState(false);
- const handleCommand = useCallback(
- (command: string) => {
- if (command === "/clear") {
- clearMessages();
+ const handleSessions = useCallback(() => {
+ setShowSessions(true);
+ }, []);
+
+ const handleScreenshot = useCallback(() => {
+ requestScreenshot();
+ router.push("/navigate");
+ }, [requestScreenshot]);
+
+ const handleHelp = useCallback(() => {
+ sendTextMessage("/h");
+ }, [sendTextMessage]);
+
+ const handleNavigate = useCallback(() => {
+ router.push("/navigate");
+ }, []);
+
+ const handleClear = useCallback(() => {
+ clearMessages();
+ }, [clearMessages]);
+
+ const handleReplay = useCallback(() => {
+ for (let i = messages.length - 1; i >= 0; i--) {
+ const msg = messages[i];
+ if (msg.role === "assistant") {
+ if (msg.audioUri) {
+ playAudio(msg.audioUri).catch(() => {});
+ }
return;
}
- sendTextMessage(command);
- },
- [sendTextMessage, clearMessages]
- );
-
- const handleSendVoice = useCallback(
- (audioUri: string, durationMs: number) => {
- sendVoiceMessage(audioUri, durationMs);
- },
- [sendVoiceMessage]
- );
+ }
+ }, [messages]);
return (
- <SafeAreaView className="flex-1 bg-pai-bg" edges={["top", "bottom"]}>
+ <SafeAreaView style={{ flex: 1, backgroundColor: "#0A0A0F" }} edges={["top", "bottom"]}>
{/* Header */}
- <View className="flex-row items-center justify-between px-4 py-3 border-b border-pai-border">
- <Text className="text-pai-text text-xl font-bold tracking-tight">
- PAILot
- </Text>
- <View className="flex-row items-center gap-3">
- <StatusDot status={status} size={10} />
- <Text className="text-pai-text-secondary text-xs">
- {status === "connected"
- ? "Connected"
- : status === "connecting"
- ? "Connecting..."
- : "Offline"}
- </Text>
- <Pressable
- onPress={() => router.push("/settings")}
- className="w-9 h-9 items-center justify-center rounded-full bg-pai-bg-tertiary"
- hitSlop={{ top: 4, bottom: 4, left: 4, right: 4 }}
+ <View
+ style={{
+ flexDirection: "row",
+ alignItems: "center",
+ justifyContent: "space-between",
+ paddingHorizontal: 16,
+ paddingVertical: 12,
+ borderBottomWidth: 1,
+ borderBottomColor: "#2E2E45",
+ }}
+ >
+ <View style={{ flexDirection: "row", alignItems: "center", gap: 10 }}>
+ <Text
+ style={{
+ color: "#E8E8F0",
+ fontSize: 22,
+ fontWeight: "800",
+ letterSpacing: -0.5,
+ }}
>
- <Text className="text-base">⚙️</Text>
- </Pressable>
+ PAILot
+ </Text>
+ <StatusDot status={status} size={8} />
</View>
+
+ <Pressable
+ onPress={() => router.push("/settings")}
+ hitSlop={{ top: 6, bottom: 6, left: 6, right: 6 }}
+ style={{
+ width: 36,
+ height: 36,
+ alignItems: "center",
+ justifyContent: "center",
+ borderRadius: 18,
+ backgroundColor: "#1E1E2E",
+ }}
+ >
+ <Text style={{ fontSize: 15 }}>⚙️</Text>
+ </Pressable>
</View>
{/* Message list */}
- <View className="flex-1">
+ <View style={{ flex: 1 }}>
{messages.length === 0 ? (
- <View className="flex-1 items-center justify-center gap-3">
- <Text className="text-5xl">🛩</Text>
- <Text className="text-pai-text text-xl font-semibold">
- PAILot
- </Text>
- <Text className="text-pai-text-muted text-sm text-center px-8">
- Voice-first AI communicator.{"\n"}Hold the mic button to talk.
- </Text>
+ <View style={{ flex: 1, alignItems: "center", justifyContent: "center", gap: 16 }}>
+ <View
+ style={{
+ width: 80,
+ height: 80,
+ borderRadius: 40,
+ backgroundColor: "#1E1E2E",
+ alignItems: "center",
+ justifyContent: "center",
+ borderWidth: 1,
+ borderColor: "#2E2E45",
+ }}
+ >
+ <Text style={{ fontSize: 36 }}>🛩</Text>
+ </View>
+ <View style={{ alignItems: "center", gap: 6 }}>
+ <Text style={{ color: "#E8E8F0", fontSize: 20, fontWeight: "700" }}>
+ PAILot
+ </Text>
+ <Text
+ style={{
+ color: "#5A5A78",
+ fontSize: 14,
+ textAlign: "center",
+ paddingHorizontal: 40,
+ lineHeight: 20,
+ }}
+ >
+ Voice-first AI communicator.{"\n"}Tap the mic to start talking.
+ </Text>
+ </View>
</View>
) : (
<MessageList messages={messages} />
@@ -76,10 +136,34 @@
</View>
{/* Command bar */}
- <CommandBar onCommand={handleCommand} />
+ {isTextMode ? (
+ <TextModeCommandBar
+ onSessions={handleSessions}
+ onScreenshot={handleScreenshot}
+ onNavigate={handleNavigate}
+ onClear={handleClear}
+ />
+ ) : (
+ <CommandBar
+ onSessions={handleSessions}
+ onScreenshot={handleScreenshot}
+ onHelp={handleHelp}
+ />
+ )}
{/* Input bar */}
- <InputBar onSendText={sendTextMessage} onSendVoice={handleSendVoice} />
+ <InputBar
+ onSendText={sendTextMessage}
+ onReplay={handleReplay}
+ isTextMode={isTextMode}
+ onToggleMode={() => setIsTextMode((v) => !v)}
+ />
+
+ {/* Session picker modal */}
+ <SessionPicker
+ visible={showSessions}
+ onClose={() => setShowSessions(false)}
+ />
</SafeAreaView>
);
}
diff --git a/app/navigate.tsx b/app/navigate.tsx
new file mode 100644
index 0000000..d8b11c7
--- /dev/null
+++ b/app/navigate.tsx
@@ -0,0 +1,167 @@
+import React, { useEffect } from "react";
+import { Image, Pressable, Text, View } from "react-native";
+import { SafeAreaView } from "react-native-safe-area-context";
+import { router } from "expo-router";
+import * as Haptics from "expo-haptics";
+import { useChat } from "../contexts/ChatContext";
+
+interface NavButton {
+ label: string;
+ key: string;
+ icon?: string;
+ wide?: boolean;
+}
+
+const NAV_BUTTONS: NavButton[][] = [
+ [
+ { label: "Esc", key: "escape" },
+ { label: "Tab", key: "tab" },
+ { label: "Enter", key: "enter" },
+ { label: "Ctrl-C", key: "ctrl-c" },
+ ],
+ [
+ { label: "", key: "left", icon: "←" },
+ { label: "", key: "up", icon: "↑" },
+ { label: "", key: "down", icon: "↓" },
+ { label: "", key: "right", icon: "→" },
+ ],
+];
+
+export default function NavigateScreen() {
+ const { latestScreenshot, requestScreenshot, sendNavKey } = useChat();
+
+ // Request a screenshot when entering navigation mode
+ useEffect(() => {
+ requestScreenshot();
+ }, [requestScreenshot]);
+
+ function handleNavPress(key: string) {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ sendNavKey(key);
+ }
+
+ return (
+ <SafeAreaView style={{ flex: 1, backgroundColor: "#0A0A0F" }} edges={["top", "bottom"]}>
+ {/* Header */}
+ <View
+ style={{
+ flexDirection: "row",
+ alignItems: "center",
+ justifyContent: "space-between",
+ paddingHorizontal: 16,
+ paddingVertical: 10,
+ borderBottomWidth: 1,
+ borderBottomColor: "#2E2E45",
+ }}
+ >
+ <Pressable
+ onPress={() => router.back()}
+ hitSlop={{ top: 8, bottom: 8, left: 8, right: 8 }}
+ style={{
+ width: 36,
+ height: 36,
+ alignItems: "center",
+ justifyContent: "center",
+ borderRadius: 18,
+ backgroundColor: "#1E1E2E",
+ }}
+ >
+ <Text style={{ color: "#E8E8F0", fontSize: 16 }}>←</Text>
+ </Pressable>
+ <Text style={{ color: "#E8E8F0", fontSize: 18, fontWeight: "700" }}>
+ Navigate
+ </Text>
+ <Pressable
+ onPress={() => requestScreenshot()}
+ hitSlop={{ top: 8, bottom: 8, left: 8, right: 8 }}
+ style={{
+ paddingHorizontal: 12,
+ paddingVertical: 8,
+ borderRadius: 12,
+ backgroundColor: "#1E1E2E",
+ }}
+ >
+ <Text style={{ color: "#4A9EFF", fontSize: 14, fontWeight: "600" }}>
+ Refresh
+ </Text>
+ </Pressable>
+ </View>
+
+ {/* Screenshot area */}
+ <View style={{ flex: 1, padding: 8 }}>
+ {latestScreenshot ? (
+ <Image
+ source={{ uri: `data:image/png;base64,${latestScreenshot}` }}
+ style={{
+ flex: 1,
+ borderRadius: 12,
+ backgroundColor: "#14141F",
+ }}
+ resizeMode="contain"
+ />
+ ) : (
+ <View
+ style={{
+ flex: 1,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: "#14141F",
+ borderRadius: 12,
+ }}
+ >
+ <Text style={{ color: "#5A5A78", fontSize: 16 }}>
+ Loading screenshot...
+ </Text>
+ </View>
+ )}
+ </View>
+
+ {/* Navigation buttons */}
+ <View
+ style={{
+ paddingHorizontal: 12,
+ paddingBottom: 8,
+ gap: 8,
+ }}
+ >
+ {NAV_BUTTONS.map((row, rowIdx) => (
+ <View
+ key={rowIdx}
+ style={{
+ flexDirection: "row",
+ gap: 8,
+ justifyContent: "center",
+ }}
+ >
+ {row.map((btn) => (
+ <Pressable
+ key={btn.key}
+ onPress={() => handleNavPress(btn.key)}
+ style={({ pressed }) => ({
+ flex: btn.wide ? 2 : 1,
+ height: 52,
+ borderRadius: 14,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: pressed ? "#4A9EFF" : "#1E1E2E",
+ borderWidth: 1,
+ borderColor: pressed ? "#4A9EFF" : "#2E2E45",
+ })}
+ >
+ <Text
+ style={{
+ color: "#E8E8F0",
+ fontSize: btn.icon ? 22 : 15,
+ fontWeight: "700",
+ }}
+ >
+ {btn.icon ?? btn.label}
+ </Text>
+ </Pressable>
+ ))}
+ </View>
+ ))}
+ </View>
+ </SafeAreaView>
+ );
+}
diff --git a/app/settings.tsx b/app/settings.tsx
index 76284db..3b36869 100644
--- a/app/settings.tsx
+++ b/app/settings.tsx
@@ -20,9 +20,9 @@
const { serverConfig, status, connect, disconnect, saveServerConfig } =
useConnection();
- const [host, setHost] = useState(serverConfig?.host ?? "");
+ const [host, setHost] = useState(serverConfig?.host ?? "192.168.1.100");
const [port, setPort] = useState(
- serverConfig?.port ? String(serverConfig.port) : ""
+ serverConfig?.port ? String(serverConfig.port) : "8765"
);
const [saved, setSaved] = useState(false);
@@ -63,14 +63,34 @@
keyboardShouldPersistTaps="handled"
>
{/* Header */}
- <View className="flex-row items-center px-4 py-3 border-b border-pai-border">
+ <View
+ style={{
+ flexDirection: "row",
+ alignItems: "center",
+ paddingHorizontal: 16,
+ paddingVertical: 12,
+ borderBottomWidth: 1,
+ borderBottomColor: "#2E2E45",
+ }}
+ >
<Pressable
onPress={() => router.back()}
- className="w-9 h-9 items-center justify-center rounded-full bg-pai-bg-tertiary mr-3"
+ hitSlop={{ top: 6, bottom: 6, left: 6, right: 6 }}
+ style={{
+ width: 36,
+ height: 36,
+ alignItems: "center",
+ justifyContent: "center",
+ borderRadius: 18,
+ backgroundColor: "#1E1E2E",
+ marginRight: 12,
+ }}
>
- <Text className="text-pai-text text-base">←</Text>
+ <Text style={{ color: "#E8E8F0", fontSize: 16 }}>←</Text>
</Pressable>
- <Text className="text-pai-text text-xl font-bold">Settings</Text>
+ <Text style={{ color: "#E8E8F0", fontSize: 22, fontWeight: "800", letterSpacing: -0.5 }}>
+ Settings
+ </Text>
</View>
<View className="px-4 mt-6">
@@ -115,7 +135,7 @@
autoCapitalize="none"
autoCorrect={false}
keyboardType="url"
- className="text-pai-text text-base"
+ style={{ color: "#E8E8F0", fontSize: 16, padding: 0 }}
/>
</View>
@@ -130,7 +150,7 @@
placeholder="8765"
placeholderTextColor="#5A5A78"
keyboardType="number-pad"
- className="text-pai-text text-base"
+ style={{ color: "#E8E8F0", fontSize: 16, padding: 0 }}
/>
</View>
</View>
diff --git a/components/SessionPicker.tsx b/components/SessionPicker.tsx
new file mode 100644
index 0000000..4d19164
--- /dev/null
+++ b/components/SessionPicker.tsx
@@ -0,0 +1,289 @@
+import React, { useCallback, useEffect, useState } from "react";
+import {
+ Modal,
+ Pressable,
+ ScrollView,
+ Text,
+ TextInput,
+ View,
+} from "react-native";
+import * as Haptics from "expo-haptics";
+import { WsSession } from "../types";
+import { useChat } from "../contexts/ChatContext";
+
+interface SessionPickerProps {
+ visible: boolean;
+ onClose: () => void;
+}
+
+export function SessionPicker({ visible, onClose }: SessionPickerProps) {
+ const { sessions, requestSessions, switchSession, renameSession } = useChat();
+ const [editingId, setEditingId] = useState<string | null>(null);
+ const [editName, setEditName] = useState("");
+
+ useEffect(() => {
+ if (visible) {
+ requestSessions();
+ }
+ }, [visible, requestSessions]);
+
+ const handleSwitch = useCallback(
+ (session: WsSession) => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
+ switchSession(session.id);
+ onClose();
+ },
+ [switchSession, onClose]
+ );
+
+ const handleStartRename = useCallback((session: WsSession) => {
+ setEditingId(session.id);
+ setEditName(session.name);
+ }, []);
+
+ const handleConfirmRename = useCallback(() => {
+ if (editingId && editName.trim()) {
+ renameSession(editingId, editName.trim());
+ }
+ setEditingId(null);
+ setEditName("");
+ }, [editingId, editName, renameSession]);
+
+ return (
+ <Modal
+ visible={visible}
+ animationType="slide"
+ transparent
+ onRequestClose={onClose}
+ >
+ <View
+ style={{
+ flex: 1,
+ backgroundColor: "rgba(0,0,0,0.6)",
+ justifyContent: "flex-end",
+ }}
+ >
+ <Pressable
+ style={{ flex: 1 }}
+ onPress={onClose}
+ />
+ <View
+ style={{
+ backgroundColor: "#14141F",
+ borderTopLeftRadius: 24,
+ borderTopRightRadius: 24,
+ maxHeight: "70%",
+ paddingBottom: 40,
+ }}
+ >
+ {/* Handle bar */}
+ <View style={{ alignItems: "center", paddingTop: 12, paddingBottom: 8 }}>
+ <View
+ style={{
+ width: 40,
+ height: 4,
+ borderRadius: 2,
+ backgroundColor: "#2E2E45",
+ }}
+ />
+ </View>
+
+ {/* Header */}
+ <View
+ style={{
+ flexDirection: "row",
+ alignItems: "center",
+ justifyContent: "space-between",
+ paddingHorizontal: 20,
+ paddingBottom: 16,
+ }}
+ >
+ <Text
+ style={{
+ color: "#E8E8F0",
+ fontSize: 20,
+ fontWeight: "700",
+ }}
+ >
+ Sessions
+ </Text>
+ <Pressable
+ onPress={() => requestSessions()}
+ hitSlop={{ top: 8, bottom: 8, left: 8, right: 8 }}
+ style={{
+ paddingHorizontal: 12,
+ paddingVertical: 6,
+ borderRadius: 12,
+ backgroundColor: "#1E1E2E",
+ }}
+ >
+ <Text style={{ color: "#9898B0", fontSize: 13 }}>Refresh</Text>
+ </Pressable>
+ </View>
+
+ {/* Session list */}
+ <ScrollView
+ style={{ paddingHorizontal: 16 }}
+ showsVerticalScrollIndicator={false}
+ >
+ {sessions.length === 0 ? (
+ <View style={{ alignItems: "center", paddingVertical: 32 }}>
+ <Text style={{ color: "#5A5A78", fontSize: 15 }}>
+ No sessions found
+ </Text>
+ </View>
+ ) : (
+ sessions.map((session) => (
+ <View key={session.id} style={{ marginBottom: 8 }}>
+ {editingId === session.id ? (
+ /* Rename mode */
+ <View
+ style={{
+ backgroundColor: "#1E1E2E",
+ borderRadius: 16,
+ padding: 16,
+ borderWidth: 2,
+ borderColor: "#4A9EFF",
+ }}
+ >
+ <TextInput
+ value={editName}
+ onChangeText={setEditName}
+ autoFocus
+ onSubmitEditing={handleConfirmRename}
+ returnKeyType="done"
+ style={{
+ color: "#E8E8F0",
+ fontSize: 17,
+ fontWeight: "600",
+ padding: 0,
+ marginBottom: 12,
+ }}
+ placeholderTextColor="#5A5A78"
+ placeholder="Session name..."
+ />
+ <View style={{ flexDirection: "row", gap: 8 }}>
+ <Pressable
+ onPress={handleConfirmRename}
+ style={{
+ flex: 1,
+ backgroundColor: "#4A9EFF",
+ borderRadius: 10,
+ paddingVertical: 10,
+ alignItems: "center",
+ }}
+ >
+ <Text style={{ color: "#FFF", fontSize: 15, fontWeight: "600" }}>
+ Save
+ </Text>
+ </Pressable>
+ <Pressable
+ onPress={() => setEditingId(null)}
+ style={{
+ flex: 1,
+ backgroundColor: "#252538",
+ borderRadius: 10,
+ paddingVertical: 10,
+ alignItems: "center",
+ }}
+ >
+ <Text style={{ color: "#9898B0", fontSize: 15 }}>Cancel</Text>
+ </Pressable>
+ </View>
+ </View>
+ ) : (
+ /* Normal session row */
+ <Pressable
+ onPress={() => handleSwitch(session)}
+ onLongPress={() => handleStartRename(session)}
+ style={({ pressed }) => ({
+ backgroundColor: pressed ? "#252538" : "#1E1E2E",
+ borderRadius: 16,
+ padding: 16,
+ flexDirection: "row",
+ alignItems: "center",
+ borderWidth: session.isActive ? 2 : 1,
+ borderColor: session.isActive ? "#4A9EFF" : "#2E2E45",
+ })}
+ >
+ {/* Number badge */}
+ <View
+ style={{
+ width: 36,
+ height: 36,
+ borderRadius: 18,
+ backgroundColor: session.isActive ? "#4A9EFF" : "#252538",
+ alignItems: "center",
+ justifyContent: "center",
+ marginRight: 14,
+ }}
+ >
+ <Text
+ style={{
+ color: session.isActive ? "#FFF" : "#9898B0",
+ fontSize: 16,
+ fontWeight: "700",
+ }}
+ >
+ {session.index}
+ </Text>
+ </View>
+
+ {/* Session info */}
+ <View style={{ flex: 1 }}>
+ <Text
+ style={{
+ color: "#E8E8F0",
+ fontSize: 17,
+ fontWeight: "600",
+ }}
+ numberOfLines={1}
+ >
+ {session.name}
+ </Text>
+ <Text
+ style={{
+ color: "#5A5A78",
+ fontSize: 12,
+ marginTop: 2,
+ }}
+ >
+ {session.type === "terminal" ? "Terminal" : "Claude"}
+ {session.isActive ? " — active" : ""}
+ </Text>
+ </View>
+
+ {/* Active indicator */}
+ {session.isActive && (
+ <View
+ style={{
+ width: 10,
+ height: 10,
+ borderRadius: 5,
+ backgroundColor: "#2ED573",
+ }}
+ />
+ )}
+ </Pressable>
+ )}
+ </View>
+ ))
+ )}
+
+ {/* Hint */}
+ <Text
+ style={{
+ color: "#5A5A78",
+ fontSize: 12,
+ textAlign: "center",
+ paddingVertical: 12,
+ }}
+ >
+ Tap to switch — Long press to rename
+ </Text>
+ </ScrollView>
+ </View>
+ </View>
+ </Modal>
+ );
+}
diff --git a/components/chat/CommandBar.tsx b/components/chat/CommandBar.tsx
index 9853d8f..de9c0f4 100644
--- a/components/chat/CommandBar.tsx
+++ b/components/chat/CommandBar.tsx
@@ -1,67 +1,103 @@
import React, { useState } from "react";
-import { Pressable, ScrollView, Text, View } from "react-native";
-
-interface Command {
- label: string;
- value: string;
-}
-
-const DEFAULT_COMMANDS: Command[] = [
- { label: "/s", value: "/s" },
- { label: "/ss", value: "/ss" },
- { label: "/clear", value: "/clear" },
- { label: "/help", value: "/help" },
- { label: "/status", value: "/status" },
-];
+import { Pressable, Text, View, useWindowDimensions } from "react-native";
+import * as Haptics from "expo-haptics";
interface CommandBarProps {
- onCommand: (command: string) => void;
- commands?: Command[];
+ onSessions: () => void;
+ onScreenshot: () => void;
+ onHelp: () => void;
}
-export function CommandBar({
- onCommand,
- commands = DEFAULT_COMMANDS,
-}: CommandBarProps) {
- const [activeCommand, setActiveCommand] = useState<string | null>(null);
+export function CommandBar({ onSessions, onScreenshot, onHelp }: CommandBarProps) {
+ return (
+ <View
+ style={{
+ flexDirection: "row",
+ paddingHorizontal: 12,
+ paddingVertical: 6,
+ gap: 8,
+ }}
+ >
+ <CmdBtn icon="📋" label="Sessions" bg="#1A2744" border="#2E4A7A" onPress={onSessions} />
+ <CmdBtn icon="📸" label="Screen" bg="#1A3A2A" border="#2E6A4A" onPress={onScreenshot} />
+ <CmdBtn icon="❓" label="Help" bg="#3A1A2A" border="#6A2E4A" onPress={onHelp} />
+ </View>
+ );
+}
- function handlePress(command: Command) {
- setActiveCommand(command.value);
- onCommand(command.value);
- setTimeout(() => setActiveCommand(null), 200);
- }
+interface TextModeCommandBarProps {
+ onSessions: () => void;
+ onScreenshot: () => void;
+ onNavigate: () => void;
+ onClear: () => void;
+}
+
+export function TextModeCommandBar({
+ onSessions,
+ onScreenshot,
+ onNavigate,
+ onClear,
+}: TextModeCommandBarProps) {
+ return (
+ <View
+ style={{
+ flexDirection: "row",
+ paddingHorizontal: 12,
+ paddingVertical: 6,
+ gap: 8,
+ }}
+ >
+ <CmdBtn icon="📋" label="Sessions" bg="#1A2744" border="#2E4A7A" onPress={onSessions} />
+ <CmdBtn icon="📸" label="Screen" bg="#1A3A2A" border="#2E6A4A" onPress={onScreenshot} />
+ <CmdBtn icon="🧭" label="Navigate" bg="#2A2A1A" border="#5A5A2E" onPress={onNavigate} />
+ <CmdBtn icon="🗑" label="Clear" bg="#3A1A1A" border="#6A2E2E" onPress={onClear} />
+ </View>
+ );
+}
+
+function CmdBtn({
+ icon,
+ label,
+ bg,
+ border,
+ onPress,
+}: {
+ icon: string;
+ label: string;
+ bg: string;
+ border: string;
+ onPress: () => void;
+}) {
+ const [pressed, setPressed] = useState(false);
+ const { width } = useWindowDimensions();
return (
- <View className="border-t border-pai-border">
- <ScrollView
- horizontal
- showsHorizontalScrollIndicator={false}
- contentContainerStyle={{ paddingHorizontal: 12, paddingVertical: 8, gap: 8 }}
+ <View style={{ flex: 1 }}>
+ <Pressable
+ onPressIn={() => setPressed(true)}
+ onPressOut={() => setPressed(false)}
+ onPress={() => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ onPress();
+ }}
>
- {commands.map((cmd) => (
- <Pressable
- key={cmd.value}
- onPress={() => handlePress(cmd)}
- className="rounded-full px-4 py-2"
- style={({ pressed }) => ({
- backgroundColor:
- activeCommand === cmd.value || pressed
- ? "#4A9EFF"
- : "#1E1E2E",
- })}
- >
- <Text
- className="text-sm font-medium"
- style={{
- color:
- activeCommand === cmd.value ? "#FFFFFF" : "#9898B0",
- }}
- >
- {cmd.label}
- </Text>
- </Pressable>
- ))}
- </ScrollView>
+ <View
+ style={{
+ height: 68,
+ borderRadius: 16,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: pressed ? "#4A9EFF" : bg,
+ borderWidth: 1.5,
+ borderColor: pressed ? "#4A9EFF" : border,
+ }}
+ >
+ <Text style={{ fontSize: 26, marginBottom: 2 }}>{icon}</Text>
+ <Text style={{ color: "#C8C8E0", fontSize: 13, fontWeight: "700" }}>
+ {label}
+ </Text>
+ </View>
+ </Pressable>
</View>
);
}
diff --git a/components/chat/InputBar.tsx b/components/chat/InputBar.tsx
index 71a576a..6f1bc35 100644
--- a/components/chat/InputBar.tsx
+++ b/components/chat/InputBar.tsx
@@ -6,16 +6,23 @@
TextInput,
View,
} from "react-native";
+import * as Haptics from "expo-haptics";
import { VoiceButton } from "./VoiceButton";
interface InputBarProps {
onSendText: (text: string) => void;
- onSendVoice: (audioUri: string, durationMs: number) => void;
+ onReplay: () => void;
+ isTextMode: boolean;
+ onToggleMode: () => void;
}
-export function InputBar({ onSendText, onSendVoice }: InputBarProps) {
+export function InputBar({
+ onSendText,
+ onReplay,
+ isTextMode,
+ onToggleMode,
+}: InputBarProps) {
const [text, setText] = useState("");
- const [isVoiceMode, setIsVoiceMode] = useState(false);
const inputRef = useRef<TextInput>(null);
const handleSend = useCallback(() => {
@@ -25,42 +32,108 @@
setText("");
}, [text, onSendText]);
- const toggleMode = useCallback(() => {
- setIsVoiceMode((prev) => {
- if (prev) {
- // Switching to text mode — focus input after mode switch
- setTimeout(() => inputRef.current?.focus(), 100);
- } else {
- Keyboard.dismiss();
- }
- return !prev;
- });
- }, []);
-
- if (isVoiceMode) {
+ if (!isTextMode) {
+ // Voice mode: [Replay] [Talk] [Aa]
return (
- <View className="border-t border-pai-border bg-pai-bg">
- {/* Mode toggle */}
+ <View
+ style={{
+ flexDirection: "row",
+ gap: 10,
+ paddingHorizontal: 16,
+ paddingVertical: 10,
+ paddingBottom: 6,
+ borderTopWidth: 1,
+ borderTopColor: "#2E2E45",
+ alignItems: "center",
+ }}
+ >
+ {/* Replay last message */}
<Pressable
- onPress={toggleMode}
- className="absolute top-3 right-4 z-10 w-10 h-10 items-center justify-center"
+ onPress={() => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ onReplay();
+ }}
>
- <Text className="text-2xl">⌨️</Text>
+ <View
+ style={{
+ width: 68,
+ height: 68,
+ borderRadius: 34,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: "#1A2E1A",
+ borderWidth: 1.5,
+ borderColor: "#3A6A3A",
+ }}
+ >
+ <Text style={{ fontSize: 24 }}>▶</Text>
+ <Text style={{ color: "#8ABF8A", fontSize: 10, marginTop: 1, fontWeight: "600" }}>Replay</Text>
+ </View>
</Pressable>
- <VoiceButton onVoiceMessage={onSendVoice} />
+ {/* Talk button — center, biggest */}
+ <View style={{ flex: 1, alignItems: "center" }}>
+ <VoiceButton onTranscript={onSendText} />
+ </View>
+
+ {/* Text mode toggle */}
+ <Pressable
+ onPress={() => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ onToggleMode();
+ setTimeout(() => inputRef.current?.focus(), 150);
+ }}
+ >
+ <View
+ style={{
+ width: 68,
+ height: 68,
+ borderRadius: 34,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: "#1A1A3E",
+ borderWidth: 1.5,
+ borderColor: "#3A3A7A",
+ }}
+ >
+ <Text style={{ fontSize: 22, color: "#9898D0", fontWeight: "700" }}>Aa</Text>
+ </View>
+ </Pressable>
</View>
);
}
+ // Text mode: [Mic] [TextInput] [Send]
return (
- <View className="border-t border-pai-border bg-pai-bg px-3 py-2 flex-row items-end gap-2">
+ <View
+ style={{
+ flexDirection: "row",
+ gap: 8,
+ paddingHorizontal: 12,
+ paddingVertical: 8,
+ borderTopWidth: 1,
+ borderTopColor: "#2E2E45",
+ alignItems: "flex-end",
+ }}
+ >
{/* Voice mode toggle */}
<Pressable
- onPress={toggleMode}
- className="w-10 h-10 items-center justify-center rounded-full bg-pai-bg-tertiary mb-0.5"
+ onPress={() => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ Keyboard.dismiss();
+ onToggleMode();
+ }}
+ style={{
+ width: 40,
+ height: 40,
+ borderRadius: 20,
+ alignItems: "center",
+ justifyContent: "center",
+ backgroundColor: "#1E1E2E",
+ marginBottom: 2,
+ }}
>
- <Text className="text-xl">🎤</Text>
+ <Text style={{ fontSize: 20 }}>🎤</Text>
</Pressable>
{/* Text input */}
@@ -75,19 +148,39 @@
onSubmitEditing={handleSend}
returnKeyType="send"
blurOnSubmit
- className="flex-1 bg-pai-bg-tertiary rounded-2xl px-4 py-2.5 text-pai-text text-base"
- style={{ maxHeight: 120 }}
+ style={{
+ flex: 1,
+ backgroundColor: "#1E1E2E",
+ borderRadius: 20,
+ paddingHorizontal: 16,
+ paddingVertical: 10,
+ maxHeight: 120,
+ color: "#E8E8F0",
+ fontSize: 16,
+ }}
/>
{/* Send button */}
<Pressable
onPress={handleSend}
disabled={!text.trim()}
- className={`w-10 h-10 rounded-full items-center justify-center mb-0.5 ${
- text.trim() ? "bg-pai-accent" : "bg-pai-bg-tertiary"
- }`}
+ style={{
+ width: 40,
+ height: 40,
+ borderRadius: 20,
+ alignItems: "center",
+ justifyContent: "center",
+ marginBottom: 2,
+ backgroundColor: text.trim() ? "#4A9EFF" : "#1E1E2E",
+ }}
>
- <Text className={`text-xl ${text.trim() ? "text-white" : "text-pai-text-muted"}`}>
+ <Text
+ style={{
+ fontSize: 18,
+ fontWeight: "bold",
+ color: text.trim() ? "#FFFFFF" : "#5A5A78",
+ }}
+ >
↑
</Text>
</Pressable>
diff --git a/components/chat/MessageBubble.tsx b/components/chat/MessageBubble.tsx
index e7fbbd8..fd8c0b6 100644
--- a/components/chat/MessageBubble.tsx
+++ b/components/chat/MessageBubble.tsx
@@ -1,5 +1,5 @@
import React, { useCallback, useState } from "react";
-import { Pressable, Text, View } from "react-native";
+import { Image, Pressable, Text, View } from "react-native";
import { Message } from "../../types";
import { playAudio, stopPlayback } from "../../services/audio";
@@ -57,7 +57,32 @@
: "bg-pai-surface rounded-tl-sm"
}`}
>
- {message.type === "voice" ? (
+ {message.type === "image" && message.imageBase64 ? (
+ /* Image message */
+ <View>
+ <Image
+ source={{ uri: `data:image/png;base64,${message.imageBase64}` }}
+ style={{
+ width: 260,
+ height: 180,
+ borderRadius: 10,
+ backgroundColor: "#14141F",
+ }}
+ resizeMode="contain"
+ />
+ {message.content ? (
+ <Text
+ style={{
+ color: isUser ? "#FFF" : "#9898B0",
+ fontSize: 12,
+ marginTop: 4,
+ }}
+ >
+ {message.content}
+ </Text>
+ ) : null}
+ </View>
+ ) : message.type === "voice" ? (
<Pressable
onPress={handleVoicePress}
className="flex-row items-center gap-3"
diff --git a/components/chat/VoiceButton.tsx b/components/chat/VoiceButton.tsx
index b86a370..9ebaa82 100644
--- a/components/chat/VoiceButton.tsx
+++ b/components/chat/VoiceButton.tsx
@@ -1,121 +1,192 @@
-import React, { useCallback, useRef, useState } from "react";
+import React, { useCallback, useEffect, useRef, useState } from "react";
import { Animated, Pressable, Text, View } from "react-native";
import * as Haptics from "expo-haptics";
-import { startRecording, stopRecording } from "../../services/audio";
-import { Audio } from "expo-av";
+import {
+ ExpoSpeechRecognitionModule,
+ useSpeechRecognitionEvent,
+} from "expo-speech-recognition";
interface VoiceButtonProps {
- onVoiceMessage: (audioUri: string, durationMs: number) => void;
+ onTranscript: (text: string) => void;
}
-const VOICE_BUTTON_SIZE = 88;
+const VOICE_BUTTON_SIZE = 72;
-export function VoiceButton({ onVoiceMessage }: VoiceButtonProps) {
- const [isRecording, setIsRecording] = useState(false);
- const recordingRef = useRef<Audio.Recording | null>(null);
- const scaleAnim = useRef(new Animated.Value(1)).current;
+/**
+ * Tap-to-toggle voice button using on-device speech recognition.
+ * - Tap once: start listening
+ * - Tap again: stop and send transcript
+ * - Long-press while listening: cancel (discard)
+ */
+export function VoiceButton({ onTranscript }: VoiceButtonProps) {
+ const [isListening, setIsListening] = useState(false);
+ const [transcript, setTranscript] = useState("");
const pulseAnim = useRef(new Animated.Value(1)).current;
+ const glowAnim = useRef(new Animated.Value(0)).current;
const pulseLoop = useRef<Animated.CompositeAnimation | null>(null);
+ const cancelledRef = useRef(false);
+
+ // Speech recognition events
+ useSpeechRecognitionEvent("start", () => {
+ setIsListening(true);
+ });
+
+ useSpeechRecognitionEvent("end", () => {
+ setIsListening(false);
+ stopPulse();
+
+ // Send transcript if we have one and weren't cancelled
+ if (!cancelledRef.current && transcript.trim()) {
+ onTranscript(transcript.trim());
+ }
+ setTranscript("");
+ cancelledRef.current = false;
+ });
+
+ useSpeechRecognitionEvent("result", (event) => {
+ const text = event.results[0]?.transcript ?? "";
+ setTranscript(text);
+ });
+
+ useSpeechRecognitionEvent("error", (event) => {
+ console.error("Speech recognition error:", event.error, event.message);
+ setIsListening(false);
+ stopPulse();
+ setTranscript("");
+ });
const startPulse = useCallback(() => {
pulseLoop.current = Animated.loop(
Animated.sequence([
Animated.timing(pulseAnim, {
toValue: 1.15,
- duration: 600,
+ duration: 700,
useNativeDriver: true,
}),
Animated.timing(pulseAnim, {
toValue: 1,
- duration: 600,
+ duration: 700,
useNativeDriver: true,
}),
])
);
pulseLoop.current.start();
- }, [pulseAnim]);
+ Animated.timing(glowAnim, {
+ toValue: 1,
+ duration: 300,
+ useNativeDriver: true,
+ }).start();
+ }, [pulseAnim, glowAnim]);
const stopPulse = useCallback(() => {
pulseLoop.current?.stop();
pulseAnim.setValue(1);
- }, [pulseAnim]);
-
- const handlePressIn = useCallback(async () => {
- Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
-
- Animated.spring(scaleAnim, {
- toValue: 0.92,
+ Animated.timing(glowAnim, {
+ toValue: 0,
+ duration: 200,
useNativeDriver: true,
}).start();
+ }, [pulseAnim, glowAnim]);
- const recording = await startRecording();
- if (recording) {
- recordingRef.current = recording;
- setIsRecording(true);
- startPulse();
- }
- }, [scaleAnim, startPulse]);
+ const startListening = useCallback(async () => {
+ const result = await ExpoSpeechRecognitionModule.requestPermissionsAsync();
+ if (!result.granted) return;
- const handlePressOut = useCallback(async () => {
- Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
+ cancelledRef.current = false;
+ setTranscript("");
+ startPulse();
- Animated.spring(scaleAnim, {
- toValue: 1,
- useNativeDriver: true,
- }).start();
+ ExpoSpeechRecognitionModule.start({
+ lang: "en-US",
+ interimResults: true,
+ continuous: true,
+ });
+ }, [startPulse]);
+ const stopAndSend = useCallback(() => {
stopPulse();
- setIsRecording(false);
+ cancelledRef.current = false;
+ ExpoSpeechRecognitionModule.stop();
+ }, [stopPulse]);
- if (recordingRef.current) {
- const result = await stopRecording();
- recordingRef.current = null;
+ const cancelListening = useCallback(() => {
+ Haptics.notificationAsync(Haptics.NotificationFeedbackType.Warning);
+ stopPulse();
+ cancelledRef.current = true;
+ setTranscript("");
+ ExpoSpeechRecognitionModule.abort();
+ }, [stopPulse]);
- if (result && result.durationMs > 500) {
- onVoiceMessage(result.uri, result.durationMs);
- }
+ const handleTap = useCallback(async () => {
+ Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium);
+ if (isListening) {
+ stopAndSend();
+ } else {
+ await startListening();
}
- }, [scaleAnim, stopPulse, onVoiceMessage]);
+ }, [isListening, stopAndSend, startListening]);
+
+ const handleLongPress = useCallback(() => {
+ if (isListening) {
+ cancelListening();
+ }
+ }, [isListening, cancelListening]);
return (
- <View className="items-center justify-center py-4">
- {/* Pulse ring — only visible while recording */}
+ <View style={{ alignItems: "center", justifyContent: "center" }}>
+ {/* Outer pulse ring */}
<Animated.View
style={{
position: "absolute",
width: VOICE_BUTTON_SIZE + 24,
height: VOICE_BUTTON_SIZE + 24,
borderRadius: (VOICE_BUTTON_SIZE + 24) / 2,
- backgroundColor: isRecording ? "rgba(255, 159, 67, 0.15)" : "transparent",
+ backgroundColor: isListening ? "rgba(255, 159, 67, 0.12)" : "transparent",
transform: [{ scale: pulseAnim }],
+ opacity: glowAnim,
}}
/>
{/* Button */}
- <Animated.View style={{ transform: [{ scale: scaleAnim }] }}>
- <Pressable
- onPressIn={handlePressIn}
- onPressOut={handlePressOut}
+ <Pressable
+ onPress={handleTap}
+ onLongPress={handleLongPress}
+ delayLongPress={600}
+ >
+ <View
style={{
width: VOICE_BUTTON_SIZE,
height: VOICE_BUTTON_SIZE,
borderRadius: VOICE_BUTTON_SIZE / 2,
- backgroundColor: isRecording ? "#FF9F43" : "#4A9EFF",
+ backgroundColor: isListening ? "#FF9F43" : "#4A9EFF",
alignItems: "center",
justifyContent: "center",
- shadowColor: isRecording ? "#FF9F43" : "#4A9EFF",
+ shadowColor: isListening ? "#FF9F43" : "#4A9EFF",
shadowOffset: { width: 0, height: 4 },
shadowOpacity: 0.4,
shadowRadius: 12,
elevation: 8,
}}
>
- <Text style={{ fontSize: 32 }}>{isRecording ? "🎙" : "🎤"}</Text>
- </Pressable>
- </Animated.View>
+ <Text style={{ fontSize: 28 }}>{isListening ? "⏹" : "🎤"}</Text>
+ </View>
+ </Pressable>
- <Text className="text-pai-text-muted text-xs mt-3">
- {isRecording ? "Release to send" : "Hold to talk"}
+ {/* Label / transcript preview */}
+ <Text
+ style={{
+ color: isListening ? "#FF9F43" : "#5A5A78",
+ fontSize: 11,
+ marginTop: 4,
+ fontWeight: isListening ? "600" : "400",
+ maxWidth: 200,
+ textAlign: "center",
+ }}
+ numberOfLines={2}
+ >
+ {isListening
+ ? transcript || "Listening..."
+ : "Tap to talk"}
</Text>
</View>
);
diff --git a/contexts/ChatContext.tsx b/contexts/ChatContext.tsx
index a6c1ef9..a0b62fc 100644
--- a/contexts/ChatContext.tsx
+++ b/contexts/ChatContext.tsx
@@ -6,9 +6,9 @@
useRef,
useState,
} from "react";
-import { Message, WebSocketMessage } from "../types";
+import { Message, WsIncoming, WsSession } from "../types";
import { useConnection } from "./ConnectionContext";
-import { playAudio } from "../services/audio";
+import { playAudio, encodeAudioToBase64 } from "../services/audio";
function generateId(): string {
return Date.now().toString(36) + Math.random().toString(36).slice(2);
@@ -19,13 +19,29 @@
sendTextMessage: (text: string) => void;
sendVoiceMessage: (audioUri: string, durationMs?: number) => void;
clearMessages: () => void;
+ // Session management
+ sessions: WsSession[];
+ requestSessions: () => void;
+ switchSession: (sessionId: string) => void;
+ renameSession: (sessionId: string, name: string) => void;
+ // Screenshot / navigation
+ latestScreenshot: string | null;
+ requestScreenshot: () => void;
+ sendNavKey: (key: string) => void;
}
const ChatContext = createContext<ChatContextValue | null>(null);
export function ChatProvider({ children }: { children: React.ReactNode }) {
const [messages, setMessages] = useState<Message[]>([]);
- const { sendTextMessage: wsSend, sendVoiceMessage: wsVoice, onMessageReceived } = useConnection();
+ const [sessions, setSessions] = useState<WsSession[]>([]);
+ const [latestScreenshot, setLatestScreenshot] = useState<string | null>(null);
+ const {
+ sendTextMessage: wsSend,
+ sendVoiceMessage: wsVoice,
+ sendCommand,
+ onMessageReceived,
+ } = useConnection();
const addMessage = useCallback((msg: Message) => {
setMessages((prev) => [...prev, msg]);
@@ -42,34 +58,92 @@
// Handle incoming WebSocket messages
useEffect(() => {
- onMessageReceived.current = (data: WebSocketMessage) => {
- if (data.type === "text") {
- const msg: Message = {
- id: generateId(),
- role: "assistant",
- type: "text",
- content: data.content,
- timestamp: Date.now(),
- status: "sent",
- };
- setMessages((prev) => [...prev, msg]);
- } else if (data.type === "voice") {
- const msg: Message = {
- id: generateId(),
- role: "assistant",
- type: "voice",
- content: data.content ?? "",
- audioUri: data.audioBase64
- ? `data:audio/mp4;base64,${data.audioBase64}`
- : undefined,
- timestamp: Date.now(),
- status: "sent",
- };
- setMessages((prev) => [...prev, msg]);
-
- // Auto-play incoming voice messages
- if (msg.audioUri) {
- playAudio(msg.audioUri).catch(() => {});
+ onMessageReceived.current = (data: WsIncoming) => {
+ switch (data.type) {
+ case "text": {
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "text",
+ content: data.content,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ setMessages((prev) => [...prev, msg]);
+ break;
+ }
+ case "voice": {
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "voice",
+ content: data.content ?? "",
+ audioUri: data.audioBase64
+ ? `data:audio/mp4;base64,${data.audioBase64}`
+ : undefined,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ setMessages((prev) => [...prev, msg]);
+ if (msg.audioUri) {
+ playAudio(msg.audioUri).catch(() => {});
+ }
+ break;
+ }
+ case "image": {
+ // Store as latest screenshot for navigation mode
+ setLatestScreenshot(data.imageBase64);
+ // Also add to chat as an image message
+ const msg: Message = {
+ id: generateId(),
+ role: "assistant",
+ type: "image",
+ content: data.caption ?? "Screenshot",
+ imageBase64: data.imageBase64,
+ timestamp: Date.now(),
+ status: "sent",
+ };
+ setMessages((prev) => [...prev, msg]);
+ break;
+ }
+ case "sessions": {
+ setSessions(data.sessions);
+ break;
+ }
+ case "session_switched": {
+ const msg: Message = {
+ id: generateId(),
+ role: "system",
+ type: "text",
+ content: `Switched to ${data.name}`,
+ timestamp: Date.now(),
+ };
+ setMessages((prev) => [...prev, msg]);
+ break;
+ }
+ case "session_renamed": {
+ const msg: Message = {
+ id: generateId(),
+ role: "system",
+ type: "text",
+ content: `Renamed to ${data.name}`,
+ timestamp: Date.now(),
+ };
+ setMessages((prev) => [...prev, msg]);
+ // Refresh sessions to show updated name
+ sendCommand("sessions");
+ break;
+ }
+ case "error": {
+ const msg: Message = {
+ id: generateId(),
+ role: "system",
+ type: "text",
+ content: data.message,
+ timestamp: Date.now(),
+ };
+ setMessages((prev) => [...prev, msg]);
+ break;
}
}
};
@@ -77,7 +151,7 @@
return () => {
onMessageReceived.current = null;
};
- }, [onMessageReceived]);
+ }, [onMessageReceived, sendCommand]);
const sendTextMessage = useCallback(
(text: string) => {
@@ -91,7 +165,6 @@
status: "sending",
};
addMessage(msg);
-
const sent = wsSend(text);
updateMessageStatus(id, sent ? "sent" : "error");
},
@@ -99,7 +172,7 @@
);
const sendVoiceMessage = useCallback(
- (audioUri: string, durationMs?: number) => {
+ async (audioUri: string, durationMs?: number) => {
const id = generateId();
const msg: Message = {
id,
@@ -112,10 +185,14 @@
duration: durationMs,
};
addMessage(msg);
-
- // For now, send with empty base64 since we'd need expo-file-system to encode
- const sent = wsVoice("", "Voice message");
- updateMessageStatus(id, sent ? "sent" : "error");
+ try {
+ const base64 = await encodeAudioToBase64(audioUri);
+ const sent = wsVoice(base64);
+ updateMessageStatus(id, sent ? "sent" : "error");
+ } catch (err) {
+ console.error("Failed to encode audio:", err);
+ updateMessageStatus(id, "error");
+ }
},
[wsVoice, addMessage, updateMessageStatus]
);
@@ -124,9 +201,52 @@
setMessages([]);
}, []);
+ // --- Session management ---
+ const requestSessions = useCallback(() => {
+ sendCommand("sessions");
+ }, [sendCommand]);
+
+ const switchSession = useCallback(
+ (sessionId: string) => {
+ sendCommand("switch", { sessionId });
+ },
+ [sendCommand]
+ );
+
+ const renameSession = useCallback(
+ (sessionId: string, name: string) => {
+ sendCommand("rename", { sessionId, name });
+ },
+ [sendCommand]
+ );
+
+ // --- Screenshot / navigation ---
+ const requestScreenshot = useCallback(() => {
+ sendCommand("screenshot");
+ }, [sendCommand]);
+
+ const sendNavKey = useCallback(
+ (key: string) => {
+ sendCommand("nav", { key });
+ },
+ [sendCommand]
+ );
+
return (
<ChatContext.Provider
- value={{ messages, sendTextMessage, sendVoiceMessage, clearMessages }}
+ value={{
+ messages,
+ sendTextMessage,
+ sendVoiceMessage,
+ clearMessages,
+ sessions,
+ requestSessions,
+ switchSession,
+ renameSession,
+ latestScreenshot,
+ requestScreenshot,
+ sendNavKey,
+ }}
>
{children}
</ChatContext.Provider>
diff --git a/contexts/ConnectionContext.tsx b/contexts/ConnectionContext.tsx
index 69d6193..860ec59 100644
--- a/contexts/ConnectionContext.tsx
+++ b/contexts/ConnectionContext.tsx
@@ -7,7 +7,12 @@
useState,
} from "react";
import * as SecureStore from "expo-secure-store";
-import { ConnectionStatus, ServerConfig, WebSocketMessage } from "../types";
+import {
+ ConnectionStatus,
+ ServerConfig,
+ WsIncoming,
+ WsOutgoing,
+} from "../types";
import { wsClient } from "../services/websocket";
const SECURE_STORE_KEY = "pailot_server_config";
@@ -19,9 +24,10 @@
disconnect: () => void;
sendTextMessage: (text: string) => boolean;
sendVoiceMessage: (audioBase64: string, transcript?: string) => boolean;
+ sendCommand: (command: string, args?: Record<string, unknown>) => boolean;
saveServerConfig: (config: ServerConfig) => Promise<void>;
onMessageReceived: React.MutableRefObject<
- ((data: WebSocketMessage) => void) | null
+ ((data: WsIncoming) => void) | null
>;
}
@@ -34,9 +40,7 @@
}) {
const [serverConfig, setServerConfig] = useState<ServerConfig | null>(null);
const [status, setStatus] = useState<ConnectionStatus>("disconnected");
- const onMessageReceived = useRef<((data: WebSocketMessage) => void) | null>(
- null
- );
+ const onMessageReceived = useRef<((data: WsIncoming) => void) | null>(null);
useEffect(() => {
loadConfig();
@@ -48,7 +52,7 @@
onClose: () => setStatus("disconnected"),
onError: () => setStatus("disconnected"),
onMessage: (data) => {
- onMessageReceived.current?.(data);
+ onMessageReceived.current?.(data as WsIncoming);
},
});
}, []);
@@ -92,18 +96,24 @@
}, []);
const sendTextMessage = useCallback((text: string): boolean => {
- const msg: WebSocketMessage = { type: "text", content: text };
- return wsClient.send(msg);
+ return wsClient.send({ type: "text", content: text });
}, []);
const sendVoiceMessage = useCallback(
(audioBase64: string, transcript: string = ""): boolean => {
- const msg: WebSocketMessage = {
+ return wsClient.send({
type: "voice",
content: transcript,
audioBase64,
- };
- return wsClient.send(msg);
+ });
+ },
+ []
+ );
+
+ const sendCommand = useCallback(
+ (command: string, args?: Record<string, unknown>): boolean => {
+ const msg: WsOutgoing = { type: "command", command, args };
+ return wsClient.send(msg as any);
},
[]
);
@@ -117,6 +127,7 @@
disconnect,
sendTextMessage,
sendVoiceMessage,
+ sendCommand,
saveServerConfig,
onMessageReceived,
}}
diff --git a/package-lock.json b/package-lock.json
index adaa4c1..936eae7 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,24 +8,37 @@
"name": "pailot",
"version": "1.0.0",
"dependencies": {
+ "@react-navigation/bottom-tabs": "^7.15.3",
+ "@react-navigation/native": "^7.1.31",
"expo": "~55.0.4",
- "expo-av": "^16.0.8",
+ "expo-audio": "^55.0.8",
+ "expo-constants": "~55.0.7",
+ "expo-file-system": "~55.0.10",
"expo-haptics": "~55.0.8",
+ "expo-linking": "~55.0.7",
"expo-router": "~55.0.3",
"expo-secure-store": "~55.0.8",
+ "expo-speech-recognition": "^3.1.1",
+ "expo-splash-screen": "~55.0.10",
"expo-status-bar": "~55.0.4",
+ "expo-system-ui": "~55.0.9",
+ "expo-web-browser": "~55.0.9",
"nativewind": "^4",
"react": "19.2.0",
+ "react-dom": "^19.2.4",
"react-native": "0.83.2",
"react-native-gesture-handler": "~2.30.0",
"react-native-reanimated": "4.2.1",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0",
- "react-native-svg": "15.15.3"
+ "react-native-svg": "15.15.3",
+ "react-native-web": "^0.21.0",
+ "react-native-worklets": "0.7.2"
},
"devDependencies": {
"@types/react": "~19.2.2",
"babel-plugin-module-resolver": "^5.0.2",
+ "babel-preset-expo": "^55.0.10",
"tailwindcss": "^3.4.19",
"typescript": "~5.9.2"
}
@@ -1317,6 +1330,21 @@
"@babel/core": "^7.0.0-0"
}
},
+ "node_modules/@babel/plugin-transform-template-literals": {
+ "version": "7.27.1",
+ "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.27.1.tgz",
+ "integrity": "sha512-fBJKiV7F2DxZUkg5EtHKXQdbsbURW3DZKQUWphDum0uRP6eHGGa/He9mc0mypL680pb+e/lDIthRohlv8NCHkg==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-plugin-utils": "^7.27.1"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
"node_modules/@babel/plugin-transform-typescript": {
"version": "7.28.6",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.28.6.tgz",
@@ -1733,6 +1761,27 @@
"@xmldom/xmldom": "^0.8.8",
"base64-js": "^1.5.1",
"xmlbuilder": "^15.1.1"
+ }
+ },
+ "node_modules/@expo/prebuild-config": {
+ "version": "55.0.8",
+ "resolved": "https://registry.npmjs.org/@expo/prebuild-config/-/prebuild-config-55.0.8.tgz",
+ "integrity": "sha512-VJNJiOmmZgyDnR7JMmc3B8Z0ZepZ17I8Wtw+wAH/2+UCUsFg588XU+bwgYcFGw+is28kwGjY46z43kfufpxOnA==",
+ "license": "MIT",
+ "dependencies": {
+ "@expo/config": "~55.0.8",
+ "@expo/config-plugins": "~55.0.6",
+ "@expo/config-types": "^55.0.5",
+ "@expo/image-utils": "^0.8.12",
+ "@expo/json-file": "^10.0.12",
+ "@react-native/normalize-colors": "0.83.2",
+ "debug": "^4.3.1",
+ "resolve-from": "^5.0.0",
+ "semver": "^7.6.0",
+ "xml2js": "0.6.0"
+ },
+ "peerDependencies": {
+ "expo": "*"
}
},
"node_modules/@expo/require-utils": {
@@ -3466,6 +3515,54 @@
"@babel/core": "^7.0.0 || ^8.0.0-0"
}
},
+ "node_modules/babel-preset-expo": {
+ "version": "55.0.10",
+ "resolved": "https://registry.npmjs.org/babel-preset-expo/-/babel-preset-expo-55.0.10.tgz",
+ "integrity": "sha512-aRtW7qJKohGU2V0LUJ6IeP7py3+kVUo9zcc8+v1Kix8jGGuIvqvpo9S6W1Fmn9VFP2DBwkFDLiyzkCZS85urVA==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/generator": "^7.20.5",
+ "@babel/helper-module-imports": "^7.25.9",
+ "@babel/plugin-proposal-decorators": "^7.12.9",
+ "@babel/plugin-proposal-export-default-from": "^7.24.7",
+ "@babel/plugin-syntax-export-default-from": "^7.24.7",
+ "@babel/plugin-transform-class-static-block": "^7.27.1",
+ "@babel/plugin-transform-export-namespace-from": "^7.25.9",
+ "@babel/plugin-transform-flow-strip-types": "^7.25.2",
+ "@babel/plugin-transform-modules-commonjs": "^7.24.8",
+ "@babel/plugin-transform-object-rest-spread": "^7.24.7",
+ "@babel/plugin-transform-parameters": "^7.24.7",
+ "@babel/plugin-transform-private-methods": "^7.24.7",
+ "@babel/plugin-transform-private-property-in-object": "^7.24.7",
+ "@babel/plugin-transform-runtime": "^7.24.7",
+ "@babel/preset-react": "^7.22.15",
+ "@babel/preset-typescript": "^7.23.0",
+ "@react-native/babel-preset": "0.83.2",
+ "babel-plugin-react-compiler": "^1.0.0",
+ "babel-plugin-react-native-web": "~0.21.0",
+ "babel-plugin-syntax-hermes-parser": "^0.32.0",
+ "babel-plugin-transform-flow-enums": "^0.0.2",
+ "debug": "^4.3.4",
+ "resolve-from": "^5.0.0"
+ },
+ "peerDependencies": {
+ "@babel/runtime": "^7.20.0",
+ "expo": "*",
+ "expo-widgets": "^55.0.2",
+ "react-refresh": ">=0.14.0 <1.0.0"
+ },
+ "peerDependenciesMeta": {
+ "@babel/runtime": {
+ "optional": true
+ },
+ "expo": {
+ "optional": true
+ },
+ "expo-widgets": {
+ "optional": true
+ }
+ }
+ },
"node_modules/babel-preset-jest": {
"version": "29.6.3",
"resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz",
@@ -4048,6 +4145,15 @@
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
"license": "MIT"
},
+ "node_modules/cross-fetch": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz",
+ "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==",
+ "license": "MIT",
+ "dependencies": {
+ "node-fetch": "^2.7.0"
+ }
+ },
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -4060,6 +4166,15 @@
},
"engines": {
"node": ">= 8"
+ }
+ },
+ "node_modules/css-in-js-utils": {
+ "version": "3.1.0",
+ "resolved": "https://registry.npmjs.org/css-in-js-utils/-/css-in-js-utils-3.1.0.tgz",
+ "integrity": "sha512-fJAcud6B3rRu+KHYk+Bwf+WFL2MDCJJ1XG9x137tJQ0xYxor7XziQtuGFbWNdqrvF4Tk26O3H73nfVqXt/fW1A==",
+ "license": "MIT",
+ "dependencies": {
+ "hyphenate-style-name": "^1.0.3"
}
},
"node_modules/css-select": {
@@ -4457,21 +4572,16 @@
}
}
},
- "node_modules/expo-av": {
- "version": "16.0.8",
- "resolved": "https://registry.npmjs.org/expo-av/-/expo-av-16.0.8.tgz",
- "integrity": "sha512-cmVPftGR/ca7XBgs7R6ky36lF3OC0/MM/lpgX/yXqfv0jASTsh7AYX9JxHCwFmF+Z6JEB1vne9FDx4GiLcGreQ==",
+ "node_modules/expo-audio": {
+ "version": "55.0.8",
+ "resolved": "https://registry.npmjs.org/expo-audio/-/expo-audio-55.0.8.tgz",
+ "integrity": "sha512-X61pQSikE2rsP2ZTMFUMThOmgGyYEHcmZpGVMrKJgcYtRCFKuctB/z69dFQPoumL+zTz8qlBoGohjkHVvA9P8A==",
"license": "MIT",
"peerDependencies": {
"expo": "*",
+ "expo-asset": "*",
"react": "*",
- "react-native": "*",
- "react-native-web": "*"
- },
- "peerDependenciesMeta": {
- "react-native-web": {
- "optional": true
- }
+ "react-native": "*"
}
},
"node_modules/expo-constants": {
@@ -4483,6 +4593,16 @@
"@expo/config": "~55.0.8",
"@expo/env": "~2.1.1"
},
+ "peerDependencies": {
+ "expo": "*",
+ "react-native": "*"
+ }
+ },
+ "node_modules/expo-file-system": {
+ "version": "55.0.10",
+ "resolved": "https://registry.npmjs.org/expo-file-system/-/expo-file-system-55.0.10.tgz",
+ "integrity": "sha512-ysFdVdUgtfj2ApY0Cn+pBg+yK4xp+SNwcaH8j2B91JJQ4OXJmnyCSmrNZYz7J4mdYVuv2GzxIP+N/IGlHQG3Yw==",
+ "license": "MIT",
"peerDependencies": {
"expo": "*",
"react-native": "*"
@@ -4540,6 +4660,20 @@
"react-native-web": {
"optional": true
}
+ }
+ },
+ "node_modules/expo-linking": {
+ "version": "55.0.7",
+ "resolved": "https://registry.npmjs.org/expo-linking/-/expo-linking-55.0.7.tgz",
+ "integrity": "sha512-MiGCedere1vzQTEi2aGrkzd7eh/rPSz4w6F3GMBuAJzYl+/0VhIuyhozpEGrueyDIXWfzaUVOcn3SfxVi+kwQQ==",
+ "license": "MIT",
+ "dependencies": {
+ "expo-constants": "~55.0.7",
+ "invariant": "^2.2.4"
+ },
+ "peerDependencies": {
+ "react": "*",
+ "react-native": "*"
}
},
"node_modules/expo-modules-autolinking": {
@@ -4729,6 +4863,29 @@
"node": ">=20.16.0"
}
},
+ "node_modules/expo-speech-recognition": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/expo-speech-recognition/-/expo-speech-recognition-3.1.1.tgz",
+ "integrity": "sha512-+1rviv+ZecAokY8PUfr3XJuhS4t0uKccewIPPUk5ooeEt5xKEWr6XYpKm3ggapPdJQbgMTjWbmSPT1ahTMyIqA==",
+ "license": "MIT",
+ "peerDependencies": {
+ "expo": "*",
+ "react": "*",
+ "react-native": "*"
+ }
+ },
+ "node_modules/expo-splash-screen": {
+ "version": "55.0.10",
+ "resolved": "https://registry.npmjs.org/expo-splash-screen/-/expo-splash-screen-55.0.10.tgz",
+ "integrity": "sha512-RN5qqrxudxFlRIjLFr/Ifmt+mUCLRc0gs66PekP6flzNS/JYEuoCbwJ+NmUwwJtPA+vyy60DYiky0QmS98ydmQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@expo/prebuild-config": "^55.0.8"
+ },
+ "peerDependencies": {
+ "expo": "*"
+ }
+ },
"node_modules/expo-status-bar": {
"version": "55.0.4",
"resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-55.0.4.tgz",
@@ -4755,6 +4912,36 @@
"expo": "*",
"expo-font": "*",
"react": "*",
+ "react-native": "*"
+ }
+ },
+ "node_modules/expo-system-ui": {
+ "version": "55.0.9",
+ "resolved": "https://registry.npmjs.org/expo-system-ui/-/expo-system-ui-55.0.9.tgz",
+ "integrity": "sha512-8ygP1B0uFAFI8s7eHY2IcGnE83GhFeZYwHBr/fQ4dSXnc7iVT9zp2PvyTyiDiibQ69dBG+fauMQ4KlPcOO51kQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@react-native/normalize-colors": "0.83.2",
+ "debug": "^4.3.2"
+ },
+ "peerDependencies": {
+ "expo": "*",
+ "react-native": "*",
+ "react-native-web": "*"
+ },
+ "peerDependenciesMeta": {
+ "react-native-web": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/expo-web-browser": {
+ "version": "55.0.9",
+ "resolved": "https://registry.npmjs.org/expo-web-browser/-/expo-web-browser-55.0.9.tgz",
+ "integrity": "sha512-PvAVsG401QmZabtTsYh1cYcpPiqvBPs8oiOkSrp0jIXnneiM466HxmeNtvo+fNxqJ2nwOBz9qLPiWRO91VBfsQ==",
+ "license": "MIT",
+ "peerDependencies": {
+ "expo": "*",
"react-native": "*"
}
},
@@ -4839,27 +5026,6 @@
}
}
},
- "node_modules/expo/node_modules/@expo/cli/node_modules/@expo/prebuild-config": {
- "version": "55.0.8",
- "resolved": "https://registry.npmjs.org/@expo/prebuild-config/-/prebuild-config-55.0.8.tgz",
- "integrity": "sha512-VJNJiOmmZgyDnR7JMmc3B8Z0ZepZ17I8Wtw+wAH/2+UCUsFg588XU+bwgYcFGw+is28kwGjY46z43kfufpxOnA==",
- "license": "MIT",
- "dependencies": {
- "@expo/config": "~55.0.8",
- "@expo/config-plugins": "~55.0.6",
- "@expo/config-types": "^55.0.5",
- "@expo/image-utils": "^0.8.12",
- "@expo/json-file": "^10.0.12",
- "@react-native/normalize-colors": "0.83.2",
- "debug": "^4.3.1",
- "resolve-from": "^5.0.0",
- "semver": "^7.6.0",
- "xml2js": "0.6.0"
- },
- "peerDependencies": {
- "expo": "*"
- }
- },
"node_modules/expo/node_modules/@expo/cli/node_modules/@expo/router-server": {
"version": "55.0.9",
"resolved": "https://registry.npmjs.org/@expo/router-server/-/router-server-55.0.9.tgz",
@@ -4940,54 +5106,6 @@
"react-native": "*"
}
},
- "node_modules/expo/node_modules/babel-preset-expo": {
- "version": "55.0.10",
- "resolved": "https://registry.npmjs.org/babel-preset-expo/-/babel-preset-expo-55.0.10.tgz",
- "integrity": "sha512-aRtW7qJKohGU2V0LUJ6IeP7py3+kVUo9zcc8+v1Kix8jGGuIvqvpo9S6W1Fmn9VFP2DBwkFDLiyzkCZS85urVA==",
- "license": "MIT",
- "dependencies": {
- "@babel/generator": "^7.20.5",
- "@babel/helper-module-imports": "^7.25.9",
- "@babel/plugin-proposal-decorators": "^7.12.9",
- "@babel/plugin-proposal-export-default-from": "^7.24.7",
- "@babel/plugin-syntax-export-default-from": "^7.24.7",
- "@babel/plugin-transform-class-static-block": "^7.27.1",
- "@babel/plugin-transform-export-namespace-from": "^7.25.9",
- "@babel/plugin-transform-flow-strip-types": "^7.25.2",
- "@babel/plugin-transform-modules-commonjs": "^7.24.8",
- "@babel/plugin-transform-object-rest-spread": "^7.24.7",
- "@babel/plugin-transform-parameters": "^7.24.7",
- "@babel/plugin-transform-private-methods": "^7.24.7",
- "@babel/plugin-transform-private-property-in-object": "^7.24.7",
- "@babel/plugin-transform-runtime": "^7.24.7",
- "@babel/preset-react": "^7.22.15",
- "@babel/preset-typescript": "^7.23.0",
- "@react-native/babel-preset": "0.83.2",
- "babel-plugin-react-compiler": "^1.0.0",
- "babel-plugin-react-native-web": "~0.21.0",
- "babel-plugin-syntax-hermes-parser": "^0.32.0",
- "babel-plugin-transform-flow-enums": "^0.0.2",
- "debug": "^4.3.4",
- "resolve-from": "^5.0.0"
- },
- "peerDependencies": {
- "@babel/runtime": "^7.20.0",
- "expo": "*",
- "expo-widgets": "^55.0.2",
- "react-refresh": ">=0.14.0 <1.0.0"
- },
- "peerDependenciesMeta": {
- "@babel/runtime": {
- "optional": true
- },
- "expo": {
- "optional": true
- },
- "expo-widgets": {
- "optional": true
- }
- }
- },
"node_modules/expo/node_modules/ci-info": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz",
@@ -5015,16 +5133,6 @@
"peerDependencies": {
"expo": "*",
"react": "*",
- "react-native": "*"
- }
- },
- "node_modules/expo/node_modules/expo-file-system": {
- "version": "55.0.10",
- "resolved": "https://registry.npmjs.org/expo-file-system/-/expo-file-system-55.0.10.tgz",
- "integrity": "sha512-ysFdVdUgtfj2ApY0Cn+pBg+yK4xp+SNwcaH8j2B91JJQ4OXJmnyCSmrNZYz7J4mdYVuv2GzxIP+N/IGlHQG3Yw==",
- "license": "MIT",
- "peerDependencies": {
- "expo": "*",
"react-native": "*"
}
},
@@ -5148,6 +5256,36 @@
"license": "Apache-2.0",
"dependencies": {
"bser": "2.1.1"
+ }
+ },
+ "node_modules/fbjs": {
+ "version": "3.0.5",
+ "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-3.0.5.tgz",
+ "integrity": "sha512-ztsSx77JBtkuMrEypfhgc3cI0+0h+svqeie7xHbh1k/IKdcydnvadp/mUaGgjAOXQmQSxsqgaRhS3q9fy+1kxg==",
+ "license": "MIT",
+ "dependencies": {
+ "cross-fetch": "^3.1.5",
+ "fbjs-css-vars": "^1.0.0",
+ "loose-envify": "^1.0.0",
+ "object-assign": "^4.1.0",
+ "promise": "^7.1.1",
+ "setimmediate": "^1.0.5",
+ "ua-parser-js": "^1.0.35"
+ }
+ },
+ "node_modules/fbjs-css-vars": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/fbjs-css-vars/-/fbjs-css-vars-1.0.2.tgz",
+ "integrity": "sha512-b2XGFAFdWZWg0phtAWLHCk836A1Xann+I+Dgd3Gk64MHKZO44FfoD1KxyvbSh0qZsIoXQGGlVztIY+oitJPpRQ==",
+ "license": "MIT"
+ },
+ "node_modules/fbjs/node_modules/promise": {
+ "version": "7.3.1",
+ "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
+ "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+ "license": "MIT",
+ "dependencies": {
+ "asap": "~2.0.3"
}
},
"node_modules/fetch-nodeshim": {
@@ -5481,6 +5619,12 @@
"node": ">= 14"
}
},
+ "node_modules/hyphenate-style-name": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.1.0.tgz",
+ "integrity": "sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw==",
+ "license": "BSD-3-Clause"
+ },
"node_modules/ignore": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
@@ -5530,6 +5674,15 @@
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
+ },
+ "node_modules/inline-style-prefixer": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/inline-style-prefixer/-/inline-style-prefixer-7.0.1.tgz",
+ "integrity": "sha512-lhYo5qNTQp3EvSSp3sRvXMbVQTLrvGV6DycRMJ5dm2BLMiJ30wpXKdDdgX+GmJZ5uQMucwRKHamXSst3Sj/Giw==",
+ "license": "MIT",
+ "dependencies": {
+ "css-in-js-utils": "^3.1.0"
+ }
},
"node_modules/invariant": {
"version": "2.2.4",
@@ -6840,6 +6993,26 @@
"node": ">= 0.6"
}
},
+ "node_modules/node-fetch": {
+ "version": "2.7.0",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+ "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+ "license": "MIT",
+ "dependencies": {
+ "whatwg-url": "^5.0.0"
+ },
+ "engines": {
+ "node": "4.x || >=6.0.0"
+ },
+ "peerDependencies": {
+ "encoding": "^0.1.0"
+ },
+ "peerDependenciesMeta": {
+ "encoding": {
+ "optional": true
+ }
+ }
+ },
"node_modules/node-forge": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.3.tgz",
@@ -6919,7 +7092,6 @@
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
- "dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@@ -7499,7 +7671,6 @@
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
- "dev": true,
"license": "MIT"
},
"node_modules/pretty-format": {
@@ -7642,6 +7813,18 @@
"dependencies": {
"shell-quote": "^1.6.1",
"ws": "^7"
+ }
+ },
+ "node_modules/react-dom": {
+ "version": "19.2.4",
+ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz",
+ "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==",
+ "license": "MIT",
+ "dependencies": {
+ "scheduler": "^0.27.0"
+ },
+ "peerDependencies": {
+ "react": "^19.2.4"
}
},
"node_modules/react-fast-compare": {
@@ -8086,6 +8269,160 @@
"peerDependencies": {
"react": "*",
"react-native": "*"
+ }
+ },
+ "node_modules/react-native-web": {
+ "version": "0.21.2",
+ "resolved": "https://registry.npmjs.org/react-native-web/-/react-native-web-0.21.2.tgz",
+ "integrity": "sha512-SO2t9/17zM4iEnFvlu2DA9jqNbzNhoUP+AItkoCOyFmDMOhUnBBznBDCYN92fGdfAkfQlWzPoez6+zLxFNsZEg==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/runtime": "^7.18.6",
+ "@react-native/normalize-colors": "^0.74.1",
+ "fbjs": "^3.0.4",
+ "inline-style-prefixer": "^7.0.1",
+ "memoize-one": "^6.0.0",
+ "nullthrows": "^1.1.1",
+ "postcss-value-parser": "^4.2.0",
+ "styleq": "^0.1.3"
+ },
+ "peerDependencies": {
+ "react": "^18.0.0 || ^19.0.0",
+ "react-dom": "^18.0.0 || ^19.0.0"
+ }
+ },
+ "node_modules/react-native-web/node_modules/@react-native/normalize-colors": {
+ "version": "0.74.89",
+ "resolved": "https://registry.npmjs.org/@react-native/normalize-colors/-/normalize-colors-0.74.89.tgz",
+ "integrity": "sha512-qoMMXddVKVhZ8PA1AbUCk83trpd6N+1nF2A6k1i6LsQObyS92fELuk8kU/lQs6M7BsMHwqyLCpQJ1uFgNvIQXg==",
+ "license": "MIT"
+ },
+ "node_modules/react-native-web/node_modules/memoize-one": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz",
+ "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==",
+ "license": "MIT"
+ },
+ "node_modules/react-native-worklets": {
+ "version": "0.7.2",
+ "resolved": "https://registry.npmjs.org/react-native-worklets/-/react-native-worklets-0.7.2.tgz",
+ "integrity": "sha512-DuLu1kMV/Uyl9pQHp3hehAlThoLw7Yk2FwRTpzASOmI+cd4845FWn3m2bk9MnjUw8FBRIyhwLqYm2AJaXDXsog==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/plugin-transform-arrow-functions": "7.27.1",
+ "@babel/plugin-transform-class-properties": "7.27.1",
+ "@babel/plugin-transform-classes": "7.28.4",
+ "@babel/plugin-transform-nullish-coalescing-operator": "7.27.1",
+ "@babel/plugin-transform-optional-chaining": "7.27.1",
+ "@babel/plugin-transform-shorthand-properties": "7.27.1",
+ "@babel/plugin-transform-template-literals": "7.27.1",
+ "@babel/plugin-transform-unicode-regex": "7.27.1",
+ "@babel/preset-typescript": "7.27.1",
+ "convert-source-map": "2.0.0",
+ "semver": "7.7.3"
+ },
+ "peerDependencies": {
+ "@babel/core": "*",
+ "react": "*",
+ "react-native": "*"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/@babel/plugin-transform-class-properties": {
+ "version": "7.27.1",
+ "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.27.1.tgz",
+ "integrity": "sha512-D0VcalChDMtuRvJIu3U/fwWjf8ZMykz5iZsg77Nuj821vCKI3zCyRLwRdWbsuJ/uRwZhZ002QtCqIkwC/ZkvbA==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-create-class-features-plugin": "^7.27.1",
+ "@babel/helper-plugin-utils": "^7.27.1"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/@babel/plugin-transform-classes": {
+ "version": "7.28.4",
+ "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.28.4.tgz",
+ "integrity": "sha512-cFOlhIYPBv/iBoc+KS3M6et2XPtbT2HiCRfBXWtfpc9OAyostldxIf9YAYB6ypURBBbx+Qv6nyrLzASfJe+hBA==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-annotate-as-pure": "^7.27.3",
+ "@babel/helper-compilation-targets": "^7.27.2",
+ "@babel/helper-globals": "^7.28.0",
+ "@babel/helper-plugin-utils": "^7.27.1",
+ "@babel/helper-replace-supers": "^7.27.1",
+ "@babel/traverse": "^7.28.4"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/@babel/plugin-transform-nullish-coalescing-operator": {
+ "version": "7.27.1",
+ "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.27.1.tgz",
+ "integrity": "sha512-aGZh6xMo6q9vq1JGcw58lZ1Z0+i0xB2x0XaauNIUXd6O1xXc3RwoWEBlsTQrY4KQ9Jf0s5rgD6SiNkaUdJegTA==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-plugin-utils": "^7.27.1"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/@babel/plugin-transform-optional-chaining": {
+ "version": "7.27.1",
+ "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.27.1.tgz",
+ "integrity": "sha512-BQmKPPIuc8EkZgNKsv0X4bPmOoayeu4F1YCwx2/CfmDSXDbp7GnzlUH+/ul5VGfRg1AoFPsrIThlEBj2xb4CAg==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-plugin-utils": "^7.27.1",
+ "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/@babel/preset-typescript": {
+ "version": "7.27.1",
+ "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.27.1.tgz",
+ "integrity": "sha512-l7WfQfX0WK4M0v2RudjuQK4u99BS6yLHYEmdtVPP7lKV013zr9DygFuWNlnbvQ9LR+LS0Egz/XAvGx5U9MX0fQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@babel/helper-plugin-utils": "^7.27.1",
+ "@babel/helper-validator-option": "^7.27.1",
+ "@babel/plugin-syntax-jsx": "^7.27.1",
+ "@babel/plugin-transform-modules-commonjs": "^7.27.1",
+ "@babel/plugin-transform-typescript": "^7.27.1"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ },
+ "peerDependencies": {
+ "@babel/core": "^7.0.0-0"
+ }
+ },
+ "node_modules/react-native-worklets/node_modules/semver": {
+ "version": "7.7.3",
+ "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
+ "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
+ "license": "ISC",
+ "bin": {
+ "semver": "bin/semver.js"
+ },
+ "engines": {
+ "node": ">=10"
}
},
"node_modules/react-native/node_modules/@react-native/virtualized-lists": {
@@ -8648,6 +8985,12 @@
"integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==",
"license": "MIT"
},
+ "node_modules/setimmediate": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+ "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==",
+ "license": "MIT"
+ },
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
@@ -8900,6 +9243,12 @@
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/structured-headers/-/structured-headers-0.4.1.tgz",
"integrity": "sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==",
+ "license": "MIT"
+ },
+ "node_modules/styleq": {
+ "version": "0.1.3",
+ "resolved": "https://registry.npmjs.org/styleq/-/styleq-0.1.3.tgz",
+ "integrity": "sha512-3ZUifmCDCQanjeej1f6kyl/BeP/Vae5EYkQ9iJfUm/QwZvlgnZzyflqAsAWYURdtea8Vkvswu2GrC57h3qffcA==",
"license": "MIT"
},
"node_modules/sucrase": {
@@ -9223,6 +9572,12 @@
"integrity": "sha512-FWAPzCIHZHnrE/5/w9MPk0kK25hSQSH2IKhYh9PyjS3SG/+IEMvlwIHbhz+oF7xl54I+ueZlVnMjyzdSwLmAwA==",
"license": "MIT"
},
+ "node_modules/tr46": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+ "license": "MIT"
+ },
"node_modules/ts-interface-checker": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz",
@@ -9266,6 +9621,32 @@
},
"engines": {
"node": ">=14.17"
+ }
+ },
+ "node_modules/ua-parser-js": {
+ "version": "1.0.41",
+ "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.41.tgz",
+ "integrity": "sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==",
+ "funding": [
+ {
+ "type": "opencollective",
+ "url": "https://opencollective.com/ua-parser-js"
+ },
+ {
+ "type": "paypal",
+ "url": "https://paypal.me/faisalman"
+ },
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/faisalman"
+ }
+ ],
+ "license": "MIT",
+ "bin": {
+ "ua-parser-js": "script/cli.js"
+ },
+ "engines": {
+ "node": "*"
}
},
"node_modules/undici-types": {
@@ -9500,12 +9881,28 @@
"defaults": "^1.0.3"
}
},
+ "node_modules/webidl-conversions": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+ "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+ "license": "BSD-2-Clause"
+ },
"node_modules/whatwg-fetch": {
"version": "3.6.20",
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
"integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==",
"license": "MIT"
},
+ "node_modules/whatwg-url": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+ "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+ "license": "MIT",
+ "dependencies": {
+ "tr46": "~0.0.3",
+ "webidl-conversions": "^3.0.0"
+ }
+ },
"node_modules/whatwg-url-minimum": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/whatwg-url-minimum/-/whatwg-url-minimum-0.1.1.tgz",
diff --git a/package.json b/package.json
index f4e1a7a..1dec761 100644
--- a/package.json
+++ b/package.json
@@ -4,29 +4,42 @@
"main": "expo-router/entry",
"scripts": {
"start": "expo start",
- "android": "expo start --android",
- "ios": "expo start --ios",
+ "android": "expo run:android",
+ "ios": "expo run:ios",
"web": "expo start --web"
},
"dependencies": {
+ "@react-navigation/bottom-tabs": "^7.15.3",
+ "@react-navigation/native": "^7.1.31",
"expo": "~55.0.4",
- "expo-av": "^16.0.8",
+ "expo-audio": "^55.0.8",
+ "expo-constants": "~55.0.7",
+ "expo-file-system": "~55.0.10",
"expo-haptics": "~55.0.8",
+ "expo-linking": "~55.0.7",
"expo-router": "~55.0.3",
"expo-secure-store": "~55.0.8",
+ "expo-speech-recognition": "^3.1.1",
+ "expo-splash-screen": "~55.0.10",
"expo-status-bar": "~55.0.4",
+ "expo-system-ui": "~55.0.9",
+ "expo-web-browser": "~55.0.9",
"nativewind": "^4",
"react": "19.2.0",
+ "react-dom": "^19.2.4",
"react-native": "0.83.2",
"react-native-gesture-handler": "~2.30.0",
"react-native-reanimated": "4.2.1",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0",
- "react-native-svg": "15.15.3"
+ "react-native-svg": "15.15.3",
+ "react-native-web": "^0.21.0",
+ "react-native-worklets": "0.7.2"
},
"devDependencies": {
"@types/react": "~19.2.2",
"babel-plugin-module-resolver": "^5.0.2",
+ "babel-preset-expo": "^55.0.10",
"tailwindcss": "^3.4.19",
"typescript": "~5.9.2"
},
diff --git a/services/audio.ts b/services/audio.ts
index 31c7dd8..769d299 100644
--- a/services/audio.ts
+++ b/services/audio.ts
@@ -1,112 +1,65 @@
-import { Audio, AVPlaybackStatus } from "expo-av";
+import {
+ createAudioPlayer,
+ requestRecordingPermissionsAsync,
+ setAudioModeAsync,
+} from "expo-audio";
export interface RecordingResult {
uri: string;
durationMs: number;
}
-let currentRecording: Audio.Recording | null = null;
-let currentSound: Audio.Sound | null = null;
+let currentPlayer: ReturnType<typeof createAudioPlayer> | null = null;
-async function requestPermissions(): Promise<boolean> {
- const { status } = await Audio.requestPermissionsAsync();
+export async function requestPermissions(): Promise<boolean> {
+ const { status } = await requestRecordingPermissionsAsync();
return status === "granted";
-}
-
-export async function startRecording(): Promise<Audio.Recording | null> {
- const granted = await requestPermissions();
- if (!granted) return null;
-
- try {
- await Audio.setAudioModeAsync({
- allowsRecordingIOS: true,
- playsInSilentModeIOS: true,
- });
-
- const { recording } = await Audio.Recording.createAsync(
- Audio.RecordingOptionsPresets.HIGH_QUALITY
- );
-
- currentRecording = recording;
- return recording;
- } catch (error) {
- console.error("Failed to start recording:", error);
- return null;
- }
-}
-
-export async function stopRecording(): Promise<RecordingResult | null> {
- if (!currentRecording) return null;
-
- try {
- await currentRecording.stopAndUnloadAsync();
- const status = await currentRecording.getStatusAsync();
- const uri = currentRecording.getURI();
- currentRecording = null;
-
- await Audio.setAudioModeAsync({
- allowsRecordingIOS: false,
- });
-
- if (!uri) return null;
-
- const durationMs = (status as { durationMillis?: number }).durationMillis ?? 0;
- return { uri, durationMs };
- } catch (error) {
- console.error("Failed to stop recording:", error);
- currentRecording = null;
- return null;
- }
}
export async function playAudio(
uri: string,
onFinish?: () => void
-): Promise<Audio.Sound | null> {
+): Promise<void> {
try {
await stopPlayback();
- await Audio.setAudioModeAsync({
- allowsRecordingIOS: false,
- playsInSilentModeIOS: true,
+ await setAudioModeAsync({
+ playsInSilentMode: true,
});
- const { sound } = await Audio.Sound.createAsync(
- { uri },
- { shouldPlay: true }
- );
+ const player = createAudioPlayer(uri);
+ currentPlayer = player;
- currentSound = sound;
-
- sound.setOnPlaybackStatusUpdate((status: AVPlaybackStatus) => {
- if (status.isLoaded && status.didJustFinish) {
+ player.addListener("playbackStatusUpdate", (status) => {
+ if (!status.playing && status.currentTime >= status.duration && status.duration > 0) {
onFinish?.();
- sound.unloadAsync().catch(() => {});
- currentSound = null;
+ player.remove();
+ if (currentPlayer === player) currentPlayer = null;
}
});
- return sound;
+ player.play();
} catch (error) {
console.error("Failed to play audio:", error);
- return null;
}
}
export async function stopPlayback(): Promise<void> {
- if (currentSound) {
+ if (currentPlayer) {
try {
- await currentSound.stopAsync();
- await currentSound.unloadAsync();
+ currentPlayer.pause();
+ currentPlayer.remove();
} catch {
- // Ignore errors during cleanup
+ // Ignore cleanup errors
}
- currentSound = null;
+ currentPlayer = null;
}
}
-export function encodeAudioToBase64(uri: string): Promise<string> {
- // In React Native, we'd use FileSystem from expo-file-system
- // For now, return the URI as-is since we may not have expo-file-system
- return Promise.resolve(uri);
+export async function encodeAudioToBase64(uri: string): Promise<string> {
+ const FileSystem = await import("expo-file-system");
+ const result = await FileSystem.readAsStringAsync(uri, {
+ encoding: FileSystem.EncodingType.Base64,
+ });
+ return result;
}
diff --git a/types/index.ts b/types/index.ts
index fe5754d..aae0ecd 100644
--- a/types/index.ts
+++ b/types/index.ts
@@ -1,5 +1,5 @@
export type MessageRole = "user" | "assistant" | "system";
-export type MessageType = "text" | "voice";
+export type MessageType = "text" | "voice" | "image";
export interface Message {
id: string;
@@ -7,6 +7,7 @@
type: MessageType;
content: string;
audioUri?: string;
+ imageBase64?: string;
timestamp: number;
status?: "sending" | "sent" | "error";
duration?: number;
@@ -19,8 +20,81 @@
export type ConnectionStatus = "disconnected" | "connecting" | "connected";
-export interface WebSocketMessage {
- type: "text" | "voice";
+// --- WebSocket protocol ---
+
+/** Outgoing from app to watcher */
+export interface WsTextMessage {
+ type: "text";
+ content: string;
+}
+
+export interface WsVoiceMessage {
+ type: "voice";
+ audioBase64: string;
+ content: string;
+}
+
+export interface WsCommandMessage {
+ type: "command";
+ command: string;
+ args?: Record<string, unknown>;
+}
+
+export type WsOutgoing = WsTextMessage | WsVoiceMessage | WsCommandMessage;
+
+/** Incoming from watcher to app */
+export interface WsIncomingText {
+ type: "text";
+ content: string;
+}
+
+export interface WsIncomingVoice {
+ type: "voice";
content: string;
audioBase64?: string;
}
+
+export interface WsIncomingImage {
+ type: "image";
+ imageBase64: string;
+ caption?: string;
+}
+
+export interface WsSession {
+ index: number;
+ name: string;
+ type: "claude" | "terminal";
+ isActive: boolean;
+ id: string;
+}
+
+export interface WsIncomingSessions {
+ type: "sessions";
+ sessions: WsSession[];
+}
+
+export interface WsIncomingSessionSwitched {
+ type: "session_switched";
+ name: string;
+ sessionId: string;
+}
+
+export interface WsIncomingSessionRenamed {
+ type: "session_renamed";
+ sessionId: string;
+ name: string;
+}
+
+export interface WsIncomingError {
+ type: "error";
+ message: string;
+}
+
+export type WsIncoming =
+ | WsIncomingText
+ | WsIncomingVoice
+ | WsIncomingImage
+ | WsIncomingSessions
+ | WsIncomingSessionSwitched
+ | WsIncomingSessionRenamed
+ | WsIncomingError;
--
Gitblit v1.3.1