From fa34201bc07e5312ff0c6825933cd02ce7900254 Mon Sep 17 00:00:00 2001
From: Matthias Nott <mnott@mnsoft.org>
Date: Sat, 21 Mar 2026 20:55:10 +0100
Subject: [PATCH] fix: voice caption ordering, background audio, image persistence
---
lib/screens/settings_screen.dart | 2
lib/services/wol_service.dart | 49 +++++--
lib/services/message_store.dart | 27 ++++
lib/services/websocket_service.dart | 2
lib/services/audio_service.dart | 14 ++
ios/Runner/Info.plist | 4
lib/widgets/message_bubble.dart | 56 ++++++---
lib/models/message.dart | 5
lib/screens/chat_screen.dart | 172 ++++++++++++++++++----------
9 files changed, 225 insertions(+), 106 deletions(-)
diff --git a/ios/Runner/Info.plist b/ios/Runner/Info.plist
index 8c56583..59f2f28 100644
--- a/ios/Runner/Info.plist
+++ b/ios/Runner/Info.plist
@@ -85,5 +85,9 @@
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
+ <key>UIBackgroundModes</key>
+ <array>
+ <string>audio</string>
+ </array>
</dict>
</plist>
diff --git a/lib/models/message.dart b/lib/models/message.dart
index 002788b..6ea436b 100644
--- a/lib/models/message.dart
+++ b/lib/models/message.dart
@@ -114,7 +114,7 @@
};
}
- /// Lightweight JSON for persistence (strips heavy binary fields).
+ /// Lightweight JSON for persistence (strips temp audio paths, keeps images).
Map<String, dynamic> toJsonLight() {
return {
'id': id,
@@ -124,6 +124,9 @@
'timestamp': timestamp,
if (status != null) 'status': status!.name,
if (duration != null) 'duration': duration,
+ // Keep imageBase64 — images are typically 50-200 KB and must survive restart.
+ // audioUri is intentionally omitted: it is a temp file path that won't survive restart.
+ if (imageBase64 != null) 'imageBase64': imageBase64,
};
}
diff --git a/lib/screens/chat_screen.dart b/lib/screens/chat_screen.dart
index 417781d..0d6cc0f 100644
--- a/lib/screens/chat_screen.dart
+++ b/lib/screens/chat_screen.dart
@@ -47,14 +47,16 @@
void initState() {
super.initState();
WidgetsBinding.instance.addObserver(this);
- _loadLastSeq();
- _initConnection();
+ _initAll();
_scrollController.addListener(_onScroll);
}
- Future<void> _loadLastSeq() async {
+ Future<void> _initAll() async {
+ // Load lastSeq BEFORE connecting so catch_up sends the right value
final prefs = await SharedPreferences.getInstance();
_lastSeq = prefs.getInt('lastSeq') ?? 0;
+ if (!mounted) return;
+ _initConnection();
}
void _saveLastSeq() {
@@ -538,25 +540,8 @@
textCaption = '';
}
- // Send all images together — first with caption, rest without
- for (var i = 0; i < encodedImages.length; i++) {
- final isFirst = i == 0;
- final msgCaption = isFirst ? textCaption : '';
-
- _ws?.send({
- 'type': 'image',
- 'imageBase64': encodedImages[i],
- 'mimeType': 'image/jpeg',
- 'caption': msgCaption,
- if (isFirst && voiceB64 != null) 'audioBase64': voiceB64,
- 'sessionId': ref.read(activeSessionIdProvider),
- // Signal how many images follow so receiving session can wait
- if (isFirst && encodedImages.length > 1)
- 'totalImages': encodedImages.length,
- });
- }
-
- // If voice caption, also send the voice message so it gets transcribed
+ // Send voice FIRST so Whisper transcribes it and the [PAILot:voice] prefix
+ // sets the reply channel. Images follow — Claude sees transcript + images together.
if (voiceB64 != null) {
final voiceMsg = Message.voice(
role: MessageRole.user,
@@ -569,6 +554,20 @@
'audioBase64': voiceB64,
'content': '',
'messageId': voiceMsg.id,
+ 'sessionId': ref.read(activeSessionIdProvider),
+ });
+ }
+
+ // Send images — first with text caption (if any), rest without
+ for (var i = 0; i < encodedImages.length; i++) {
+ final isFirst = i == 0;
+ final msgCaption = isFirst ? textCaption : '';
+
+ _ws?.send({
+ 'type': 'image',
+ 'imageBase64': encodedImages[i],
+ 'mimeType': 'image/jpeg',
+ 'caption': msgCaption,
'sessionId': ref.read(activeSessionIdProvider),
});
}
@@ -591,8 +590,9 @@
final captionController = TextEditingController();
String? voicePath;
bool isVoiceRecording = false;
+ bool hasVoiceCaption = false;
- return showModalBottomSheet<String>(
+ final result = await showModalBottomSheet<String>(
context: context,
isScrollControlled: true,
builder: (ctx) => StatefulBuilder(
@@ -611,58 +611,94 @@
style: Theme.of(ctx).textTheme.titleSmall,
),
const SizedBox(height: 12),
- TextField(
- controller: captionController,
- decoration: InputDecoration(
- hintText: 'Add a caption (optional)',
- border: const OutlineInputBorder(),
- suffixIcon: IconButton(
- icon: Icon(
- isVoiceRecording ? Icons.stop_circle : Icons.mic,
- color: isVoiceRecording ? Colors.red : null,
- ),
- onPressed: () async {
- if (isVoiceRecording) {
- final path = await AudioService.stopRecording();
- setSheetState(() => isVoiceRecording = false);
- if (path != null) {
- voicePath = path;
- captionController.text = '🎤 Voice caption recorded';
- }
- } else {
- final path = await AudioService.startRecording();
- if (path != null) {
- setSheetState(() => isVoiceRecording = true);
- }
- }
- },
+ // Text caption input
+ if (!isVoiceRecording && !hasVoiceCaption)
+ TextField(
+ controller: captionController,
+ decoration: const InputDecoration(
+ hintText: 'Add a text caption (optional)',
+ border: OutlineInputBorder(),
+ ),
+ autofocus: true,
+ maxLines: 3,
+ ),
+ // Voice recording indicator
+ if (isVoiceRecording)
+ Container(
+ padding: const EdgeInsets.symmetric(vertical: 20),
+ child: const Row(
+ mainAxisAlignment: MainAxisAlignment.center,
+ children: [
+ Icon(Icons.fiber_manual_record, color: Colors.red, size: 16),
+ SizedBox(width: 8),
+ Text('Recording voice caption...', style: TextStyle(fontSize: 16)),
+ ],
),
),
- autofocus: true,
- maxLines: 3,
- enabled: !isVoiceRecording,
- ),
+ // Voice recorded confirmation
+ if (hasVoiceCaption && !isVoiceRecording)
+ Container(
+ padding: const EdgeInsets.symmetric(vertical: 20),
+ child: const Row(
+ mainAxisAlignment: MainAxisAlignment.center,
+ children: [
+ Icon(Icons.check_circle, color: Colors.green, size: 20),
+ SizedBox(width: 8),
+ Text('Voice caption recorded', style: TextStyle(fontSize: 16)),
+ ],
+ ),
+ ),
const SizedBox(height: 12),
+ // Action row: mic/stop + cancel + send
Row(
- mainAxisAlignment: MainAxisAlignment.end,
children: [
+ // Mic / Stop button — large and clear
+ if (!hasVoiceCaption)
+ IconButton.filled(
+ onPressed: () async {
+ if (isVoiceRecording) {
+ final path = await AudioService.stopRecording();
+ setSheetState(() {
+ isVoiceRecording = false;
+ if (path != null) {
+ voicePath = path;
+ hasVoiceCaption = true;
+ }
+ });
+ } else {
+ final path = await AudioService.startRecording();
+ if (path != null) {
+ setSheetState(() => isVoiceRecording = true);
+ }
+ }
+ },
+ icon: Icon(isVoiceRecording ? Icons.stop : Icons.mic),
+ style: IconButton.styleFrom(
+ backgroundColor: isVoiceRecording ? Colors.red : null,
+ foregroundColor: isVoiceRecording ? Colors.white : null,
+ ),
+ ),
+ const Spacer(),
TextButton(
- onPressed: () {
- if (isVoiceRecording) AudioService.cancelRecording();
- Navigator.pop(ctx);
+ onPressed: () async {
+ if (isVoiceRecording) {
+ await AudioService.cancelRecording();
+ }
+ if (ctx.mounted) Navigator.pop(ctx);
},
child: const Text('Cancel'),
),
const SizedBox(width: 8),
FilledButton(
- onPressed: () {
- if (voicePath != null) {
- // Voice caption: send as voice message with images
- Navigator.pop(ctx, '__voice__:$voicePath');
- } else {
- Navigator.pop(ctx, captionController.text);
- }
- },
+ onPressed: isVoiceRecording
+ ? null // disable Send while recording
+ : () {
+ if (voicePath != null) {
+ Navigator.pop(ctx, '__voice__:$voicePath');
+ } else {
+ Navigator.pop(ctx, captionController.text);
+ }
+ },
child: const Text('Send'),
),
],
@@ -673,6 +709,14 @@
),
),
);
+
+ // Safety net: clean up recording if sheet dismissed by swipe/tap outside
+ if (isVoiceRecording) {
+ await AudioService.cancelRecording();
+ }
+
+ captionController.dispose();
+ return result;
}
void _clearChat() {
diff --git a/lib/screens/settings_screen.dart b/lib/screens/settings_screen.dart
index faddac6..6cb89e7 100644
--- a/lib/screens/settings_screen.dart
+++ b/lib/screens/settings_screen.dart
@@ -84,7 +84,7 @@
setState(() => _isWaking = true);
try {
- await WolService.wake(mac);
+ await WolService.wake(mac, localHost: _localHostController.text.trim());
if (mounted) {
ScaffoldMessenger.of(context).showSnackBar(
const SnackBar(content: Text('Wake-on-LAN packet sent')),
diff --git a/lib/services/audio_service.dart b/lib/services/audio_service.dart
index 587c368..8e4db7c 100644
--- a/lib/services/audio_service.dart
+++ b/lib/services/audio_service.dart
@@ -28,6 +28,20 @@
// Listen for app lifecycle changes to suppress autoplay when backgrounded
WidgetsBinding.instance.addObserver(_LifecycleObserver());
+ // Configure audio session for playback — allows audio to continue
+ // when screen locks or app goes to background
+ _player.setAudioContext(AudioContext(
+ iOS: AudioContextIOS(
+ category: AVAudioSessionCategory.playback,
+ options: {AVAudioSessionOptions.mixWithOthers},
+ ),
+ android: const AudioContextAndroid(
+ isSpeakerphoneOn: false,
+ audioMode: AndroidAudioMode.normal,
+ audioFocus: AndroidAudioFocus.gain,
+ ),
+ ));
+
_player.onPlayerComplete.listen((_) {
if (_isChainPlaying) {
_playNext();
diff --git a/lib/services/message_store.dart b/lib/services/message_store.dart
index 448390e..424d73a 100644
--- a/lib/services/message_store.dart
+++ b/lib/services/message_store.dart
@@ -82,7 +82,7 @@
final jsonStr = await file.readAsString();
final List<dynamic> jsonList = jsonDecode(jsonStr) as List<dynamic>;
final allMessages = jsonList
- .map((j) => Message.fromJson(j as Map<String, dynamic>))
+ .map((j) => _messageFromJson(j as Map<String, dynamic>))
.where((m) => !m.isEmptyVoice) // Filter out voice msgs with no content
.toList();
@@ -106,7 +106,7 @@
final jsonStr = await file.readAsString();
final List<dynamic> jsonList = jsonDecode(jsonStr) as List<dynamic>;
return jsonList
- .map((j) => Message.fromJson(j as Map<String, dynamic>))
+ .map((j) => _messageFromJson(j as Map<String, dynamic>))
.where((m) => !m.isEmptyVoice)
.toList();
} catch (e) {
@@ -114,6 +114,29 @@
}
}
+ /// Deserialize a message from JSON, applying migration rules:
+ /// - Voice messages without audioUri are downgraded to text (transcript only).
+ /// This handles messages saved before a restart, where the temp audio file
+ /// is no longer available. The transcript (content) is preserved.
+ static Message _messageFromJson(Map<String, dynamic> json) {
+ final raw = Message.fromJson(json);
+ if (raw.type == MessageType.voice &&
+ (raw.audioUri == null || raw.audioUri!.isEmpty)) {
+ // Downgrade to text so the bubble shows the transcript instead of a
+ // broken play button.
+ return Message(
+ id: raw.id,
+ role: raw.role,
+ type: MessageType.text,
+ content: raw.content,
+ timestamp: raw.timestamp,
+ status: raw.status,
+ duration: raw.duration,
+ );
+ }
+ return raw;
+ }
+
/// Delete stored messages for a session.
static Future<void> delete(String sessionId) async {
try {
diff --git a/lib/services/websocket_service.dart b/lib/services/websocket_service.dart
index b444e4e..96e21fa 100644
--- a/lib/services/websocket_service.dart
+++ b/lib/services/websocket_service.dart
@@ -60,7 +60,7 @@
// Send Wake-on-LAN if MAC configured
if (config.macAddress != null && config.macAddress!.isNotEmpty) {
try {
- await WolService.wake(config.macAddress!);
+ await WolService.wake(config.macAddress!, localHost: config.localHost);
} catch (_) {}
}
diff --git a/lib/services/wol_service.dart b/lib/services/wol_service.dart
index edc0ae1..08fc639 100644
--- a/lib/services/wol_service.dart
+++ b/lib/services/wol_service.dart
@@ -32,9 +32,18 @@
return packet.toBytes();
}
+ /// Derive subnet broadcast from an IP address (e.g., 192.168.1.100 → 192.168.1.255).
+ static String? _subnetBroadcast(String? ip) {
+ if (ip == null || ip.isEmpty) return null;
+ final parts = ip.split('.');
+ if (parts.length != 4) return null;
+ return '${parts[0]}.${parts[1]}.${parts[2]}.255';
+ }
+
/// Send a Wake-on-LAN packet for the given MAC address.
- /// Broadcasts to 255.255.255.255:9 and optionally to a subnet broadcast.
- static Future<void> wake(String macAddress, {String? subnetBroadcast}) async {
+ /// Broadcasts to 255.255.255.255 and subnet broadcast derived from localHost.
+ /// Sends on ports 7 and 9 for maximum compatibility.
+ static Future<void> wake(String macAddress, {String? localHost}) async {
final macBytes = _parseMac(macAddress);
if (macBytes == null) {
throw ArgumentError('Invalid MAC address: $macAddress');
@@ -48,25 +57,31 @@
);
socket.broadcastEnabled = true;
- // Send to broadcast address
- final broadcastAddr = InternetAddress('255.255.255.255');
- socket.send(packet, broadcastAddr, 9);
+ final targets = <InternetAddress>[
+ InternetAddress('255.255.255.255'),
+ ];
- // Also send to subnet broadcast if provided
- if (subnetBroadcast != null && subnetBroadcast.isNotEmpty) {
+ // Add subnet broadcast derived from localHost
+ final subnet = _subnetBroadcast(localHost);
+ if (subnet != null) {
try {
- final subnetAddr = InternetAddress(subnetBroadcast);
- socket.send(packet, subnetAddr, 9);
- } catch (_) {
- // Ignore invalid subnet broadcast address
- }
+ targets.add(InternetAddress(subnet));
+ } catch (_) {}
}
- // Send a few extra packets for reliability
- await Future.delayed(const Duration(milliseconds: 100));
- socket.send(packet, broadcastAddr, 9);
- await Future.delayed(const Duration(milliseconds: 100));
- socket.send(packet, broadcastAddr, 9);
+ // Send to all targets on both common WoL ports
+ for (final addr in targets) {
+ socket.send(packet, addr, 9);
+ socket.send(packet, addr, 7);
+ }
+
+ // Repeat for reliability
+ for (var i = 0; i < 3; i++) {
+ await Future.delayed(const Duration(milliseconds: 100));
+ for (final addr in targets) {
+ socket.send(packet, addr, 9);
+ }
+ }
socket.close();
}
diff --git a/lib/widgets/message_bubble.dart b/lib/widgets/message_bubble.dart
index 2d4584b..e9c4882 100644
--- a/lib/widgets/message_bubble.dart
+++ b/lib/widgets/message_bubble.dart
@@ -214,28 +214,44 @@
: message.imageBase64!,
);
- return GestureDetector(
- onTap: () {
- Navigator.of(context).push(
- MaterialPageRoute(
- builder: (_) => ImageViewer(imageBytes: bytes),
- ),
- );
- },
- child: ClipRRect(
- borderRadius: BorderRadius.circular(8),
- child: Image.memory(
- bytes,
- width: 260,
- height: 180,
- fit: BoxFit.cover,
- errorBuilder: (_, e, st) => const SizedBox(
- width: 260,
- height: 60,
- child: Center(child: Text('Image decode error')),
+ return Column(
+ crossAxisAlignment: CrossAxisAlignment.start,
+ children: [
+ GestureDetector(
+ onTap: () {
+ Navigator.of(context).push(
+ MaterialPageRoute(
+ builder: (_) => ImageViewer(imageBytes: bytes),
+ ),
+ );
+ },
+ child: ClipRRect(
+ borderRadius: BorderRadius.circular(8),
+ child: Image.memory(
+ bytes,
+ width: 260,
+ height: 180,
+ fit: BoxFit.cover,
+ errorBuilder: (_, e, st) => const SizedBox(
+ width: 260,
+ height: 60,
+ child: Center(child: Text('Image decode error')),
+ ),
+ ),
),
),
- ),
+ if (message.content.isNotEmpty) ...[
+ const SizedBox(height: 6),
+ Text(
+ message.content,
+ style: TextStyle(
+ fontSize: 14,
+ color: _isUser ? Colors.white.withAlpha(220) : null,
+ height: 1.3,
+ ),
+ ),
+ ],
+ ],
);
}
--
Gitblit v1.3.1