Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions Sources/VoxClawCore/Network/CloudSpeechRelay.swift
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public actor CloudSpeechRelay {
private let zoneID: CKRecordZone.ID
private let log = Logger(subsystem: "com.malpern.voxclaw", category: "CloudSpeechRelay")

/// Set once the custom zone has been created this session, so steady-state
/// `send()`/`ensureSubscription()` calls skip the redundant zone-save round-trip.
private var zoneEnsured = false

public init(containerID: String = CloudSpeechRelay.defaultContainerID) {
self.container = CKContainer(identifier: containerID)
self.database = container.privateCloudDatabase
Expand All @@ -44,7 +48,9 @@ public actor CloudSpeechRelay {
/// zone is a no-op). Database subscriptions only fire for custom zones, so
/// both the sender and the receiver must ensure it exists.
private func ensureZone() async throws {
if zoneEnsured { return }
_ = try await database.save(CKRecordZone(zoneID: zoneID))
zoneEnsured = true
}

/// Human-readable iCloud account status for diagnostics ("available",
Expand Down Expand Up @@ -78,6 +84,10 @@ public actor CloudSpeechRelay {
public var projectId: String?
public var agentId: String?
public var engine: VoiceEngineType?
/// The sender's timestamp for the record. Set on fetch so the receiver can
/// advance its dedup watermark to the newest record it actually saw,
/// rather than its own wall clock (which drifts vs the sender).
public var sentAt: Date?

public init(
text: String,
Expand All @@ -86,7 +96,8 @@ public actor CloudSpeechRelay {
instructions: String? = nil,
projectId: String? = nil,
agentId: String? = nil,
engine: VoiceEngineType? = nil
engine: VoiceEngineType? = nil,
sentAt: Date? = nil
) {
self.text = text
self.voice = voice
Expand All @@ -95,6 +106,7 @@ public actor CloudSpeechRelay {
self.projectId = projectId
self.agentId = agentId
self.engine = engine
self.sentAt = sentAt
}
}

Expand Down Expand Up @@ -159,7 +171,8 @@ public actor CloudSpeechRelay {
instructions: record["instructions"] as? String,
projectId: record["projectId"] as? String,
agentId: record["agentId"] as? String,
engine: (record["engine"] as? String).flatMap(VoiceEngineType.init(rawValue:))
engine: (record["engine"] as? String).flatMap(VoiceEngineType.init(rawValue:)),
sentAt: record["sentAt"] as? Date
)
)
}
Expand Down
18 changes: 13 additions & 5 deletions Sources/VoxClawCore/Settings/SettingsManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -658,20 +658,28 @@ public final class SettingsManager {
/// to reproduce the sender's per-agent voice verbatim (the sender owns the
/// agent→voice mapping; receivers must not re-derive it). Falls back to Apple
/// when the requested engine isn't configured on this device.
public func makeRelayEngine(engine: VoiceEngineType, voice: String?) -> any SpeechEngine {
let instructions = readingStyle.isEmpty ? nil : readingStyle
public func makeRelayEngine(
engine: VoiceEngineType,
voice: String?,
rate: Float? = nil,
instructions: String? = nil
) -> any SpeechEngine {
// Honor the sender's rate/prosody so the agent sounds identical across
// devices; fall back to this device's settings when the request omits them.
let effectiveRate = rate ?? voiceSpeed
let effectiveInstructions = instructions ?? (readingStyle.isEmpty ? nil : readingStyle)
let apple = AppleSpeechEngine(
voiceIdentifier: (engine == .apple ? voice : nil) ?? appleVoiceIdentifier,
rate: voiceSpeed
rate: effectiveRate
)
switch engine {
case .apple:
return apple
case .openai where isOpenAIConfigured:
let primary = OpenAISpeechEngine(apiKey: openAIAPIKey, voice: voice ?? openAIVoice, speed: voiceSpeed, instructions: instructions)
let primary = OpenAISpeechEngine(apiKey: openAIAPIKey, voice: voice ?? openAIVoice, speed: effectiveRate, instructions: effectiveInstructions)
return FallbackSpeechEngine(primary: primary, fallback: apple)
case .elevenlabs where isElevenLabsConfigured:
let primary = ElevenLabsSpeechEngine(apiKey: elevenLabsAPIKey, voiceID: voice ?? elevenLabsVoiceID, speed: voiceSpeed, turbo: elevenLabsTurbo)
let primary = ElevenLabsSpeechEngine(apiKey: elevenLabsAPIKey, voiceID: voice ?? elevenLabsVoiceID, speed: effectiveRate, turbo: elevenLabsTurbo)
return FallbackSpeechEngine(primary: primary, fallback: apple)
default:
return apple
Expand Down
45 changes: 45 additions & 0 deletions VoxClawIOS/VoxClawIOS/ReadAloudIntent.swift
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import AppIntents
import VoxClawCore

/// Globally-reachable iOS app state shared between the SwiftUI app and any code
Expand All @@ -8,3 +9,47 @@ enum SharedIOSApp {
static let settings = SettingsManager()
static let coordinator = iOSCoordinator()
}

/// "Read Text Aloud" — exposes VoxClaw to Siri, Spotlight, and Shortcuts on iOS.
/// (macOS has its own `ReadTextIntent` in VoxClawCore; this is the iOS counterpart.)
struct ReadTextIntent: AppIntent {
static let title: LocalizedStringResource = "Read Text Aloud"
static let description = IntentDescription(
"Reads the provided text aloud using VoxClaw's text-to-speech.",
categoryName: "Reading"
)
// Bring the app forward so the audio session is active and the teleprompter shows.
static let openAppWhenRun = true

@Parameter(title: "Text to Read")
var text: String

static var parameterSummary: some ParameterSummary {
Summary("Read \(\.$text) aloud")
}

@MainActor
func perform() async throws -> some IntentResult {
await SharedIOSApp.coordinator.readText(
text,
appState: SharedIOSApp.appState,
settings: SharedIOSApp.settings
)
return .result()
}
}

/// Registers the discoverable shortcut/phrases for Siri and Spotlight.
struct VoxClawShortcuts: AppShortcutsProvider {
static var appShortcuts: [AppShortcut] {
AppShortcut(
intent: ReadTextIntent(),
phrases: [
"Read with \(.applicationName)",
"Read text using \(.applicationName)",
],
shortTitle: "Read Text",
systemImageName: "waveform"
)
}
}
22 changes: 22 additions & 0 deletions VoxClawIOS/VoxClawIOS/VoxClawIOSApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ struct VoxClawIOSApp: App {
// Bridges remote-notification callbacks (CloudKit relay wake) into the app.
@UIApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate

@Environment(\.scenePhase) private var scenePhase
@State private var lastHandledClipboardRequest: TimeInterval = 0

// App Group shared with the widget/control extension (which can't play audio).
private static let appGroup = "group.com.malpern.voxclaw"
private static let pendingReadClipboardKey = "voxclaw.pendingReadClipboard"

var body: some Scene {
WindowGroup {
ContentView(appState: appState, settings: settings, coordinator: coordinator)
Expand All @@ -36,12 +43,27 @@ struct VoxClawIOSApp: App {
UIApplication.shared.registerForRemoteNotifications()
Task { await coordinator.ensureCloudRelay(settings: settings) }
}
.onChange(of: scenePhase) { _, phase in
if phase == .active { handlePendingClipboardRead() }
}
// The "Now reading" Live Activity is driven by iOSCoordinator (not
// here), so it also runs when the app is woken in the background by
// a relay push — the view's observers don't fire when locked.
}
}

/// When the Control Center control / widget asks to read the clipboard, it
/// opens the app and sets a timestamp in the App Group; we read it here.
@MainActor
private func handlePendingClipboardRead() {
guard let defaults = UserDefaults(suiteName: Self.appGroup) else { return }
let ts = defaults.double(forKey: Self.pendingReadClipboardKey)
guard ts > lastHandledClipboardRequest else { return }
lastHandledClipboardRequest = ts
guard let text = UIPasteboard.general.string, !text.isEmpty else { return }
Task { await coordinator.readText(text, appState: appState, settings: settings) }
}

private func configureAudioSession() {
do {
let session = AVAudioSession.sharedInstance()
Expand Down
19 changes: 16 additions & 3 deletions VoxClawIOS/VoxClawIOS/iOSCoordinator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ final class iOSCoordinator: SpeechQueueDelegate {
// A relayed request carries the sender's resolved voice/engine —
// honor them so the agent sounds the same here as on the source.
let engineOverride: (any SpeechEngine)? = request.relayed
? settings.makeRelayEngine(engine: request.engine ?? settings.voiceEngine, voice: request.voice)
? settings.makeRelayEngine(
engine: request.engine ?? settings.voiceEngine,
voice: request.voice,
rate: request.rate,
instructions: request.instructions
)
: nil
self.queue.enqueue(
request.text,
Expand Down Expand Up @@ -127,7 +132,9 @@ final class iOSCoordinator: SpeechQueueDelegate {
// so the agent sounds identical across devices.
let engine = settings.makeRelayEngine(
engine: payload.engine ?? settings.voiceEngine,
voice: payload.voice
voice: payload.voice,
rate: payload.rate,
instructions: payload.instructions
)
queue.enqueue(
payload.text,
Expand All @@ -140,7 +147,13 @@ final class iOSCoordinator: SpeechQueueDelegate {
)
postLockScreenText(payload.text)
}
UserDefaults.standard.set(Date.now.timeIntervalSince1970, forKey: Self.lastCloudFetchKey)
// Advance the watermark to the newest record we actually fetched (the
// sender's clock), not this device's wall clock — using Date.now risks
// skipping records written between the fetch and now, and is sensitive
// to cross-device clock skew. Records carry their own sentAt.
if let newest = pending.compactMap(\.sentAt).max() {
UserDefaults.standard.set(newest.timeIntervalSince1970, forKey: Self.lastCloudFetchKey)
}
} catch {
print("CloudKit fetch failed: \(error)")
}
Expand Down
84 changes: 84 additions & 0 deletions VoxClawIOS/VoxClawWidgets/VoxClawWidgets.swift
Original file line number Diff line number Diff line change
@@ -1,7 +1,89 @@
import ActivityKit
import AppIntents
import SwiftUI
import WidgetKit

// App Group shared between the app and this extension. The control/widget can't
// play audio from the extension process, so they signal the app (which reads the
// clipboard aloud when it becomes active).
enum WidgetBridge {
static let appGroup = "group.com.malpern.voxclaw"
static let pendingReadClipboardKey = "voxclaw.pendingReadClipboard"

static func requestClipboardRead() {
UserDefaults(suiteName: appGroup)?.set(Date.now.timeIntervalSince1970, forKey: pendingReadClipboardKey)
}
}

/// Reads the clipboard aloud. Opens the app (audio can't play from an extension),
/// signalling it via the shared App Group; the app performs the read on activation.
struct ReadClipboardIntent: AppIntent {
static let title: LocalizedStringResource = "Read Clipboard Aloud"
static let description = IntentDescription("Reads the clipboard aloud with VoxClaw.")
static let openAppWhenRun = true

func perform() async throws -> some IntentResult {
WidgetBridge.requestClipboardRead()
return .result()
}
}

// MARK: - Control Center control

struct ReadClipboardControl: ControlWidget {
var body: some ControlWidgetConfiguration {
StaticControlConfiguration(kind: "com.malpern.voxclaw.readClipboard") {
ControlWidgetButton(action: ReadClipboardIntent()) {
Label("Read Clipboard", systemImage: "waveform")
}
}
.displayName("Read Clipboard")
.description("Read the clipboard aloud with VoxClaw.")
}
}

// MARK: - Interactive home-screen widget

private struct WidgetEntry: TimelineEntry {
let date: Date
}

private struct WidgetProvider: TimelineProvider {
func placeholder(in context: Context) -> WidgetEntry { WidgetEntry(date: .now) }
func getSnapshot(in context: Context, completion: @escaping (WidgetEntry) -> Void) {
completion(WidgetEntry(date: .now))
}
func getTimeline(in context: Context, completion: @escaping (Timeline<WidgetEntry>) -> Void) {
completion(Timeline(entries: [WidgetEntry(date: .now)], policy: .never))
}
}

private struct ReadClipboardWidgetView: View {
var body: some View {
VStack(spacing: 8) {
Image(systemName: "waveform")
.font(.title2)
Button(intent: ReadClipboardIntent()) {
Text("Read Clipboard")
.font(.caption.weight(.semibold))
}
.buttonStyle(.borderedProminent)
}
.containerBackground(.fill.tertiary, for: .widget)
}
}

struct ReadClipboardWidget: Widget {
var body: some WidgetConfiguration {
StaticConfiguration(kind: "VoxClawReadClipboardWidget", provider: WidgetProvider()) { _ in
ReadClipboardWidgetView()
}
.configurationDisplayName("Read Clipboard")
.description("Tap to read the clipboard aloud.")
.supportedFamilies([.systemSmall])
}
}

// MARK: - Live Activity ("Now reading…")

struct VoxClawLiveActivity: Widget {
Expand Down Expand Up @@ -52,6 +134,8 @@ struct VoxClawLiveActivity: Widget {
@main
struct VoxClawWidgetBundle: WidgetBundle {
var body: some Widget {
ReadClipboardWidget()
ReadClipboardControl()
VoxClawLiveActivity()
}
}
Loading