From 1c09d121ba6a904781d7e7b58f73e200bd3ec4d5 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 00:50:39 -0600 Subject: [PATCH 1/5] wip(mobile): partial port of offline iPhone voice input (#8614) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NOT READY TO MERGE. Parked as a branch commit so the conflict resolution is not lost. Done: all five conflicts resolved, the two new patched native deps (@react-native-ai/apple, expo-audio) installed, the voice-input feature directories and client-runtime module landed intact, and ThreadComposer compiles with the controller wired (composerOwnerKey, useVoiceInputController, resolveVoiceComposerPresentation, showsCompactDictation). Also resolved use-composer-command-menu.ts Pylon-first: upstream's file is far smaller than Pylon's, so every Pylon export (buildComposerCommandItems, resolveComposerProviderSlashCommands, and the ranking helpers) is kept and only upstream's composerSelectionAtEnd helper and owner-key ref are added. Not done, and the reason this is parked: the dictation UI is imported but not rendered. ComposerDictationToolbar, ComposerDictationPrimaryAction, ComposerDictationStatus and ComposerDictationCancelAction are all unused, so the mic never appears. Wiring them means restructuring Pylon's composer rather than patching it — upstream wraps its toolbar row directly, while Pylon's is ComposerToolbarRow > ComposerToolbarScroller with thirteen controls. Pylon also declares canSend roughly 750 lines above where voiceInput can exist, so even `canSend && !voiceInput.blocksSubmission` needs the declaration order changed. Shipping it compiling-but-inert would look done and do nothing, so it waits. --- apps/mobile/app.config.ts | 9 + .../ios/T3ComposerEditorModule.swift | 3 + .../ios/T3ComposerEditorView.swift | 42 +- apps/mobile/package.json | 5 +- apps/mobile/src/components/GlassSurface.tsx | 6 +- .../features/threads/NewTaskDraftScreen.tsx | 212 ++++--- .../src/features/threads/ThreadComposer.tsx | 281 +++++---- .../features/threads/ThreadDetailScreen.tsx | 19 +- .../threads/use-composer-command-menu.test.ts | 8 +- .../threads/use-composer-command-menu.ts | 19 +- .../voice-input/ComposerDictationControl.tsx | 360 ++++++++++++ .../voice-input/useVoiceInputController.ts | 217 +++++++ .../voice-input/voiceInputMetering.test.ts | 56 ++ .../voice-input/voiceInputMetering.ts | 14 + .../voiceInputPresentation.test.ts | 69 +++ .../voice-input/voiceInputPresentation.ts | 65 +++ .../src/native/T3ComposerEditor.ios.tsx | 2 + .../src/native/T3ComposerEditor.native.tsx | 2 +- apps/mobile/src/native/T3ComposerEditor.tsx | 2 + .../src/native/T3ComposerEditor.types.ts | 2 + .../src/native/voiceTranscription.ios.test.ts | 140 +++++ .../src/native/voiceTranscription.ios.ts | 98 ++++ apps/mobile/src/native/voiceTranscription.ts | 5 + docs/README.md | 1 + docs/internals/voice-input.md | 102 ++++ docs/user/composer.md | 16 + packages/client-runtime/README.md | 30 +- packages/client-runtime/package.json | 4 + .../src/voice-input/controller.test.ts | 535 ++++++++++++++++++ .../src/voice-input/controller.ts | 493 ++++++++++++++++ .../client-runtime/src/voice-input/index.ts | 21 + .../src/voice-input/transcription.ts | 37 ++ patches/@react-native-ai__apple@0.12.0.patch | 194 +++++++ patches/expo-audio@57.0.4.patch | 24 + pnpm-lock.yaml | 70 ++- pnpm-workspace.yaml | 2 + 36 files changed, 2943 insertions(+), 222 deletions(-) create mode 100644 apps/mobile/src/features/voice-input/ComposerDictationControl.tsx create mode 100644 apps/mobile/src/features/voice-input/useVoiceInputController.ts create mode 100644 apps/mobile/src/features/voice-input/voiceInputMetering.test.ts create mode 100644 apps/mobile/src/features/voice-input/voiceInputMetering.ts create mode 100644 apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts create mode 100644 apps/mobile/src/features/voice-input/voiceInputPresentation.ts create mode 100644 apps/mobile/src/native/voiceTranscription.ios.test.ts create mode 100644 apps/mobile/src/native/voiceTranscription.ios.ts create mode 100644 apps/mobile/src/native/voiceTranscription.ts create mode 100644 docs/internals/voice-input.md create mode 100644 packages/client-runtime/src/voice-input/controller.test.ts create mode 100644 packages/client-runtime/src/voice-input/controller.ts create mode 100644 packages/client-runtime/src/voice-input/index.ts create mode 100644 packages/client-runtime/src/voice-input/transcription.ts create mode 100644 patches/@react-native-ai__apple@0.12.0.patch create mode 100644 patches/expo-audio@57.0.4.patch diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 4bf265e24..5bd798577 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -338,6 +338,15 @@ const config: ExpoConfig = { }, }, ], + [ + "expo-audio", + { + microphonePermission: "Allow T3 Code to use your microphone for voice input.", + recordAudioAndroid: false, + enableBackgroundPlayback: false, + enableBackgroundRecording: false, + }, + ], [ "expo-camera", { diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift index a56619b7d..06dab5e07 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift @@ -29,6 +29,9 @@ public class T3ComposerEditorModule: Module { Prop("editable") { (view: T3ComposerEditorView, editable: Bool) in view.setEditable(editable) } + Prop("readOnly") { (view: T3ComposerEditorView, readOnly: Bool) in + view.setReadOnly(readOnly) + } Prop("scrollEnabled") { (view: T3ComposerEditorView, scrollEnabled: Bool) in view.setScrollEnabled(scrollEnabled) } diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift index 2a8fb8c4e..fe63acc8e 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift @@ -60,10 +60,21 @@ private final class ComposerTextAttachment: NSTextAttachment { private final class ComposerTextView: UITextView { private static let pastedImageDirectoryName = "t3-composer-paste" private static let stalePastedImageAge: TimeInterval = 60 * 60 + private static let readOnlyActions = Set([ + "cut:", + "delete:", + "paste:", + "redo:", + "toggleBoldface:", + "toggleItalics:", + "toggleUnderline:", + "undo:", + ]) var onPasteImages: (([String]) -> Void)? var onAttributedMutation: (() -> Void)? var onSubmit: (() -> Void)? + var isReadOnly = false override var keyCommands: [UIKeyCommand]? { var commands = super.keyCommands ?? [] @@ -83,6 +94,9 @@ private final class ComposerTextView: UITextView { } override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool { + if isReadOnly && Self.readOnlyActions.contains(NSStringFromSelector(action)) { + return false + } if action == #selector(paste(_:)) { let pasteboard = UIPasteboard.general if pasteboard.hasImages || @@ -96,6 +110,9 @@ private final class ComposerTextView: UITextView { } override func paste(_ sender: Any?) { + guard !isReadOnly else { + return + } let pasteboard = UIPasteboard.general let imageProviders = pasteboard.itemProviders.filter { $0.canLoadObject(ofClass: UIImage.self) @@ -117,6 +134,9 @@ private final class ComposerTextView: UITextView { } override func deleteBackward() { + guard !isReadOnly else { + return + } guard selectedRange.length == 0, selectedRange.location > 0 else { super.deleteBackward() return @@ -160,9 +180,12 @@ private final class ComposerTextView: UITextView { } group.notify(queue: .main) { [weak self] in + guard let self, !self.isReadOnly else { + return + } let urls = images.compactMap { $0 }.compactMap(Self.writeTemporaryImage) if !urls.isEmpty { - self?.onPasteImages?(urls) + self.onPasteImages?(urls) } } } @@ -175,6 +198,9 @@ private final class ComposerTextView: UITextView { } override func cut(_ sender: Any?) { + guard !isReadOnly else { + return + } guard isEditable, selectedRange.length > 0 else { return super.cut(sender) } @@ -306,6 +332,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro private var contentInsetVertical: CGFloat = 0 private var shouldAutoFocus = false private var didAutoFocus = false + private var isReadOnly = false private var isApplyingControlledValue = false private var nativeEventCount = 0 private var lastContentSize = CGSize.zero @@ -451,6 +478,11 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro textView.isEditable = editable } + func setReadOnly(_ readOnly: Bool) { + isReadOnly = readOnly + textView.isReadOnly = readOnly + } + func setScrollEnabled(_ scrollEnabled: Bool) { textView.isScrollEnabled = scrollEnabled } @@ -504,13 +536,16 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro replacementText text: String ) -> Bool { restoreBaseTypingAttributes() - return true + return !isReadOnly } public func textDroppableView( _ textDroppableView: UIView & UITextDroppable, proposalForDrop drop: UITextDropRequest ) -> UITextDropProposal { + guard !isReadOnly else { + return UITextDropProposal(operation: .cancel) + } guard droppedImageProviders(in: drop) != nil else { return drop.suggestedProposal } @@ -527,6 +562,9 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro _ textDroppableView: UIView & UITextDroppable, willPerformDrop drop: UITextDropRequest ) { + guard !isReadOnly else { + return + } guard let imageProviders = droppedImageProviders(in: drop) else { return } diff --git a/apps/mobile/package.json b/apps/mobile/package.json index b540c5186..7ae9b69e8 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -53,6 +53,7 @@ "@noble/curves": "catalog:", "@noble/hashes": "catalog:", "@pierre/diffs": "catalog:", + "@react-native-ai/apple": "0.12.0", "@react-native-menu/menu": "^2.0.0", "@react-navigation/elements": "2.9.26", "@react-navigation/native": "7.3.4", @@ -73,6 +74,7 @@ "effect": "catalog:", "expo": "~57.0.18", "expo-asset": "~57.0.15", + "expo-audio": "~57.0.4", "expo-auth-session": "~57.0.10", "expo-blur": "~57.0.2", "expo-build-properties": "~57.0.15", @@ -141,7 +143,8 @@ "autolinking": { "buildFromSource": [ "react-native-screens", - "@react-native-menu/menu" + "@react-native-menu/menu", + "expo-audio" ] } }, diff --git a/apps/mobile/src/components/GlassSurface.tsx b/apps/mobile/src/components/GlassSurface.tsx index c194d6b55..577c43aa8 100644 --- a/apps/mobile/src/components/GlassSurface.tsx +++ b/apps/mobile/src/components/GlassSurface.tsx @@ -1,5 +1,5 @@ import { GlassView, isGlassEffectAPIAvailable } from "expo-glass-effect"; -import type { ReactNode } from "react"; +import type { ReactNode, Ref } from "react"; import { Platform, useColorScheme, @@ -20,6 +20,7 @@ const ThemedGlassView = withUniwind(GlassView, { }); interface GlassSurfaceProps extends ViewProps { + readonly ref?: Ref; readonly children: ReactNode; readonly glassEffectStyle?: "clear" | "regular" | "none"; readonly tintColor?: ColorValue; @@ -32,6 +33,7 @@ interface GlassSurfaceProps extends ViewProps { } export function GlassSurface({ + ref, children, glassEffectStyle = "regular", chrome = "default", @@ -68,6 +70,7 @@ export function GlassSurface({ return ( ) : null} {promptEditor} + - - - { - if (selectedEnvironmentServerConfig?.environment.capabilities.fileAttachments) { - Alert.alert("Add attachment", undefined, [ - { text: "Photos", onPress: () => void handlePickImages() }, - { text: "Files", onPress: () => void handlePickFiles() }, - { text: "Cancel", style: "cancel" }, - ]); - return; - } - void handlePickImages(); - }} - showChevron={false} - /> - - } - label={flow.selectedModelOption?.label ?? "Choose model"} - maxWidth={152} - onPress={settingsSheetPresentation.open} - /> - {flow.planModeEnabled ? ( - - flow.setInteractionMode(flow.interactionMode === "plan" ? "default" : "plan") - } - showChevron={false} + + + + - ) : null} - - void handleStart()} - showChevron={false} - variant="primary" - /> - + {isVoiceInputPresented ? ( + + ) : ( + + { + if ( + selectedEnvironmentServerConfig?.environment.capabilities.fileAttachments + ) { + Alert.alert("Add attachment", undefined, [ + { text: "Photos", onPress: () => void handlePickImages() }, + { text: "Files", onPress: () => void handlePickFiles() }, + { text: "Cancel", style: "cancel" }, + ]); + return; + } + void handlePickImages(); + }} + showChevron={false} + /> + + } + label={flow.selectedModelOption?.label ?? "Choose model"} + maxWidth={152} + onPress={settingsSheetPresentation.open} + /> + {flow.planModeEnabled ? ( + + flow.setInteractionMode( + flow.interactionMode === "plan" ? "default" : "plan", + ) + } + showChevron={false} + /> + ) : null} + + )} + + {voicePresentation.showsSend ? ( + void handleStart()} + showChevron={false} + variant="primary" + /> + ) : null} + + + ); @@ -1230,10 +1297,17 @@ export function NewTaskDraftScreen(props: { {heroViewport} - {composerDock} + + {composerDock} + ); diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 8c89e64ec..3026f0c39 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -79,6 +79,7 @@ import Animated, { FadeOut, FadeOutDown, LinearTransition, + ReduceMotion, } from "react-native-reanimated"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { presentMobileContextWindow } from "../../lib/contextWindow"; @@ -111,6 +112,15 @@ import type { RemoteClientConnectionState } from "../../lib/connection"; import { resolveProviderOptionDescriptors } from "../../lib/providerOptions"; import { ComposerCommandPopover } from "./ComposerCommandPopover"; import { useComposerCommandMenu } from "./use-composer-command-menu"; +import { + ComposerDictationCancelAction, + ComposerDictationDraftContent, + ComposerDictationPrimaryAction, + ComposerDictationStatus, + ComposerDictationToolbar, +} from "../voice-input/ComposerDictationControl"; +import { useVoiceInputController } from "../voice-input/useVoiceInputController"; +import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; import { type ExistingThreadSettingsRouteSession, useExistingThreadSettingsRoutePresentation, @@ -246,12 +256,14 @@ export const COMPOSER_TRANSITION_DURATION_MS = 220; export const COMPOSER_LAYOUT_TRANSITION = Platform.OS === "android" ? undefined - : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS); + : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS).reduceMotion(ReduceMotion.System); + +const AnimatedGlassSurface = Animated.createAnimatedComponent(GlassSurface); export function ComposerSurface(props: { readonly children: ReactNode; readonly style: ViewStyle; - /** Existing thread composers morph between pill and card layouts. */ + /** Morphs between the compact and expanded composer layouts. */ readonly animateLayout?: boolean; }) { // Drop shadow lives on a wrapper: `overflow: "hidden"` on the surface itself @@ -1179,8 +1191,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer ); // ── Composer command menu ──────────────────────────────── + const composerOwnerKey = scopedThreadKey(props.environmentId, props.selectedThread.id); + const composerMenu = useComposerCommandMenu({ draftMessage: props.draftMessage, + ownerKey: composerOwnerKey, environmentId: props.environmentId, projectCwd: props.projectCwd, selectedProviderStatus, @@ -1190,6 +1205,21 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer onChangeDraftMessage: props.onChangeDraftMessage, onUpdateInteractionMode: props.onUpdateInteractionMode, }); + const voiceInput = useVoiceInputController({ + ownerKey: composerOwnerKey, + draftMessage: props.draftMessage, + selection: composerMenu.selection, + onChangeDraftMessage: props.onChangeDraftMessage, + onChangeSelection: composerMenu.onSelectionChange, + }); + const voicePresentation = resolveVoiceComposerPresentation( + voiceInput.state, + voiceInput.elapsedSeconds, + ); + const isVoiceInputPresented = voicePresentation.statusLabel !== null; + // An open draft stays visible; only a collapsed composer becomes a voice strip. + const showsCompactDictation = isVoiceInputPresented && !isExpanded; + const isToolbarVisible = isExpanded || isVoiceInputPresented; const { onSendMessage } = props; @@ -1222,6 +1252,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer props.environmentLabel, props.selectedThread.id, props.selectedThread.title, + voiceInput.blocksSubmission, ]); const handleQueueFollowUp = useCallback(async () => { if (!canSend) return; @@ -1387,7 +1418,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer }), [currentModelOption?.capabilities, currentModelSelection.options], ); - const settingsOwnerId = scopedThreadKey(props.environmentId, props.selectedThread.id); + const settingsOwnerId = composerOwnerKey; const settingsRouteSession = useMemo( () => ({ ownerId: settingsOwnerId, @@ -1478,7 +1509,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer layout={COMPOSER_LAYOUT_TRANSITION} style={{ maxWidth: props.contentMaxWidth }} > - {composerMenu.trigger && composerMenu.items.length > 0 ? ( + {!voiceInput.isBusy && composerMenu.trigger && composerMenu.items.length > 0 ? ( - {/* Attachment strip — inside the card, above the text input */} - {isExpanded ? ( - 0 ? "pb-2.5" : undefined} - entering={FadeIn.duration(160)} - exiting={FadeOut.duration(120)} - > - + {isExpanded ? ( + 0 ? "pb-2.5" : undefined} + entering={FadeIn.duration(160)} + exiting={FadeOut.duration(120)} + layout={COMPOSER_LAYOUT_TRANSITION} + > + undefined : props.onRemoveDraftImage} + onPressImage={voiceInput.isBusy ? undefined : onPressImage} + /> + + ) : null} + + void props.onNativePasteImages(uris)} + placeholder={props.placeholder} + onFocus={handleFocus} + onBlur={handleBlur} + onSubmit={handleSend} + scrollEnabled={isExpanded} + // Android: collapsed single line centers natively (gravity) in + // a pill-height box matching the send button; iOS keeps insets. + singleLineCentered={!isExpanded} + contentInsetVertical={isExpanded || Platform.OS === "android" ? 0 : 6} + style={ + isExpanded + ? { + minHeight: 72, + maxHeight: 160, + paddingHorizontal: 4, + paddingVertical: 4, + } + : { + height: 36, + } + } + textStyle={{ + ...bodyText, + color: foregroundColor, + }} /> - - ) : null} - - - void props.onNativePasteImages(uris)} - placeholder={props.placeholder} - onFocus={handleFocus} - onBlur={handleBlur} - onSubmit={handleSend} - scrollEnabled={isExpanded} - // Android: collapsed single line centers natively (gravity) in - // a pill-height box matching the send button; iOS keeps insets. - singleLineCentered={!isExpanded} - contentInsetVertical={isExpanded || Platform.OS === "android" ? 0 : 6} - style={ - isExpanded - ? { - minHeight: 72, - maxHeight: 160, - paddingHorizontal: 4, - paddingVertical: 4, - } - : { - height: 36, - } - } - textStyle={{ - ...bodyText, - color: foregroundColor, - }} - /> - - {!isExpanded && props.draftAttachments.length > 0 ? ( - - {props.draftAttachments.slice(0, 3).map((attachment) => - attachment.type === "image" ? ( - onPressImage(attachment.previewUri)} - > - - - ) : ( - - + {!isExpanded && props.draftAttachments.length > 0 ? ( + + {props.draftAttachments.slice(0, 3).map((attachment) => + attachment.type === "image" ? ( + onPressImage(attachment.previewUri)} + > + + + ) : ( + + + + ), + )} + {props.draftAttachments.length > 3 ? ( + + + +{props.draftAttachments.length - 3} + + + ) : null} + + ) : null} + {!isExpanded && props.contextWindow ? ( + + ) : null} + {!isExpanded ? ( + + {showStopAction ? ( + + + {canQueueFollowUp ? ( + + ) : null} - ), - )} - {props.draftAttachments.length > 3 ? ( - - - +{props.draftAttachments.length - 3} - - - ) : null} - - ) : null} - {!isExpanded && props.contextWindow ? ( - - ) : null} - {!isExpanded ? ( - - {showStopAction ? ( - + ) : ( - {canQueueFollowUp ? ( - - ) : null} - - ) : ( - - )} - - ) : null} + )} + + ) : null} + {isExpanded ? ( diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index aa9cae842..dfdf88fcc 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -334,6 +334,14 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread const selectedThreadKeyRef = useRef(selectedThreadKey); const lastScrolledSubmittedMessageIdRef = useRef(null); const [composerExpanded, setComposerExpanded] = useState(false); + const [composerFocused, setComposerFocused] = useState(false); + const handleComposerFocusChange = useCallback( + (focused: boolean) => { + setComposerFocused(focused); + handleOwnedInputFocusChange(focused); + }, + [handleOwnedInputFocusChange], + ); const [anchorMessageId, setAnchorMessageId] = useState(null); const [submittedMessageId, setSubmittedMessageId] = useState(null); const [endFollowEnabled, setEndFollowEnabled] = useState(true); @@ -344,7 +352,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread // animation, so the composer would ride down flush to the screen edge and // then snap up into the inset. On iOS blur precedes the hide, so the // focus-keyed inset is already in place while the composer rides down. - const composerBottomInset = (Platform.OS === "android" ? isKeyboardVisible : composerExpanded) + // Dictation keeps that focus while the composer switches to its compact pill. + const composerBottomInset = ( + Platform.OS === "android" ? isKeyboardVisible : composerExpanded || composerFocused + ) ? 0 : Math.max(insets.bottom, 12); const contentPresentationKind = props.contentPresentation.kind; @@ -596,7 +607,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread useLayoutEffect(() => { selectedThreadKeyRef.current = selectedThreadKey; - }, [selectedThreadKey]); + // A replaced or unmounted native editor may not emit a blur event. + setComposerFocused(false); + }, [selectedThreadKey, showContent]); useEffect(() => { setAnchorMessageId(null); @@ -960,7 +973,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread onUpdateRuntimeMode={props.onUpdateThreadRuntimeMode} onUpdateInteractionMode={props.onUpdateThreadInteractionMode} onExpandedChange={setComposerExpanded} - onEditorFocusChange={handleOwnedInputFocusChange} + onEditorFocusChange={handleComposerFocusChange} /> diff --git a/apps/mobile/src/features/threads/use-composer-command-menu.test.ts b/apps/mobile/src/features/threads/use-composer-command-menu.test.ts index 3895983f4..22e792673 100644 --- a/apps/mobile/src/features/threads/use-composer-command-menu.test.ts +++ b/apps/mobile/src/features/threads/use-composer-command-menu.test.ts @@ -14,7 +14,7 @@ vi.mock("../../state/use-composer-path-search", () => ({ import type { ComposerCommandMenuProvider } from "./use-composer-command-menu"; -const { buildComposerCommandItems, composerCommandReplacement } = +const { buildComposerCommandItems, composerCommandReplacement, composerSelectionAtEnd } = await import("./use-composer-command-menu"); function skill(overrides: Partial & { name: string }): ServerProviderSkill { @@ -231,3 +231,9 @@ describe("composerCommandReplacement", () => { ).toBe("/model "); }); }); + +describe("composerSelectionAtEnd", () => { + it("resets a changed draft owner to the new draft end", () => { + expect(composerSelectionAtEnd("queued task 🧪")).toEqual({ start: 14, end: 14 }); + }); +}); diff --git a/apps/mobile/src/features/threads/use-composer-command-menu.ts b/apps/mobile/src/features/threads/use-composer-command-menu.ts index 5bb4a10dd..54af71103 100644 --- a/apps/mobile/src/features/threads/use-composer-command-menu.ts +++ b/apps/mobile/src/features/threads/use-composer-command-menu.ts @@ -25,7 +25,7 @@ import { normalizeSearchQuery, scoreQueryMatch, } from "@t3tools/shared/searchRanking"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { ComposerEditorSelection } from "../../components/ComposerEditor"; import { useComposerPathSearch } from "../../state/use-composer-path-search"; @@ -271,6 +271,10 @@ export function composerCommandReplacement(item: ComposerCommandItem): string | } } +export function composerSelectionAtEnd(draftMessage: string): ComposerEditorSelection { + return { start: draftMessage.length, end: draftMessage.length }; +} + /** * Composer autocomplete shared by the thread composer and the unsent New task * draft. Owns the caret selection it needs for trigger detection, so callers @@ -278,6 +282,7 @@ export function composerCommandReplacement(item: ComposerCommandItem): string | */ export function useComposerCommandMenu({ draftMessage, + ownerKey, environmentId, projectCwd, selectedProviderStatus, @@ -289,6 +294,7 @@ export function useComposerCommandMenu({ onUpdateInteractionMode, }: { readonly draftMessage: string; + readonly ownerKey: string | null; readonly environmentId: EnvironmentId | null; readonly projectCwd: string | null; readonly selectedProviderStatus: ServerProvider | null; @@ -299,10 +305,8 @@ export function useComposerCommandMenu({ readonly onChangeDraftMessage: (value: string) => void; readonly onUpdateInteractionMode?: (mode: ProviderInteractionMode) => void; }) { - const [selection, setSelection] = useState(() => ({ - start: draftMessage.length, - end: draftMessage.length, - })); + const [selection, setSelection] = useState(() => composerSelectionAtEnd(draftMessage)); + const previousOwnerKeyRef = useRef(ownerKey); const onSelectionChange = useCallback((nextSelection: ComposerEditorSelection) => { setSelection(nextSelection); @@ -318,6 +322,11 @@ export function useComposerCommandMenu({ return { start, end: selectionEnd }; }); }, [draftMessage.length]); + useEffect(() => { + if (previousOwnerKeyRef.current === ownerKey) return; + previousOwnerKeyRef.current = ownerKey; + setSelection(composerSelectionAtEnd(draftMessage)); + }, [draftMessage, ownerKey]); const trigger = useMemo(() => { if (!enabled || selection.start !== selection.end) { diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx new file mode 100644 index 000000000..739825a57 --- /dev/null +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -0,0 +1,360 @@ +import type { VoiceInputPhase, VoiceInputState } from "@t3tools/client-runtime/voice-input"; +import { memo, useCallback, useLayoutEffect, useState, type ReactNode } from "react"; +import { + ActivityIndicator, + Linking, + Platform, + Pressable, + View, + type LayoutChangeEvent, +} from "react-native"; +import Animated, { + Easing, + FadeIn, + FadeOut, + LinearTransition, + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, + type SharedValue, +} from "react-native-reanimated"; + +import { AppText as Text } from "../../components/AppText"; +import { SymbolView, type AppSymbolName } from "../../components/AppSymbol"; +import { cn } from "../../lib/cn"; +import type { VoiceComposerPresentation } from "./voiceInputPresentation"; +import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; + +const DICTATION_TIMING = { + duration: 220, + easing: Easing.out(Easing.cubic), + reduceMotion: ReduceMotion.System, +} as const; +const DICTATION_LAYOUT = + Platform.OS === "android" + ? undefined + : LinearTransition.duration(DICTATION_TIMING.duration).reduceMotion(ReduceMotion.System); +const DICTATION_ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System); +const DICTATION_EXITING = FadeOut.duration(120).reduceMotion(ReduceMotion.System); +const WAVEFORM_BAR_HEIGHT = 32; +const WAVEFORM_MIN_BAR_HEIGHT = 2; +const WAVEFORM_BAR_SPACING = 5; +const WAVEFORM_TIMING = { + duration: 100, + easing: Easing.out(Easing.quad), + reduceMotion: ReduceMotion.System, +} as const; + +/** Keeps the native editor mounted when compact dictation replaces the draft area. */ +export function ComposerDictationDraftContent(props: { + readonly children: ReactNode; + readonly className?: string; + readonly collapsed: boolean; +}) { + const visibility = useSharedValue(props.collapsed ? 0 : 1); + useLayoutEffect(() => { + visibility.value = withTiming(props.collapsed ? 0 : 1, DICTATION_TIMING); + }, [props.collapsed, visibility]); + const animatedStyle = useAnimatedStyle(() => ({ + opacity: visibility.value, + transform: [{ translateY: -4 * (1 - visibility.value) }], + })); + + return ( + + {props.children} + + ); +} + +/** Crossfades controls within one toolbar row while the draft keeps its position. */ +export function ComposerDictationToolbar(props: { + readonly children: ReactNode; + readonly showsDictation: boolean; + readonly visible?: boolean; +}) { + return ( + + + {props.children} + + + ); +} + +const WaveformBar = memo(function WaveformBar(props: { + readonly audioLevels: SharedValue; + readonly sampleIndex: number; +}) { + const { audioLevels, sampleIndex } = props; + const animatedStyle = useAnimatedStyle(() => { + const level = audioLevels.value[sampleIndex] ?? 0; + return { + opacity: withTiming(0.22 + level * 0.78, WAVEFORM_TIMING), + transform: [ + { + scaleY: withTiming( + (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) / + WAVEFORM_BAR_HEIGHT, + WAVEFORM_TIMING, + ), + }, + ], + }; + }); + + return ( + + ); +}); + +const VoiceWaveform = memo(function VoiceWaveform(props: { + readonly audioLevels: SharedValue; +}) { + const [barCount, setBarCount] = useState(0); + const handleLayout = useCallback((event: LayoutChangeEvent) => { + setBarCount( + Math.max( + 1, + Math.min( + VOICE_WAVEFORM_SAMPLE_COUNT, + Math.floor(event.nativeEvent.layout.width / WAVEFORM_BAR_SPACING), + ), + ), + ); + }, []); + + return ( + + {Array.from({ length: barCount }, (_, index) => ( + + ))} + + ); +}); + +function VoiceActionButton(props: { + readonly accessibilityLabel: string; + readonly disabled?: boolean; + readonly icon: AppSymbolName; + readonly loading?: boolean; + readonly onPress: () => void; + readonly variant?: "plain" | "primary"; +}) { + const variant = props.variant ?? "plain"; + const loadingVisibility = useSharedValue(props.loading ? 1 : 0); + useLayoutEffect(() => { + loadingVisibility.value = withTiming(props.loading ? 1 : 0, DICTATION_TIMING); + }, [loadingVisibility, props.loading]); + const primaryStyle = useAnimatedStyle(() => ({ opacity: 1 - loadingVisibility.value })); + + return ( + + + {variant === "primary" ? ( + + ) : null} + + {props.loading ? ( + + ) : ( + + )} + + + + ); +} + +export function ComposerDictationStatus(props: { + readonly audioLevels: SharedValue; + readonly elapsedSeconds: number; + readonly phase: VoiceInputPhase; + readonly presentation: VoiceComposerPresentation; + readonly onDismissError: () => void; +}) { + const recordingVisibility = useSharedValue(props.phase === "recording" ? 1 : 0); + useLayoutEffect(() => { + recordingVisibility.value = withTiming(props.phase === "recording" ? 1 : 0, DICTATION_TIMING); + }, [props.phase, recordingVisibility]); + const waveformStyle = useAnimatedStyle(() => ({ + opacity: recordingVisibility.value, + transform: [{ translateY: -4 * (1 - recordingVisibility.value) }], + })); + const labelStyle = useAnimatedStyle(() => ({ + opacity: 1 - recordingVisibility.value, + transform: [{ translateY: 4 * recordingVisibility.value }], + })); + + if (!props.presentation.statusLabel) return null; + const isError = props.presentation.statusKind === "error"; + const elapsedLabel = `${Math.floor(props.elapsedSeconds / 60)}:${String(props.elapsedSeconds % 60).padStart(2, "0")}`; + return ( + + {isError ? ( + + + {props.presentation.statusLabel} + + + + + + ) : ( + + + + + {elapsedLabel} + + + + + {props.presentation.statusLabel} + + + + )} + + ); +} + +export function ComposerDictationCancelAction(props: { + readonly presentation: VoiceComposerPresentation; + readonly onCancel: () => void; +}) { + if (props.presentation.leadingAction !== "cancel") return null; + return ( + + ); +} + +export function ComposerDictationPrimaryAction(props: { + readonly state: VoiceInputState; + readonly presentation: VoiceComposerPresentation; + readonly isAvailable: boolean; + readonly disabled?: boolean; + readonly onStart: () => void; + readonly onConfirm: () => void; + readonly onCancel: () => void; +}) { + if (props.presentation.trailingAction === "confirm") { + return ( + + ); + } + + if (!props.isAvailable) return null; + const openSettings = props.state.phase === "error" && props.state.errorAction === "settings"; + return ( + { + props.onCancel(); + void Linking.openSettings(); + } + : props.onStart + } + /> + ); +} diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts new file mode 100644 index 000000000..2170ff255 --- /dev/null +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -0,0 +1,217 @@ +import { + RecordingPresets, + requestRecordingPermissionsAsync, + setAudioModeAsync, + setIsAudioActiveAsync, + useAudioRecorder, + type RecordingStatus, +} from "expo-audio"; +import { File } from "expo-file-system"; +import { useFocusEffect } from "@react-navigation/native"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { AppState } from "react-native"; +import { useSharedValue } from "react-native-reanimated"; + +import type { ComposerEditorSelection } from "../../components/ComposerEditor"; +import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + voiceInputFreezesEditor, + type VoiceDraftSnapshot, + type VoiceInputState, +} from "@t3tools/client-runtime/voice-input"; +import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; + +const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const VOICE_METERING_INTERVAL_MS = 80; +const VOICE_RECORDING_OPTIONS = { + ...RecordingPresets.HIGH_QUALITY, + isMeteringEnabled: true, +}; + +async function releaseVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + // Expo does not deactivate AVAudioSession when recording stops or its + // category changes. Explicit deactivation resumes interrupted app audio. + await setIsAudioActiveAsync(false); + } +} + +async function configureVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ + allowsRecording: true, + interruptionMode: "doNotMix", + playsInSilentMode: true, + shouldPlayInBackground: false, + }); + await setIsAudioActiveAsync(true); + } catch (error) { + try { + await releaseVoiceRecordingAudio(); + } catch { + // Keep the setup error. The controller has not started a recorder yet. + } + throw error; + } +} + +export function useVoiceInputController(input: { + readonly ownerKey: string | null; + readonly draftMessage: string; + readonly selection: ComposerEditorSelection; + readonly disabled?: boolean; + readonly onChangeDraftMessage: (value: string) => void; + readonly onChangeSelection: (selection: ComposerEditorSelection) => void; +}) { + const [state, setState] = useState(INITIAL_STATE); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const elapsedSecondsRef = useRef(0); + const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); + const audioLevels = useSharedValue(audioLevelsRef.current); + const controllerRef = useRef(null); + const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage }); + const revisionRef = useRef(0); + if ( + previousDraftRef.current.ownerKey !== input.ownerKey || + previousDraftRef.current.text !== input.draftMessage + ) { + previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage }; + revisionRef.current += 1; + } + const latestInputRef = useRef(input); + latestInputRef.current = input; + + const handleRecorderStatus = useCallback((status: RecordingStatus) => { + controllerRef.current?.handleRecorderStatus({ + isFinished: status.isFinished, + hasError: status.hasError || status.mediaServicesDidReset === true, + error: status.error, + url: status.url, + }); + }, []); + const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); + + if (!controllerRef.current) { + controllerRef.current = new VoiceInputController({ + recorder, + getTranscriber: getLocalVoiceTranscriber, + requestPermission: async () => { + const permission = await requestRecordingPermissionsAsync(); + return { granted: permission.granted, canAskAgain: permission.canAskAgain }; + }, + configureRecording: configureVoiceRecordingAudio, + releaseRecording: releaseVoiceRecordingAudio, + deleteRecording: (uri) => new File(uri).delete(), + readDraft: (): VoiceDraftSnapshot | null => { + const current = latestInputRef.current; + if (!current.ownerKey) return null; + return { + ownerKey: current.ownerKey, + text: current.draftMessage, + selection: current.selection, + revision: revisionRef.current, + }; + }, + commitDraft: (text, selection) => { + const current = latestInputRef.current; + current.onChangeSelection(selection); + current.onChangeDraftMessage(text); + }, + onStateChange: setState, + }); + } + + const controller = controllerRef.current; + const previousOwnerRef = useRef(input.ownerKey); + useEffect(() => { + if (previousOwnerRef.current === input.ownerKey) return; + previousOwnerRef.current = input.ownerKey; + controller.ownerChanged(); + }, [controller, input.ownerKey]); + + useFocusEffect( + useCallback( + () => () => { + controller.dispose(); + }, + [controller], + ), + ); + + useEffect(() => { + const subscription = AppState.addEventListener("change", (nextState) => { + // iOS reports `inactive` while its permission dialog is open. Only the + // real background state cancels preparation; recorder status handles + // calls and route interruptions during capture. + if (nextState === "background") controller.appMovedToBackground(); + }); + return () => subscription.remove(); + }, [controller]); + + useEffect(() => () => controller.dispose(), [controller]); + + useEffect(() => { + if (state.phase !== "preparing" && state.phase !== "recording") return; + + if (audioLevelsRef.current.some((level) => level !== 0)) { + audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); + audioLevels.value = audioLevelsRef.current; + } + if (elapsedSecondsRef.current !== 0) { + elapsedSecondsRef.current = 0; + setElapsedSeconds(0); + } + if (state.phase !== "recording") return; + + const sampleRecording = () => { + if (controller.currentState.phase !== "recording") return; + const status = recorder.getStatus(); + if (!status.isRecording) return; + + const level = normalizeVoiceInputDecibels(status.metering); + const history = audioLevelsRef.current; + if (level !== 0 || history.some((sample) => sample !== 0)) { + const nextLevels = [...history.slice(1), level]; + audioLevelsRef.current = nextLevels; + audioLevels.value = nextLevels; + } + + const nextElapsedSeconds = Math.min( + VOICE_RECORDING_LIMIT_SECONDS, + Math.max(0, Math.floor(status.durationMillis / 1_000)), + ); + if (nextElapsedSeconds !== elapsedSecondsRef.current) { + elapsedSecondsRef.current = nextElapsedSeconds; + setElapsedSeconds(nextElapsedSeconds); + } + }; + + sampleRecording(); + const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); + return () => clearInterval(intervalId); + }, [audioLevels, controller, recorder, state.phase]); + + const start = useCallback(() => { + if (!latestInputRef.current.disabled) void controller.start(); + }, [controller]); + const stop = useCallback(() => controller.stop(), [controller]); + const cancel = useCallback(() => controller.cancel(), [controller]); + + return { + isAvailable: getLocalVoiceTranscriber() !== null, + state, + audioLevels, + elapsedSeconds, + isBusy: voiceInputBlocksSubmission(state), + freezesEditor: voiceInputFreezesEditor(state), + blocksSubmission: voiceInputBlocksSubmission(state), + start, + stop, + cancel, + }; +} diff --git a/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts b/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts new file mode 100644 index 000000000..05356eaee --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { normalizeVoiceInputDecibels } from "./voiceInputMetering"; + +describe("normalizeVoiceInputDecibels", () => { + it.each([undefined, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])( + "treats a missing or invalid reading %s as silence", + (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(0); + }, + ); + + it.each([-160, -90, -60])("keeps a reading at or below the noise floor %s silent", (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(0); + }); + + it("keeps quiet background readings close to the baseline", () => { + const quiet = normalizeVoiceInputDecibels(-50); + expect(quiet).toBeGreaterThan(0); + expect(quiet).toBeLessThan(0.05); + }); + + it("keeps loud negative speech readings distinct below full height", () => { + const levels = [-20, -18, -12, -6, -3].map(normalizeVoiceInputDecibels); + + for (const level of levels) { + expect(level).toBeGreaterThan(0); + expect(level).toBeLessThan(1); + } + expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true); + }); + + it("makes near-speech changes visible without an early ceiling", () => { + expect(normalizeVoiceInputDecibels(-6) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan( + 0.18, + ); + expect(normalizeVoiceInputDecibels(-3) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan(0.3); + }); + + it("increases throughout the usable microphone range", () => { + const levels = [-60, -55, -50, -40, -30, -20, -12, -6, -3, -0.001, 0].map( + normalizeVoiceInputDecibels, + ); + expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true); + }); + + it("approaches the noise floor and full scale without a jump", () => { + expect(normalizeVoiceInputDecibels(-59.999)).toBeLessThan(0.001); + expect(normalizeVoiceInputDecibels(-0.001)).toBeGreaterThan(0.999); + expect(normalizeVoiceInputDecibels(-0.001)).toBeLessThan(1); + }); + + it.each([0, 6, 160])("caps only full-scale or higher readings %s at one", (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(1); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputMetering.ts b/apps/mobile/src/features/voice-input/voiceInputMetering.ts new file mode 100644 index 000000000..06f62fc24 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputMetering.ts @@ -0,0 +1,14 @@ +export const VOICE_WAVEFORM_SAMPLE_COUNT = 64; + +const VOICE_NOISE_FLOOR_DECIBELS = -60; +const VOICE_NOISE_FLOOR_AMPLITUDE = 10 ** (VOICE_NOISE_FLOOR_DECIBELS / 20); + +/** Converts measured decibels to compressed amplitude, reserving full height for 0 dB. */ +export function normalizeVoiceInputDecibels(decibels: number | undefined) { + if (decibels === undefined || !Number.isFinite(decibels)) return 0; + if (decibels <= VOICE_NOISE_FLOOR_DECIBELS) return 0; + if (decibels >= 0) return 1; + + const amplitude = 10 ** (decibels / 20); + return Math.sqrt((amplitude - VOICE_NOISE_FLOOR_AMPLITUDE) / (1 - VOICE_NOISE_FLOOR_AMPLITUDE)); +} diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts new file mode 100644 index 000000000..caf160937 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vite-plus/test"; +import { voiceInputFreezesEditor } from "@t3tools/client-runtime/voice-input"; + +import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; + +describe("resolveVoiceComposerPresentation", () => { + it("maps voice states to stable composer actions and editor read-only state", () => { + expect( + resolveVoiceComposerPresentation({ phase: "idle", error: null, errorAction: null }, 0), + ).toEqual({ + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: null, + statusLabel: null, + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation({ phase: "preparing", error: null, errorAction: null }, 0), + ).toMatchObject({ + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusLabel: "Preparing", + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation({ phase: "recording", error: null, errorAction: null }, 64), + ).toMatchObject({ + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusLabel: "Recording 1:04", + confirmationEnabled: true, + }); + expect( + resolveVoiceComposerPresentation( + { phase: "transcribing", error: null, errorAction: null }, + 0, + ), + ).toMatchObject({ + statusLabel: "Transcribing", + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation( + { phase: "error", error: "Microphone unavailable", errorAction: "retry" }, + 0, + ), + ).toMatchObject({ + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: "error", + statusLabel: "Microphone unavailable", + }); + + expect(voiceInputFreezesEditor({ phase: "preparing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "recording", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "transcribing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "idle", error: null, errorAction: null })).toBe(false); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts new file mode 100644 index 000000000..e461e34d6 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts @@ -0,0 +1,65 @@ +import type { VoiceInputState } from "@t3tools/client-runtime/voice-input"; + +export type VoiceComposerPresentation = { + readonly leadingAction: "cancel" | null; + readonly trailingAction: "mic" | "confirm"; + readonly showsSend: boolean; + readonly statusKind: "active" | "error" | null; + readonly statusLabel: string | null; + readonly confirmationEnabled: boolean; +}; + +export function resolveVoiceComposerPresentation( + state: VoiceInputState, + elapsedSeconds: number, +): VoiceComposerPresentation { + switch (state.phase) { + case "idle": + return { + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: null, + statusLabel: null, + confirmationEnabled: false, + }; + case "error": + return { + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: "error", + statusLabel: state.error, + confirmationEnabled: false, + }; + case "preparing": + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: "Preparing", + confirmationEnabled: false, + }; + case "recording": { + const seconds = Math.max(0, Math.floor(elapsedSeconds)); + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: `Recording ${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`, + confirmationEnabled: true, + }; + } + case "transcribing": + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: "Transcribing", + confirmationEnabled: false, + }; + } +} diff --git a/apps/mobile/src/native/T3ComposerEditor.ios.tsx b/apps/mobile/src/native/T3ComposerEditor.ios.tsx index cb818245d..85decebe9 100644 --- a/apps/mobile/src/native/T3ComposerEditor.ios.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.ios.tsx @@ -62,6 +62,7 @@ interface NativeComposerEditorProps extends ViewProps { readonly lineHeight: number; readonly contentInsetVertical: number; readonly editable: boolean; + readonly readOnly: boolean; readonly scrollEnabled: boolean; readonly autoFocus: boolean; readonly autoCorrect: boolean; @@ -243,6 +244,7 @@ export function ComposerEditor({ } contentInsetVertical={contentInsetVertical} editable={props.editable ?? true} + readOnly={props.readOnly ?? false} scrollEnabled={props.scrollEnabled ?? true} autoFocus={props.autoFocus ?? false} autoCorrect={props.autoCorrect ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.native.tsx b/apps/mobile/src/native/T3ComposerEditor.native.tsx index 5f5201f3b..1a488d34f 100644 --- a/apps/mobile/src/native/T3ComposerEditor.native.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.native.tsx @@ -248,7 +248,7 @@ export function ComposerEditor({ } contentInsetVertical={contentInsetVertical} singleLineCentered={props.singleLineCentered ?? false} - editable={props.editable ?? true} + editable={(props.editable ?? true) && !(props.readOnly ?? false)} scrollEnabled={props.scrollEnabled ?? true} autoFocus={props.autoFocus ?? false} autoCorrect={props.autoCorrect ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.tsx b/apps/mobile/src/native/T3ComposerEditor.tsx index 30bc0ccee..07a409c9a 100644 --- a/apps/mobile/src/native/T3ComposerEditor.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.tsx @@ -16,6 +16,7 @@ export function ComposerEditor({ textStyle, contentInsetVertical = 0, singleLineCentered: _singleLineCentered, + readOnly = false, ...props }: ComposerEditorProps) { const inputRef = useRef(null); @@ -39,6 +40,7 @@ export function ComposerEditor({ props.onSelectionChange?.(event.nativeEvent.selection)} multiline={props.multiline ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.types.ts b/apps/mobile/src/native/T3ComposerEditor.types.ts index bfc47ed36..c8833bb4c 100644 --- a/apps/mobile/src/native/T3ComposerEditor.types.ts +++ b/apps/mobile/src/native/T3ComposerEditor.types.ts @@ -23,6 +23,8 @@ export interface ComposerEditorProps { readonly placeholder?: string; readonly autoFocus?: boolean; readonly editable?: boolean; + /** Blocks user edits while preserving focus, selection, and the software keyboard on iOS. */ + readonly readOnly?: boolean; readonly scrollEnabled?: boolean; readonly autoCorrect?: boolean; readonly spellCheck?: boolean; diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts new file mode 100644 index 000000000..b08e32cdb --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -0,0 +1,140 @@ +import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; + +const mocks = vi.hoisted(() => ({ + isAvailable: vi.fn<(locale: string) => boolean>(), + prepare: vi.fn<(locale: string) => Promise>(), + transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise>(), + readAudio: vi.fn<() => Promise>(), +})); + +vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({ + default: { + isAvailable: mocks.isAvailable, + prepare: mocks.prepare, + transcribe: mocks.transcribe, + }, +})); + +vi.mock("expo-file-system", () => ({ + File: class { + arrayBuffer = mocks.readAudio; + }, +})); + +import { getLocalVoiceTranscriber } from "./voiceTranscription.ios"; + +const audio = new ArrayBuffer(4); +const nativeTranscript: TranscriptionResult = { + duration: 2, + segments: [ + { text: " Hej", startSecond: 0, endSecond: 1 }, + { text: "världen. ", startSecond: 1, endSecond: 2 }, + ], +}; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((next) => { + resolve = next; + }); + return { promise, resolve }; +} + +beforeEach(() => { + vi.resetAllMocks(); + mocks.isAvailable.mockReturnValue(true); + mocks.prepare.mockResolvedValue("sv-SE"); + mocks.readAudio.mockResolvedValue(audio); + mocks.transcribe.mockResolvedValue(nativeTranscript); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("getLocalVoiceTranscriber", () => { + it("keeps the selected language and Apple's resolved locale when the device language changes", async () => { + const resolvedOptions = Intl.DateTimeFormat().resolvedOptions(); + const deviceLocale = vi + .spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions") + .mockReturnValue({ ...resolvedOptions, locale: "sv-FI" }); + const transcriber = getLocalVoiceTranscriber()!; + const options = { signal: new AbortController().signal }; + + deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "de-DE" }); + const prepared = await transcriber.prepare(options); + deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "en-US" }); + + await expect(prepared.transcribe("file:///voice.m4a", options)).resolves.toBe("Hej världen."); + expect(mocks.prepare).toHaveBeenCalledWith("sv-FI"); + expect(prepared.locale).toBe("sv-SE"); + expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE"); + }); + + it("does not start native transcription after cancellation during a file read", async () => { + const enteredRead = deferred(); + const readResult = deferred(); + mocks.readAudio.mockImplementation(() => { + enteredRead.resolve(); + return readResult.promise; + }); + const controller = new AbortController(); + const options = { signal: controller.signal }; + const prepared = await getLocalVoiceTranscriber()!.prepare(options); + const result = prepared + .transcribe("file:///voice.m4a", options) + .catch((error: unknown) => error); + + await enteredRead.promise; + controller.abort(); + readResult.resolve(audio); + + const error = await result; + expect(error).toBeInstanceOf(VoiceTranscriptionError); + expect(error).toMatchObject({ code: "cancelled" }); + expect(mocks.transcribe).not.toHaveBeenCalled(); + }); + + it.each(["prepare", "transcribe"] as const)( + "waits for native %s to finish before settling cancellation", + async (phase) => { + const enteredNative = deferred(); + const finishNative = deferred(); + if (phase === "prepare") { + mocks.prepare.mockImplementation(async () => { + enteredNative.resolve(); + await finishNative.promise; + return "sv-SE"; + }); + } else { + mocks.transcribe.mockImplementation(async () => { + enteredNative.resolve(); + await finishNative.promise; + return nativeTranscript; + }); + } + const controller = new AbortController(); + const options = { signal: controller.signal }; + const transcriber = getLocalVoiceTranscriber()!; + const operation = + phase === "prepare" + ? transcriber.prepare(options) + : (await transcriber.prepare(options)).transcribe("file:///voice.m4a", options); + const settled = vi.fn((value: unknown) => value); + const result = operation.then(settled, settled); + + await enteredNative.promise; + controller.abort(); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).not.toHaveBeenCalled(); + finishNative.resolve(); + + const error = await result; + expect(error).toBeInstanceOf(VoiceTranscriptionError); + expect(error).toMatchObject({ code: "cancelled" }); + }, + ); +}); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts new file mode 100644 index 000000000..216b9e958 --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -0,0 +1,98 @@ +import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription"; +import { File } from "expo-file-system"; + +import { + VoiceTranscriptionError, + throwIfVoiceTranscriptionAborted, + type PreparedVoiceTranscription, + type VoiceTranscriber, + type VoiceTranscriptionOptions, +} from "@t3tools/client-runtime/voice-input"; + +function getDeviceLocale(): string { + return Intl.DateTimeFormat().resolvedOptions().locale; +} + +function wrapError( + code: "preparation-failed" | "transcription-failed", + message: string, + cause: unknown, +): VoiceTranscriptionError { + if (cause instanceof VoiceTranscriptionError) { + return cause; + } + + return new VoiceTranscriptionError(code, message, { cause }); +} + +function getNativeErrorCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) { + return undefined; + } + + return typeof error.code === "string" ? error.code : undefined; +} + +export function getLocalVoiceTranscriber(): VoiceTranscriber | null { + const locale = getDeviceLocale(); + if (!AppleTranscription.isAvailable(locale)) return null; + return { prepare: (options) => prepareVoiceTranscription(locale, options) }; +} + +async function prepareVoiceTranscription( + locale: string, + { signal }: VoiceTranscriptionOptions, +): Promise { + throwIfVoiceTranscriptionAborted(signal); + if (!AppleTranscription.isAvailable(locale)) { + throw new VoiceTranscriptionError( + "unavailable", + "Voice transcription requires a supported device with iOS 26 or later.", + ); + } + + try { + const supportedLocale = await AppleTranscription.prepare(locale); + throwIfVoiceTranscriptionAborted(signal); + return { + locale: supportedLocale, + transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), + }; + } catch (error) { + throwIfVoiceTranscriptionAborted(signal); + if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { + throw new VoiceTranscriptionError( + "unsupported-locale", + "Voice transcription does not support this device language.", + { cause: error }, + ); + } + + throw wrapError( + "preparation-failed", + "Voice transcription could not prepare this language.", + error, + ); + } +} + +async function transcribeVoiceRecording( + uri: string, + locale: string, + { signal }: VoiceTranscriptionOptions, +): Promise { + try { + throwIfVoiceTranscriptionAborted(signal); + const audio = await new File(uri).arrayBuffer(); + throwIfVoiceTranscriptionAborted(signal); + const result = await AppleTranscription.transcribe(audio, locale); + throwIfVoiceTranscriptionAborted(signal); + return result.segments + .map((segment) => segment.text) + .join(" ") + .trim(); + } catch (error) { + throwIfVoiceTranscriptionAborted(signal); + throw wrapError("transcription-failed", "Voice transcription failed.", error); + } +} diff --git a/apps/mobile/src/native/voiceTranscription.ts b/apps/mobile/src/native/voiceTranscription.ts new file mode 100644 index 000000000..e003064ae --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ts @@ -0,0 +1,5 @@ +import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; + +export function getLocalVoiceTranscriber(): VoiceTranscriber | null { + return null; +} diff --git a/docs/README.md b/docs/README.md index a16a7ce61..c3ec64036 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ policy in [CONTRIBUTING.md](../CONTRIBUTING.md); agent rules in [AGENTS.md](../A - [Glossary](./internals/glossary.md) - [Scripts](./internals/scripts.md) - [Connection runtime](./internals/connection-runtime.md) +- [Voice input](./internals/voice-input.md) - [Providers](./internals/providers.md) - [Remote environments](./internals/remote.md) - [Server updates](./internals/server-updates.md) diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md new file mode 100644 index 000000000..dd37f2a86 --- /dev/null +++ b/docs/internals/voice-input.md @@ -0,0 +1,102 @@ +# Voice input + +> For maintainers. Using T3 Code? See [voice input on iPhone](../user/composer.md#voice-input-on-iphone). + +Voice input produces editable composer text. The current implementation records on the client and +transcribes locally with Apple's `SpeechAnalyzer` and `SpeechTranscriber` on supported iOS 26+ +devices. Environment-provided transcription and transcription on web and desktop are not implemented. + +## Current boundaries + +The shared [`VoiceInputController`][controller] in `packages/client-runtime` owns preparation, +recording, transcription, cancellation, temporary-file cleanup, and insertion into the captured +draft selection. Applications import it through the [voice-input entry point][voice-input] as +`@t3tools/client-runtime/voice-input`. Its dependencies separate capture from transcription; the +controller imports neither React Native nor an Apple transcription API. + +The shared [transcription contract][transcription] defines `VoiceTranscriber`, +`PreparedVoiceTranscription`, and transcription errors. The controller calls `getTranscriber()` once +at the start of an operation, before asking for microphone permission. Preparation returns a resolved +locale and a bound `transcribe` function. The controller retains that result for the recording, so a +selection change cannot prepare with one implementation and transcribe with another. + +[`useVoiceInputController`][hook] supplies Expo audio capture, microphone permissions, audio-session +management, waveform samples, and app and navigation lifecycle handling. It normalizes Expo's +`mediaServicesDidReset` into a generic recorder error. [`voiceTranscription.ios.ts`][ios] adapts +`@react-native-ai/apple` through `getLocalVoiceTranscriber()`, capturing the requested device locale +and binding the prepared transcriber to Apple's resolved locale. The other-platform binding returns +no local transcriber. That result describes the local implementation, not whether a client could use +an environment's transcription service. + +Mobile's [`voiceInputPresentation.ts`][presentation] maps shared state to toolbar labels and actions. +Waveform and toolbar rendering stay in mobile. The composer edits draft text without selecting a +speech vendor. +Recording captures the draft owner, revision, text, and selection. A late transcript cannot overwrite +a different or edited draft. Only normal message submission sends the resulting text to an agent. + +Each operation passes one `AbortSignal` through preparation and transcription. Cancellation +invalidates the operation and aborts that signal immediately. Implementations settle their promises +only after their underlying work stops. The Apple binding checks cancellation between asynchronous +steps but cannot interrupt an in-flight native request. The controller retains its session until +that work settles, ignores its result, and cleans up the recording. + +## Ownership decisions + +The extension boundary distinguishes transcription on the client device from transcription through +the composer's environment. These constraints apply when adding selectable transcription services: + +- Local means the client device, regardless of which machine hosts the environment. A device's lack + of local recognition does not prevent it from recording audio for an environment service. +- Remote service configuration and API keys belong to the environment. The environment calls the + external service. Clients receive service identifiers, labels, and availability information, never + credential values. Transcription services are independent of coding-agent `providerInstances`; + selecting OpenAI for transcription does not select Codex for the thread. +- The client owns its transcription preference, scoped by stable `environmentId`. Its choices are + supported local recognition and the services exposed by the composer's environment. A service ID + is meaningful only within that environment. Different clients can make different choices. +- Resolve and capture the environment, selected service, and locale when an operation starts. + Preparation and transcription use the same selection; preference changes affect the next + recording. Capture environment identity explicitly rather than recovering it from a draft key. + Keep the existing draft-owner and revision checks before inserting text. +- If the selected option is unavailable, report that state and let the user choose another option. + A local failure must not silently upload audio, and a disconnected environment must not redirect + a recording to another environment or service. +- Transcription audio is temporary input, separate from durable chat attachments and messages. + Remote adapters need cancellation of upload and transcription where supported, cleanup after + success, failure, or cancellation, and the same protection against late results as local transcription. + +## Existing integration points + +[`ServerSettingsService`][settings] and [`ServerSecretStore`][secrets] provide environment-owned +configuration and secret persistence. Existing settings redaction handles coding-provider environment +variables only. Any transcription credential fields need their own explicit separation and redaction +before settings responses or subscriptions reach client caches. + +[`ExecutionEnvironmentCapabilities`][capabilities] handles version skew. Remote transcription must be +opt-in: a missing transcription capability means unsupported. The authenticated server-config +subscription and [shared environment state][server-state] already distribute configuration per +environment. A transcription service catalog belongs behind that capability and authenticated +boundary. Older servers expose no remote transcription choices. + +The [attachment upload contracts][uploads] and [shared upload operations][attachment-state] provide a +pattern for authorized binary uploads through an environment, including remote connections. Their +existing chat-attachment retention is not a transcription cleanup policy. + +Future service selection and environment requests belong alongside the controller in +`packages/client-runtime`, with wire contracts in `packages/contracts`. Capture and native local +recognition remain client-specific. An environment-backed transcriber implements the same shared +contract, with its environment and service bound when selected. The controller does not own service +credentials, provider SDKs, or transport selection. + +[controller]: ../../packages/client-runtime/src/voice-input/controller.ts +[voice-input]: ../../packages/client-runtime/src/voice-input/index.ts +[transcription]: ../../packages/client-runtime/src/voice-input/transcription.ts +[hook]: ../../apps/mobile/src/features/voice-input/useVoiceInputController.ts +[presentation]: ../../apps/mobile/src/features/voice-input/voiceInputPresentation.ts +[ios]: ../../apps/mobile/src/native/voiceTranscription.ios.ts +[settings]: ../../apps/server/src/serverSettings.ts +[secrets]: ../../apps/server/src/auth/ServerSecretStore.ts +[capabilities]: ../../packages/contracts/src/environment.ts +[server-state]: ../../packages/client-runtime/src/state/server.ts +[uploads]: ../../packages/contracts/src/assets.ts +[attachment-state]: ../../packages/client-runtime/src/state/attachments.ts diff --git a/docs/user/composer.md b/docs/user/composer.md index 00c72c658..5932e5e28 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -49,6 +49,22 @@ uploaded. Stashed files stay uploaded on the server for 24 hours. If you restore that, the file comes back with **Attach again** next to it. Attach the file again or remove it, then send. +## Voice input on iPhone + +On supported iPhones with iOS 26 or later, tap the microphone in the composer to record a message. +An expanded composer keeps your draft visible and replaces its toolbar with waves that respond to +your voice. A collapsed composer shows a compact recording strip instead. Tap the checkmark to +finish and transcribe on your device. The waves fade into a transcription status, then the usual +controls return with the text inserted at the selection where recording started. If the keyboard +is open when you start, it stays open during voice input. You can review and edit the text before +you send it. + +The first use can download Apple's speech model and needs a network connection. Later transcription +works offline for that language. A recording can be up to five minutes long. Canceling voice input, +leaving the screen, or an audio interruption discards the new recording and keeps the existing draft +and attachments. T3 Code deletes the local audio file after transcription or cancellation. It sends +only the normal message text when you submit the draft. + ## Commands and skills Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source, diff --git a/packages/client-runtime/README.md b/packages/client-runtime/README.md index 722d6f6d3..f2864a2db 100644 --- a/packages/client-runtime/README.md +++ b/packages/client-runtime/README.md @@ -5,18 +5,19 @@ subpath. The package intentionally has no root export. ## Public subpaths -| Subpath | Responsibility | -| --------------------- | ---------------------------------------------------------------- | -| `authorization` | Bearer and DPoP authorization plus token persistence contracts | -| `connection` | Targets, catalog, supervision, retries, registry, and onboarding | -| `environment` | Environment identity, descriptors, endpoints, and scoped keys | -| `errors` | Shared client error inspection | -| `operations` | Multi-step application workflows | -| `operations/projects` | Multi-step project creation workflows | -| `platform` | Platform capability and persistence service contracts | -| `relay` | Managed relay API and environment discovery | -| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions | -| `state/` | Focused shared state, retention, reducers, and Atom constructors | +| Subpath | Responsibility | +| --------------------- | ----------------------------------------------------------------- | +| `authorization` | Bearer and DPoP authorization plus token persistence contracts | +| `connection` | Targets, catalog, supervision, retries, registry, and onboarding | +| `environment` | Environment identity, descriptors, endpoints, and scoped keys | +| `errors` | Shared client error inspection | +| `operations` | Multi-step application workflows | +| `operations/projects` | Multi-step project creation workflows | +| `platform` | Platform capability and persistence service contracts | +| `relay` | Managed relay API and environment discovery | +| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions | +| `state/` | Focused shared state, retention, reducers, and Atom constructors | +| `voice-input` | Recording lifecycle, transcription contracts, and draft insertion | ## Dependency direction @@ -25,6 +26,11 @@ capabilities with `authorization`, `relay`, and `rpc` to supervise environment sessions. Independent `state` modules consume the connection registry and expose focused state or Atom constructors to application-owned runtimes. +The `voice-input` controller accepts capture callbacks and a selected `VoiceTranscriber`. +Preparation binds transcription to its implementation and resolved locale; one cancellation +signal covers both operations. Applications provide recorder events, permissions, native +transcription implementations, and presentation. + Applications should import the narrowest relevant subpath. There is no broad `state` export: use domain paths such as `state/shell`, `state/threads`, `state/terminal`, or `state/vcs`. Subpath indices and explicitly exported domain diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0171dcd4d..cff49256e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -55,6 +55,10 @@ "types": "./src/providerSkills.ts", "default": "./src/providerSkills.ts" }, + "./voice-input": { + "types": "./src/voice-input/index.ts", + "default": "./src/voice-input/index.ts" + }, "./relay": { "types": "./src/relay/index.ts", "default": "./src/relay/index.ts" diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts new file mode 100644 index 000000000..5f26b882b --- /dev/null +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -0,0 +1,535 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + resetVoiceInputGlobalsForTests, + resolveTranscriptCommit, + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, + type VoiceRecorder, +} from "./controller.ts"; +import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +class TestRecorder implements VoiceRecorder { + uri: string | null = "file:///voice.m4a"; + readonly prepareToRecordAsync = vi.fn(async () => undefined); + readonly record = vi.fn(); + readonly stop = vi.fn(async () => undefined); +} + +function preparedTranscription( + transcribe: PreparedVoiceTranscription["transcribe"] = async () => "new text", +): PreparedVoiceTranscription { + return { locale: "en-US", transcribe }; +} + +function draft(overrides: Partial = {}): VoiceDraftSnapshot { + return { + ownerKey: "environment:thread", + text: "hello world", + selection: { start: 6, end: 11 }, + revision: 1, + ...overrides, + }; +} + +function createHarness( + overrides: Partial = {}, + initialDraft = draft(), +) { + const recorder = new TestRecorder(); + let currentDraft: VoiceDraftSnapshot | null = initialDraft; + const commits: Array<{ text: string; selection: { start: number; end: number } }> = []; + const deleted: string[] = []; + const dependencies: VoiceInputControllerDependencies = { + recorder, + getTranscriber: () => ({ prepare: async () => preparedTranscription() }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => undefined, + releaseRecording: async () => undefined, + deleteRecording: (uri) => deleted.push(uri), + readDraft: () => currentDraft, + commitDraft: (text, selection) => commits.push({ text, selection }), + onStateChange: vi.fn(), + ...overrides, + }; + return { + controller: new VoiceInputController(dependencies), + recorder, + commits, + deleted, + setDraft: (next: VoiceDraftSnapshot | null) => { + currentDraft = next; + }, + }; +} + +describe("resolveTranscriptCommit", () => { + it("replaces the recorded UTF-16 selection around emoji and composer tokens", () => { + const text = "Fix 🧪 then $review please"; + const tokenStart = text.indexOf("$review"); + const captured = draft({ + text, + selection: { start: tokenStart, end: tokenStart + "$review".length }, + }); + + expect(resolveTranscriptCommit(captured, captured, "use the mobile skill", "en-US")).toEqual({ + kind: "commit", + text: "Fix 🧪 then use the mobile skill please", + selection: { start: tokenStart + "use the mobile skill".length, end: tokenStart + 20 }, + }); + }); + + it("does not replace text after the owner, text, or revision changes", () => { + const captured = draft(); + expect( + resolveTranscriptCommit(captured, draft({ ownerKey: "other" }), "text", "en-US"), + ).toEqual({ + kind: "stale", + }); + expect(resolveTranscriptCommit(captured, draft({ text: "newer" }), "text", "en-US")).toEqual({ + kind: "stale", + }); + expect(resolveTranscriptCommit(captured, draft({ revision: 2 }), "text", "en-US")).toEqual({ + kind: "stale", + }); + }); + + it("adds English spacing at empty start, middle, and end caret boundaries", () => { + const atEnd = draft({ + text: "Fix cache.", + selection: { start: "Fix cache.".length, end: "Fix cache.".length }, + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en-US")).toMatchObject({ + kind: "commit", + text: "Fix cache. Also fix tests.", + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en_US")).toMatchObject({ + kind: "commit", + text: "Fix cache. Also fix tests.", + }); + + const atStart = draft({ text: "Fix cache.", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(atStart, atStart, "First", "en-US")).toMatchObject({ + kind: "commit", + text: "First Fix cache.", + }); + + const inMiddle = draft({ text: "Fix cache.", selection: { start: 4, end: 4 } }); + expect(resolveTranscriptCommit(inMiddle, inMiddle, "also", "en-US")).toMatchObject({ + kind: "commit", + text: "Fix also cache.", + }); + }); + + it("does not add English boundary spaces to CJK or selected inline text", () => { + const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); + expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ + kind: "commit", + text: "修正キャッシュテストも", + }); + + const selected = draft({ text: "one $skill two", selection: { start: 4, end: 10 } }); + expect(resolveTranscriptCommit(selected, selected, "new", "en-US")).toMatchObject({ + kind: "commit", + text: "one new two", + }); + }); +}); + +describe("VoiceInputController", () => { + beforeEach(() => resetVoiceInputGlobalsForTests()); + + it("checks support and permission before recording", async () => { + const unsupported = createHarness({ getTranscriber: () => null }); + await unsupported.controller.start(); + expect(unsupported.controller.currentState.error).toContain("not available"); + expect(unsupported.recorder.record).not.toHaveBeenCalled(); + + const denied = createHarness({ + requestPermission: async () => ({ granted: false, canAskAgain: false }), + }); + await denied.controller.start(); + expect(denied.controller.currentState.errorAction).toBe("settings"); + expect(denied.recorder.record).not.toHaveBeenCalled(); + }); + + it.each(["permission", "transcription"] as const)( + "clears %s errors when switching to another draft", + async (failure) => { + const harness = createHarness( + failure === "permission" + ? { requestPermission: async () => ({ granted: false, canAskAgain: false }) } + : { + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(async () => { + throw new Error("Transcription failed"); + }), + }), + }, + ); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toMatchObject({ + phase: "error", + error: expect.any(String), + errorAction: failure === "permission" ? "settings" : "retry", + }); + + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + harness.controller.ownerChanged(); + + expect(harness.controller.currentState).toEqual({ + phase: "idle", + error: null, + errorAction: null, + }); + expect(harness.commits).toEqual([]); + }, + ); + + it.each(["permission", "preparation", "recording"] as const)( + "keeps the selected transcriber when preferences change during %s", + async (changeDuring) => { + const permission = deferred<{ granted: boolean; canAskAgain: boolean }>(); + const permissionEntered = deferred(); + const preparation = deferred(); + const preparationEntered = deferred(); + const preparationSignals: AbortSignal[] = []; + const transcriptionSignals: AbortSignal[] = []; + const transcriber = (text: string): VoiceTranscriber => ({ + prepare: async ({ signal }) => { + preparationSignals.push(signal); + preparationEntered.resolve(undefined); + await preparation.promise; + return preparedTranscription(async (_uri, { signal }) => { + transcriptionSignals.push(signal); + return text; + }); + }, + }); + const first = transcriber("first choice"); + const second = transcriber("second choice"); + let selected = first; + const harness = createHarness({ + getTranscriber: () => selected, + requestPermission: () => { + permissionEntered.resolve(undefined); + return permission.promise; + }, + }); + + const starting = harness.controller.start(); + await permissionEntered.promise; + if (changeDuring === "permission") selected = second; + permission.resolve({ granted: true, canAskAgain: true }); + await preparationEntered.promise; + if (changeDuring === "preparation") selected = second; + preparation.resolve(undefined); + await starting; + if (changeDuring === "recording") selected = second; + await harness.controller.stop(); + + expect(harness.commits.map((commit) => commit.text)).toEqual(["hello first choice"]); + + await harness.controller.start(); + await harness.controller.stop(); + + expect(harness.commits.map((commit) => commit.text)).toEqual([ + "hello first choice", + "hello second choice", + ]); + expect(preparationSignals).toHaveLength(2); + expect(transcriptionSignals).toHaveLength(2); + expect(transcriptionSignals[0]).toBe(preparationSignals[0]); + expect(transcriptionSignals[1]).toBe(preparationSignals[1]); + expect(preparationSignals[1]).not.toBe(preparationSignals[0]); + }, + ); + + it("blocks submit while voice input can still change the draft", () => { + expect(voiceInputBlocksSubmission({ phase: "preparing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputBlocksSubmission({ phase: "recording", error: null, errorAction: null })).toBe( + true, + ); + expect( + voiceInputBlocksSubmission({ phase: "transcribing", error: null, errorAction: null }), + ).toBe(true); + expect(voiceInputBlocksSubmission({ phase: "idle", error: null, errorAction: null })).toBe( + false, + ); + }); + + it("uses the native five-minute cap and commits one final transcript", async () => { + const harness = createHarness(); + await harness.controller.start(); + expect(harness.recorder.record).toHaveBeenCalledWith({ + forDuration: VOICE_RECORDING_LIMIT_SECONDS, + }); + + const stopping = harness.controller.stop(); + harness.controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: "file:///voice.m4a", + }); + await stopping; + + expect(harness.commits).toEqual([ + { text: "hello new text", selection: { start: 14, end: 14 } }, + ]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + + it.each(["cancel", "dispose", "ownerChanged"] as const)( + "holds the session after %s until non-abortable transcription settles", + async (action) => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription((_uri, { signal }) => { + transcriptionEntered.resolve(signal); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + const signal = await transcriptionEntered.promise; + expect(signal.aborted).toBe(false); + if (action === "ownerChanged") { + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + } + harness.controller[action](); + expect(signal.aborted).toBe(true); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.error).toContain("already active"); + expect(next.recorder.record).not.toHaveBeenCalled(); + + transcription.resolve("late text"); + await stopping; + + expect(harness.commits).toEqual([]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.controller.currentState.phase).toBe("idle"); + + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }, + ); + + it("cancels an in-flight transcriber that rejects when its signal aborts", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription((_uri, { signal }) => { + signal.addEventListener("abort", () => transcription.reject(new Error("aborted")), { + once: true, + }); + transcriptionEntered.resolve(signal); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + const signal = await transcriptionEntered.promise; + harness.controller.cancel(); + await stopping; + + expect(signal.aborted).toBe(true); + expect(harness.commits).toEqual([]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + + it("releases the microphone before transcription starts", async () => { + const events: string[] = []; + const harness = createHarness({ + releaseRecording: async () => { + events.push("released"); + }, + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(async () => { + events.push("transcribed"); + return "done"; + }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + + expect(events).toEqual(["released", "transcribed"]); + }); + + it("retries audio-session release during final cleanup", async () => { + const releaseRecording = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("busy")) + .mockResolvedValueOnce(undefined); + const harness = createHarness({ releaseRecording }); + await harness.controller.start(); + await harness.controller.stop(); + + expect(releaseRecording).toHaveBeenCalledTimes(2); + }); + + it("leaves transcription with an error when recorder finalization fails", async () => { + const harness = createHarness(); + harness.recorder.stop.mockRejectedValueOnce(new Error("stop failed")); + await harness.controller.start(); + await harness.controller.stop(); + + expect(harness.controller.currentState.phase).toBe("error"); + expect(harness.controller.currentState.error).toContain("finish voice recording"); + }); + + it("ignores a late transcript after the draft owner changes", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(() => { + transcriptionEntered.resolve(undefined); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + await transcriptionEntered.promise; + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + transcription.resolve("late text"); + await stopping; + + expect(harness.commits).toEqual([]); + expect(harness.controller.currentState.error).toContain("draft changed"); + }); + + it("keeps the app-wide session locked until canceled preparation settles", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const first = createHarness({ + getTranscriber: () => ({ + prepare: ({ signal }) => { + preparationEntered.resolve(signal); + return preparation.promise; + }, + }), + }); + const firstStart = first.controller.start(); + const signal = await preparationEntered.promise; + first.controller.cancel(); + expect(signal.aborted).toBe(true); + + const blocked = createHarness(); + await blocked.controller.start(); + expect(blocked.controller.currentState.error).toContain("already active"); + + preparation.resolve(preparedTranscription()); + await firstStart; + expect(first.recorder.record).not.toHaveBeenCalled(); + blocked.controller.cancel(); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }); + + it("does not start the microphone for an owner that changed during preparation", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: () => { + preparationEntered.resolve(undefined); + return preparation.promise; + }, + }), + }); + const starting = harness.controller.start(); + await preparationEntered.promise; + harness.setDraft(draft({ ownerKey: "environment:other-thread", text: "other draft" })); + preparation.resolve(preparedTranscription()); + await starting; + + expect(harness.recorder.record).not.toHaveBeenCalled(); + expect(harness.controller.currentState.error).toContain("no longer available"); + }); + + it("discards recorder errors and audio interruptions without transcribing", async () => { + const transcribe = vi.fn(async () => "ignored"); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async ({ signal }) => { + preparationEntered.resolve(signal); + return preparedTranscription(transcribe); + }, + }), + }); + await harness.controller.start(); + const signal = await preparationEntered.promise; + harness.recorder.uri = "file:///reset-empty.m4a"; + await harness.controller.handleRecorderStatus({ + isFinished: true, + hasError: true, + error: "Audio route changed", + url: "file:///voice.m4a", + }); + + expect(harness.commits).toEqual([]); + expect(transcribe).not.toHaveBeenCalled(); + expect(signal.aborted).toBe(true); + expect(harness.controller.currentState.error).toBe("Audio route changed"); + expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); + }); + + it("cancels preparation when the app reaches the background", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: ({ signal }) => { + preparationEntered.resolve(signal); + return preparation.promise; + }, + }), + }); + const starting = harness.controller.start(); + const signal = await preparationEntered.promise; + harness.controller.appMovedToBackground(); + expect(signal.aborted).toBe(true); + preparation.resolve(preparedTranscription()); + await starting; + + expect(harness.recorder.record).not.toHaveBeenCalled(); + expect(harness.controller.currentState.error).toContain("background"); + }); +}); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts new file mode 100644 index 000000000..cb284ad9f --- /dev/null +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -0,0 +1,493 @@ +import { replaceTextRange } from "@t3tools/shared/composerTrigger"; + +import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts"; + +export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; + +export type VoiceInputPhase = "idle" | "preparing" | "recording" | "transcribing" | "error"; + +export type VoiceInputState = { + readonly phase: VoiceInputPhase; + readonly error: string | null; + readonly errorAction: "retry" | "settings" | null; +}; + +export function voiceInputBlocksSubmission(state: VoiceInputState): boolean { + return ( + state.phase === "preparing" || state.phase === "recording" || state.phase === "transcribing" + ); +} + +export function voiceInputFreezesEditor(state: VoiceInputState): boolean { + return voiceInputBlocksSubmission(state); +} + +export type VoiceDraftSnapshot = { + readonly ownerKey: string; + readonly text: string; + readonly selection: { readonly start: number; readonly end: number }; + readonly revision: number; +}; + +export type VoiceRecorderStatus = { + readonly isFinished: boolean; + readonly hasError: boolean; + readonly error: string | null; + readonly url: string | null; +}; + +export interface VoiceRecorder { + readonly uri: string | null; + prepareToRecordAsync(): Promise; + record(options: { readonly forDuration: number }): void; + stop(): Promise; +} + +export type VoiceInputControllerDependencies = { + readonly recorder: VoiceRecorder; + readonly getTranscriber: () => VoiceTranscriber | null; + readonly requestPermission: () => Promise<{ + readonly granted: boolean; + readonly canAskAgain: boolean; + }>; + readonly configureRecording: () => Promise; + readonly releaseRecording: () => Promise; + readonly deleteRecording: (uri: string) => void; + readonly readDraft: () => VoiceDraftSnapshot | null; + readonly commitDraft: ( + text: string, + selection: { readonly start: number; readonly end: number }, + ) => void; + readonly onStateChange: (state: VoiceInputState) => void; +}; + +type TranscriptCommitResult = + | { + readonly kind: "commit"; + readonly text: string; + readonly selection: { readonly start: number; readonly end: number }; + } + | { readonly kind: "stale" } + | { readonly kind: "empty" }; + +export function resolveTranscriptCommit( + captured: VoiceDraftSnapshot, + current: VoiceDraftSnapshot | null, + transcript: string, + locale: string, +): TranscriptCommitResult { + if ( + !current || + current.ownerKey !== captured.ownerKey || + current.text !== captured.text || + current.revision !== captured.revision + ) { + return { kind: "stale" }; + } + + const replacement = transcript.trim(); + if (replacement.length === 0) { + return { kind: "empty" }; + } + + const isEmptySelection = captured.selection.start === captured.selection.end; + const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); + const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + let insertion = replacement; + if (isEmptySelection && usesEnglishSpacing) { + const left = captured.text[captured.selection.start - 1]; + const right = captured.text[captured.selection.start]; + const leftNeedsBoundary = + left !== undefined && + /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + (right === undefined || /\s/.test(right)); + const rightNeedsBoundary = + right !== undefined && + /[A-Za-z0-9([{'"]/.test(right) && + (left === undefined || /\s/.test(left)); + insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; + } + + const result = replaceTextRange( + captured.text, + captured.selection.start, + captured.selection.end, + insertion, + ); + return { + kind: "commit", + text: result.text, + selection: { start: result.cursor, end: result.cursor }, + }; +} + +let activeSession: symbol | null = null; +let activeTranscriptionOperation: Promise | null = null; + +function acquireSession(): symbol | null { + if (activeSession) return null; + const token = Symbol("voice-input-session"); + activeSession = token; + return token; +} + +function releaseSession(token: symbol | null): void { + if (token && activeSession === token) activeSession = null; +} + +async function runTranscriptionOperation(operation: () => Promise): Promise { + if (activeTranscriptionOperation) { + throw new Error("voice-operation-busy"); + } + + const promise = operation(); + activeTranscriptionOperation = promise; + try { + return await promise; + } finally { + if (activeTranscriptionOperation === promise) activeTranscriptionOperation = null; + } +} + +function errorCode(error: unknown): string | null { + if (typeof error !== "object" || error === null || !("code" in error)) return null; + return typeof error.code === "string" ? error.code : null; +} + +function preparationErrorMessage(error: unknown): string { + if (error instanceof Error && error.message === "voice-operation-busy") { + return "Voice transcription is still finishing. Try again shortly."; + } + if (errorCode(error) === "unsupported-locale") { + return "Voice transcription is not available for this language."; + } + return "Could not prepare voice transcription."; +} + +function transcriptionErrorMessage(error: unknown): string { + if (error instanceof Error && error.message === "voice-operation-busy") { + return "Voice transcription is still finishing. Try again shortly."; + } + return "Could not transcribe this recording."; +} + +const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; + +export class VoiceInputController { + private readonly dependencies: VoiceInputControllerDependencies; + private state: VoiceInputState = IDLE_STATE; + private operationToken = 0; + private sessionToken: symbol | null = null; + private transcription: PreparedVoiceTranscription | null = null; + private transcriptionAbortController: AbortController | null = null; + private capturedDraft: VoiceDraftSnapshot | null = null; + private recordingUri: string | null = null; + private readonly ownedRecordingUris = new Set(); + private recordingConfigured = false; + private finishing = false; + + constructor(dependencies: VoiceInputControllerDependencies) { + this.dependencies = dependencies; + } + + get currentState(): VoiceInputState { + return this.state; + } + + async start(): Promise { + if (this.state.phase !== "idle" && this.state.phase !== "error") return; + const initiatingDraft = this.dependencies.readDraft(); + if (!initiatingDraft) { + this.setError("This draft is no longer available.", "retry"); + return; + } + const sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "retry"); + return; + } + + this.sessionToken = sessionToken; + const operationToken = ++this.operationToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; + this.setState({ phase: "preparing", error: null, errorAction: null }); + + try { + const transcriber = this.dependencies.getTranscriber(); + if (!transcriber) { + this.setError("Voice transcription is not available.", null); + return; + } + + const permission = await this.dependencies.requestPermission(); + if (!this.isCurrent(operationToken)) return; + if (!permission.granted) { + this.setError( + "Microphone access is required for voice input.", + permission.canAskAgain ? "retry" : "settings", + ); + return; + } + + try { + this.transcription = await runTranscriptionOperation(() => + transcriber.prepare({ signal: abortController.signal }), + ); + } catch (error) { + if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry"); + return; + } + if (!this.isCurrent(operationToken)) return; + + await this.dependencies.configureRecording(); + this.recordingConfigured = true; + if (!this.isCurrent(operationToken)) return; + await this.dependencies.recorder.prepareToRecordAsync(); + if (!this.isCurrent(operationToken)) return; + this.recordingUri = this.dependencies.recorder.uri; + this.rememberRecordingUri(this.recordingUri); + + const capturedDraft = this.dependencies.readDraft(); + if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) { + this.setError("This draft is no longer available.", "retry"); + return; + } + this.capturedDraft = capturedDraft; + this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); + this.setState({ phase: "recording", error: null, errorAction: null }); + } catch { + if (this.isCurrent(operationToken)) + this.setError("Could not start voice recording.", "retry"); + } finally { + if (this.isCurrent(operationToken) && this.state.phase === "error") { + await this.releaseResources(); + } else if (!this.isCurrent(operationToken) && !this.finishing) { + await this.releaseResources(); + } + } + } + + stop(): Promise { + if (this.state.phase !== "recording") return Promise.resolve(); + return this.finishRecording(false, null); + } + + cancel(): void { + switch (this.state.phase) { + case "idle": + return; + case "error": + this.setState(IDLE_STATE); + return; + case "preparing": + this.invalidateOperation(); + this.setState(IDLE_STATE); + return; + case "recording": + this.discardRecording(null); + return; + case "transcribing": + this.invalidateOperation(); + this.setState(IDLE_STATE); + return; + } + } + + interruptRecording( + message = "Voice recording was interrupted.", + completedUri: string | null = null, + ): Promise | void { + if (this.state.phase !== "recording") return; + this.rememberRecordingUri(completedUri); + this.recordingUri = completedUri ?? this.recordingUri; + return this.discardRecording(message); + } + + appMovedToBackground(): Promise | void { + if (this.state.phase === "preparing") { + this.invalidateOperation(); + this.setError("Voice input stopped when the app moved to the background.", "retry"); + return; + } + return this.interruptRecording(); + } + + handleRecorderStatus(status: VoiceRecorderStatus): Promise | void { + if (this.state.phase !== "recording") return; + if (status.hasError) { + return this.interruptRecording( + status.error ?? "Voice recording was interrupted.", + status.url, + ); + } + if (status.isFinished) { + if (!status.url) { + return this.interruptRecording(); + } + return this.finishRecording(true, status.url); + } + } + + ownerChanged(): void { + if (this.state.phase === "idle") return; + this.cancel(); + } + + dispose(): void { + if (this.state.phase === "recording") { + this.discardRecording(null); + return; + } + if (this.state.phase === "preparing" || this.state.phase === "transcribing") { + this.invalidateOperation(); + this.setState(IDLE_STATE); + } + } + + private async finishRecording( + alreadyStopped: boolean, + completedUri: string | null, + ): Promise { + if (this.finishing || this.state.phase !== "recording") return; + this.finishing = true; + const operationToken = this.operationToken; + this.setState({ phase: "transcribing", error: null, errorAction: null }); + + try { + if (!alreadyStopped) await this.dependencies.recorder.stop(); + await this.releaseAudioSession(); + this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; + this.rememberRecordingUri(this.recordingUri); + if (!this.isCurrent(operationToken)) return; + if ( + !this.recordingUri || + !this.transcription || + !this.transcriptionAbortController || + !this.capturedDraft + ) { + this.setError("Could not finish voice recording.", "retry"); + return; + } + + const recordingUri = this.recordingUri; + const transcription = this.transcription; + const signal = this.transcriptionAbortController.signal; + const capturedDraft = this.capturedDraft; + let transcript: string; + try { + transcript = await runTranscriptionOperation(() => + transcription.transcribe(recordingUri, { signal }), + ); + } catch (error) { + if (this.isCurrent(operationToken)) { + this.setError(transcriptionErrorMessage(error), "retry"); + } + return; + } + if (!this.isCurrent(operationToken)) return; + + const result = resolveTranscriptCommit( + capturedDraft, + this.dependencies.readDraft(), + transcript, + transcription.locale, + ); + if (result.kind === "stale") { + this.setError( + "The draft changed while voice input was running. The transcript was not added.", + "retry", + ); + return; + } + if (result.kind === "empty") { + this.setError("No speech was detected.", "retry"); + return; + } + + this.dependencies.commitDraft(result.text, result.selection); + this.setState(IDLE_STATE); + } catch { + if (this.isCurrent(operationToken)) { + this.setError("Could not finish voice recording.", "retry"); + } + } finally { + this.finishing = false; + await this.releaseResources(); + } + } + + private async discardRecording(error: string | null): Promise { + this.invalidateOperation(); + this.setState( + error + ? { phase: "error", error, errorAction: "retry" } + : { phase: "idle", error: null, errorAction: null }, + ); + try { + await this.dependencies.recorder.stop(); + this.rememberRecordingUri(this.dependencies.recorder.uri); + } catch { + this.rememberRecordingUri(this.dependencies.recorder.uri); + } finally { + await this.releaseResources(); + } + } + + private async releaseResources(): Promise { + this.rememberRecordingUri(this.recordingUri); + this.rememberRecordingUri(this.dependencies.recorder.uri); + this.recordingUri = null; + for (const uri of this.ownedRecordingUris) { + try { + this.dependencies.deleteRecording(uri); + } catch { + // The cache may already have removed a failed or interrupted recording. + } + } + this.ownedRecordingUris.clear(); + await this.releaseAudioSession(); + releaseSession(this.sessionToken); + this.sessionToken = null; + this.capturedDraft = null; + this.transcription = null; + this.transcriptionAbortController = null; + } + + private rememberRecordingUri(uri: string | null): void { + if (uri) this.ownedRecordingUris.add(uri); + } + + private async releaseAudioSession(): Promise { + if (!this.recordingConfigured) return; + try { + await this.dependencies.releaseRecording(); + this.recordingConfigured = false; + } catch { + // Final cleanup retries if the prompt release before transcription fails. + } + } + + private invalidateOperation(): void { + this.operationToken += 1; + this.transcriptionAbortController?.abort(); + } + + private isCurrent(operationToken: number): boolean { + return operationToken === this.operationToken; + } + + private setError(error: string, errorAction: VoiceInputState["errorAction"]): void { + this.setState({ phase: "error", error, errorAction }); + } + + private setState(state: VoiceInputState): void { + this.state = state; + this.dependencies.onStateChange(state); + } +} + +export function resetVoiceInputGlobalsForTests(): void { + activeSession = null; + activeTranscriptionOperation = null; +} diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts new file mode 100644 index 000000000..c8c8da455 --- /dev/null +++ b/packages/client-runtime/src/voice-input/index.ts @@ -0,0 +1,21 @@ +export { + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + resolveTranscriptCommit, + voiceInputBlocksSubmission, + voiceInputFreezesEditor, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, + type VoiceInputPhase, + type VoiceInputState, + type VoiceRecorder, + type VoiceRecorderStatus, +} from "./controller.ts"; +export { + VoiceTranscriptionError, + throwIfVoiceTranscriptionAborted, + type PreparedVoiceTranscription, + type VoiceTranscriber, + type VoiceTranscriptionErrorCode, + type VoiceTranscriptionOptions, +} from "./transcription.ts"; diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts new file mode 100644 index 000000000..f3ce377ac --- /dev/null +++ b/packages/client-runtime/src/voice-input/transcription.ts @@ -0,0 +1,37 @@ +/** Cancellation is cooperative: settle only after the underlying work has stopped. */ +export type VoiceTranscriptionOptions = { + readonly signal: AbortSignal; +}; + +/** Binds a recording to its selected implementation and resolved locale. */ +export type PreparedVoiceTranscription = { + readonly locale: string; + readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise; +}; + +export type VoiceTranscriber = { + readonly prepare: (options: VoiceTranscriptionOptions) => Promise; +}; + +export type VoiceTranscriptionErrorCode = + | "unavailable" + | "unsupported-locale" + | "preparation-failed" + | "transcription-failed" + | "cancelled"; + +export class VoiceTranscriptionError extends Error { + readonly code: VoiceTranscriptionErrorCode; + + constructor(code: VoiceTranscriptionErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.name = "VoiceTranscriptionError"; + this.code = code; + } +} + +export function throwIfVoiceTranscriptionAborted(signal: AbortSignal): void { + if (signal.aborted) { + throw new VoiceTranscriptionError("cancelled", "Voice transcription was cancelled."); + } +} diff --git a/patches/@react-native-ai__apple@0.12.0.patch b/patches/@react-native-ai__apple@0.12.0.patch new file mode 100644 index 000000000..b2a7aaf22 --- /dev/null +++ b/patches/@react-native-ai__apple@0.12.0.patch @@ -0,0 +1,194 @@ +diff --git a/ios/transcription/AppleTranscriptionImpl.swift b/ios/transcription/AppleTranscriptionImpl.swift +index 188371a5f55fadae19187108c4a688569d0223e3..3e1cdd77b0bb8ae6c97a40d29c1854200d931eee 100644 +--- a/ios/transcription/AppleTranscriptionImpl.swift ++++ b/ios/transcription/AppleTranscriptionImpl.swift +@@ -12,6 +12,11 @@ import UniformTypeIdentifiers + + @objc + public class AppleTranscriptionImpl: NSObject { ++ private struct CollectedSegment: Sendable { ++ let text: String ++ let startSecond: Double ++ let endSecond: Double ++ } + + @available(iOS 26, *) + private func createTranscriber(for locale: Locale) -> SpeechTranscriber { +@@ -41,7 +46,7 @@ public class AppleTranscriptionImpl: NSObject { + let locale = Locale(identifier: language) + + guard let supportedLocale = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { +- reject("AppleTranscription", "Locale not supported: \(language)", nil) ++ reject("AppleTranscriptionUnsupportedLocale", "Locale not supported: \(language)", nil) + return + } + +@@ -51,16 +56,14 @@ public class AppleTranscriptionImpl: NSObject { + + switch status { + case .installed: +- resolve(nil) ++ resolve(supportedLocale.identifier) + case .supported, .downloading: +- if let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { ++ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() +- resolve(nil) +- } else { +- resolve(nil) + } ++ resolve(supportedLocale.identifier) + case .unsupported: +- reject("AppleTranscription", "Assets not supported for locale: \(supportedLocale.identifier)", nil) ++ reject("AppleTranscriptionUnsupportedLocale", "Assets not supported for locale: \(supportedLocale.identifier)", nil) + @unknown default: + reject ("AppleTranscription", "Unknown asset inventory status", nil) + } +@@ -83,57 +86,77 @@ public class AppleTranscriptionImpl: NSObject { + do { + try audioData.write(to: fileURL) + +- guard let audioFile = try? AVAudioFile(forReading: fileURL) else { +- reject("AppleTranscription", "Invalid audio data", nil) ++ let audioFile: AVAudioFile ++ do { ++ audioFile = try AVAudioFile(forReading: fileURL) ++ } catch { ++ try? FileManager.default.removeItem(at: fileURL) ++ reject("AppleTranscription", "Invalid audio data", error) + return + } + + Task { ++ defer { ++ try? FileManager.default.removeItem(at: fileURL) ++ } ++ + do { + let transcriber = createTranscriber(for: Locale(identifier: language)) + + let analyzer = SpeechAnalyzer(modules: [transcriber]) +- +- defer { +- try? FileManager.default.removeItem(at: fileURL) +- } +- +- var segments: [[String: Any]] = [] +- +- Task { ++ ++ let collectorTask = Task { () throws -> [CollectedSegment] in ++ var segments: [CollectedSegment] = [] ++ + for try await result in transcriber.results { + if result.isFinal { +- let segment: [String: Any] = [ +- "text": String(result.text.characters), +- "startSecond": CMTimeGetSeconds(result.range.start), +- "endSecond": CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) +- ] +- segments.append(segment) ++ segments.append( ++ CollectedSegment( ++ text: String(result.text.characters), ++ startSecond: CMTimeGetSeconds(result.range.start), ++ endSecond: CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) ++ ) ++ ) + } + } ++ ++ return segments + } +- +- let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) +- +- if let lastSampleTime { +- try await analyzer.finalizeAndFinish(through: lastSampleTime) +- } else { ++ ++ do { ++ let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) ++ ++ if let lastSampleTime { ++ try await analyzer.finalizeAndFinish(through: lastSampleTime) ++ } else { ++ await analyzer.cancelAndFinishNow() ++ } ++ ++ let segments: [[String: Any]] = try await collectorTask.value.map { segment in ++ [ ++ "text": segment.text, ++ "startSecond": segment.startSecond, ++ "endSecond": segment.endSecond ++ ] ++ } ++ let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } ++ ++ resolve([ ++ "segments": segments, ++ "duration": totalDuration ++ ]) ++ } catch { ++ collectorTask.cancel() + await analyzer.cancelAndFinishNow() ++ _ = try? await collectorTask.value ++ throw error + } +- +- let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } +- +- let result: [String: Any] = [ +- "segments": segments, +- "duration": totalDuration +- ] +- +- resolve(result) + } catch { + reject("AppleTranscription", "Transcription failed: \(error.localizedDescription)", error) + } + } + } catch { ++ try? FileManager.default.removeItem(at: fileURL) + reject("AppleTranscription", "Failed to write audio data: \(error.localizedDescription)", error) + } + } else { +@@ -141,4 +164,3 @@ public class AppleTranscriptionImpl: NSObject { + } + } + } +- +diff --git a/lib/typescript/NativeAppleTranscription.d.ts b/lib/typescript/NativeAppleTranscription.d.ts +index 985b6b3593a884c41d689346be9d73d86c55eae1..86e7936f23a8673d099f7fe18af9a31abae5a13a 100644 +--- a/lib/typescript/NativeAppleTranscription.d.ts ++++ b/lib/typescript/NativeAppleTranscription.d.ts +@@ -10,14 +10,14 @@ export interface TranscriptionResult { + } + export interface Spec extends TurboModule { + isAvailable(language: string): boolean; +- prepare(language: string): Promise; ++ prepare(language: string): Promise; + } + declare global { + function __apple__llm__transcribe__(data: ArrayBufferLike, language: string): Promise; + } + declare const _default: { + transcribe: (data: ArrayBufferLike, language: string) => Promise; +- prepare: (language: string) => Promise; ++ prepare: (language: string) => Promise; + isAvailable: (language: string) => boolean; + }; + export default _default; +diff --git a/src/NativeAppleTranscription.ts b/src/NativeAppleTranscription.ts +index 13332a0176000b60b6f6f043de681db55a7b4bef..5389fb4b3a53de48721c307d2145b6770624c852 100644 +--- a/src/NativeAppleTranscription.ts ++++ b/src/NativeAppleTranscription.ts +@@ -14,7 +14,7 @@ export interface TranscriptionResult { + + export interface Spec extends TurboModule { + isAvailable(language: string): boolean +- prepare(language: string): Promise ++ prepare(language: string): Promise + } + + declare global { diff --git a/patches/expo-audio@57.0.4.patch b/patches/expo-audio@57.0.4.patch new file mode 100644 index 000000000..1452eca31 --- /dev/null +++ b/patches/expo-audio@57.0.4.patch @@ -0,0 +1,24 @@ +diff --git a/ios/AudioRecorder.swift b/ios/AudioRecorder.swift +index 20020bff9f8ba7bb6e7f61c99b4de5d29eee00db..284c22f12e30c8c2ed16feb141876cbfd8897c90 100644 +--- a/ios/AudioRecorder.swift ++++ b/ios/AudioRecorder.swift +@@ -216,15 +216,15 @@ class AudioRecorder: SharedRef, RecordingResultHandler { + } + + func didFinish(_ recorder: AVAudioRecorder, successfully flag: Bool) { +- // Update internal state when recording finishes automatically (e.g., from recordForDuration) +- currentState = .stopped ++ // Update internal state when AVAudioRecorder finishes or fails. ++ currentState = flag ? .stopped : .error + resetDurationTracking() + + emit(event: recordingStatus, payload: [ + "id": id, + "isFinished": true, +- "hasError": false, +- "error": nil, ++ "hasError": !flag, ++ "error": flag ? nil : "Recording failed", + "url": recorder.url.absoluteString + ]) + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 006f3fc53..7149a07f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,9 +89,11 @@ patchedDependencies: '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca368 '@legendapp/list@3.3.5': d14852e52d4bfb128bf8c3c1476cb8ca4dd392999f21f13fd62ac3bf04a62c40 '@pierre/diffs@1.3.0-beta.10': 7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa + '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d '@react-native-menu/menu@2.0.0': c7f66d121c726ade4f5c4e1aed11a691e5711d244c544084e289ac26132a0045 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e2295552 effect@4.0.0-beta.103: af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6 + expo-audio@57.0.4: fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a expo-modules-jsi@57.0.6: 0794db2805abb43f770292fea9afbd80a85726082d33709237182a8d1568f133 expo-sharing@57.0.16: 8d2e3b10eb3f52036a9a086800180ec6cebf3b75bccc5b1775117a7244d4ac45 react-native-gesture-handler@2.32.0: 808eb26f9e57cf4945efd3985af4d9c764da6f91f4c9764433cc868602bbf4d3 @@ -237,6 +239,9 @@ importers: '@pierre/diffs': specifier: 'catalog:' version: 1.3.0-beta.10(patch_hash=7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@react-native-ai/apple': + specifier: 0.12.0 + version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)) '@react-native-menu/menu': specifier: ^2.0.0 version: 2.0.0(patch_hash=c7f66d121c726ade4f5c4e1aed11a691e5711d244c544084e289ac26132a0045)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -297,6 +302,9 @@ importers: expo-asset: specifier: ~57.0.15 version: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + expo-audio: + specifier: ~57.0.4 + version: 57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) expo-auth-session: specifier: ~57.0.10 version: 57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -997,6 +1005,16 @@ importers: packages: + '@ai-sdk/provider-utils@4.0.49': + resolution: {integrity: sha512-8e7pd+82bobqrFOaD5dG/PiEuvLYr5olaE3I56ch0jipR0H7sGD6ohwTUynv6k8O8QidWiyIbEsZCtr/2dyXIA==} + engines: {node: '>=18.17'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@3.0.15': + resolution: {integrity: sha512-XeZW1CcDF2GMbH4wejW6xBRI2QCOgnkVYUnxoeDadB1mf85riL2bMUeDoh+6gJ/r4mjNfzUPW8OjLjvwTP0u1Q==} + engines: {node: '>=18'} + '@alcalzone/ansi-tokenize@0.2.5': resolution: {integrity: sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw==} engines: {node: '>=18'} @@ -3872,6 +3890,11 @@ packages: resolution: {integrity: sha512-gMDYY2rw6OWajCcDlXSIgs2LC432YJXSb3Lm5yM187uhRgBYddoEVULi36h+IolX3r7jSb3ew7vn9FfI8NSo0A==} hasBin: true + '@react-native-ai/apple@0.12.0': + resolution: {integrity: sha512-BC/kEDbCZprv1xcQCWsgapXm/WEsj5lieBDvU1vJXStrN+BG+hWM01zoqDGTD/j71zvt9DEfHcMvEqbrGY2uAA==} + peerDependencies: + react-native: '>=0.76.0' + '@react-native-masked-view/masked-view@0.3.2': resolution: {integrity: sha512-XwuQoW7/GEgWRMovOQtX3A4PrXhyaZm0lVUiY8qJDvdngjLms9Cpdck6SmGAUNqQwcj2EadHC1HwL0bEyoa/SQ==} peerDependencies: @@ -6627,6 +6650,14 @@ packages: react: '*' react-native: '*' + expo-audio@57.0.4: + resolution: {integrity: sha512-TLP8rt1UvUDzgxGnyQ0TR9hV6tNP/UJQdDu7mSK+dgEydMFoptq55D3hcUoB1gF39f3/3AUuWfOtpGM+4N4X1A==} + peerDependencies: + expo: '*' + expo-asset: '*' + react: '*' + react-native: '*' + expo-auth-session@57.0.10: resolution: {integrity: sha512-i1RY93LouEgal2e/Ly1r2Ytwed7/y6i4AUShVj4Gg49h/Okf8QpZO7XylwL43IHfKRrKkqapRDEN8mWe7z/HgA==} peerDependencies: @@ -7618,6 +7649,9 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} @@ -10080,8 +10114,8 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - undici@6.26.0: - resolution: {integrity: sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A==} + undici@6.28.0: + resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==} engines: {node: '>=18.17'} undici@7.27.1: @@ -10738,6 +10772,18 @@ packages: snapshots: + '@ai-sdk/provider-utils@4.0.49(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.15 + '@standard-schema/spec': 1.1.0 + eventsource-parser: 3.1.0 + undici: 6.28.0 + zod: 4.4.3 + + '@ai-sdk/provider@3.0.15': + dependencies: + json-schema: 0.4.0 + '@alcalzone/ansi-tokenize@0.2.5': dependencies: ansi-styles: 6.2.3 @@ -14094,6 +14140,13 @@ snapshots: prompts: 2.4.2 tinyexec: 1.2.4 + '@react-native-ai/apple@0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))': + dependencies: + '@ai-sdk/provider': 3.0.15 + '@ai-sdk/provider-utils': 4.0.49(zod@4.4.3) + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + zod: 4.4.3 + '@react-native-masked-view/masked-view@0.3.2(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)': dependencies: react: 19.2.3 @@ -16817,6 +16870,13 @@ snapshots: - typescript optional: true + expo-audio@57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): + dependencies: + expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc) + expo-asset: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + expo-auth-session@57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): dependencies: expo-application: 57.0.2(expo@57.0.18) @@ -18100,6 +18160,8 @@ snapshots: json-schema-typed@8.0.2: {} + json-schema@0.4.0: {} + json-stringify-safe@5.0.1: optional: true @@ -19364,7 +19426,7 @@ snapshots: semver: 7.8.5 tar: 7.5.16 tinyglobby: 0.2.17 - undici: 6.26.0 + undici: 6.28.0 which: 6.0.1 node-int64@0.4.0: {} @@ -21239,7 +21301,7 @@ snapshots: undici-types@7.16.0: {} - undici@6.26.0: {} + undici@6.28.0: {} undici@7.27.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 417d11062..a616b04a3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -153,10 +153,12 @@ patchedDependencies: "@ff-labs/fff-node@0.9.4": patches/@ff-labs__fff-node@0.9.4.patch "@legendapp/list@3.3.5": patches/@legendapp__list@3.3.5.patch "@pierre/diffs@1.3.0-beta.10": patches/@pierre%2Fdiffs@1.3.0-beta.10.patch + "@react-native-ai/apple@0.12.0": patches/@react-native-ai__apple@0.12.0.patch "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch effect@4.0.0-beta.103: patches/effect@4.0.0-beta.103.patch expo-modules-jsi@57.0.6: patches/expo-modules-jsi@57.0.6.patch + expo-audio@57.0.4: patches/expo-audio@57.0.4.patch expo-sharing@57.0.16: patches/expo-sharing@57.0.16.patch react-native-gesture-handler@2.32.0: patches/react-native-gesture-handler@2.32.0.patch react-native-keyboard-controller@1.21.13: patches/react-native-keyboard-controller@1.21.13.patch From ceaae36aa8dbcf99a654ce27378f1c0d009fa727 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 03:55:40 -0600 Subject: [PATCH 2/5] feat(mobile): adopt upstream's composer shape morph, keep Pylon's toolbar #8793 changed two independent things in the mobile composer, and treating them as one decision has cost a conflict in every cherry-pick from upstream's 2026-08-30 batch since. The first is ComposerSurface: animate borderRadius on a shared value, put the glass on an absolute layer, render children in their own animated view, and bound the collapsed pill radius so the morph interpolates instead of travelling from 999. Nothing in it touches the toolbar row. The second is the toolbar row: drop ComposerToolbarScroller for a fixed flex row. That one genuinely conflicts. ComposerToolbarScroller is upstream's own component and they still ship it; they stopped using it because their toolbar holds four controls. Pylon's holds sixteen, because ControlPillMenu - which does not exist upstream at all - carries Refine, session goal, context window, agent count, the input queue, depth, resources, and reload. Those need the scroller. So take the first, decline the second. ComposerSurface is now structurally identical to upstream (animatedBorderRadius, AnimatedGlassSurface, layoutTransition, animatedShapeStyle, and the bounded radius all match), and the toolbar is untouched: scroller, 13 ControlPillMenus, QuickQuestionTrigger and ContextWindowIndicator all at their previous counts. Pylon's shadow wrapper survives with its comment; upstream has no equivalent, and the radius it carries is now animated alongside the surface. --- .../src/features/threads/ThreadComposer.tsx | 60 +++++++++++++++---- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 3026f0c39..693f372da 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -59,7 +59,16 @@ import type { } from "@t3tools/contracts"; import { StackActions, useFocusEffect, useNavigation } from "@react-navigation/native"; import type { ReactNode } from "react"; -import { memo, useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; +import { + memo, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, + type RefObject, +} from "react"; import { ActivityIndicator, Alert, @@ -80,6 +89,9 @@ import Animated, { FadeOutDown, LinearTransition, ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, } from "react-native-reanimated"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { presentMobileContextWindow } from "../../lib/contextWindow"; @@ -276,8 +288,28 @@ export function ComposerSurface(props: { // style and `shadowOpacity: 1` would fall back to RN's default opaque black. const shadowColor = useUniwindTheme()["--color-primary-shadow"]; const isDarkMode = useColorScheme() === "dark"; + // #8793's shape morph, adopted without its toolbar restructure. Animating the + // radius on a shared value keeps the pill/card corners interpolating with the + // layout instead of snapping on the first frame. Every native frame carries + // the same transition: animating only the outer clip leaves the glass and the + // content at their final height immediately. + const targetBorderRadius = + typeof props.style.borderRadius === "number" ? props.style.borderRadius : 0; + const animatedBorderRadius = useSharedValue(targetBorderRadius); + const shouldAnimate = props.animateLayout !== false && Platform.OS !== "android"; + useLayoutEffect(() => { + animatedBorderRadius.value = shouldAnimate + ? withTiming(targetBorderRadius, { + duration: COMPOSER_TRANSITION_DURATION_MS, + reduceMotion: ReduceMotion.System, + }) + : targetBorderRadius; + }, [animatedBorderRadius, shouldAnimate, targetBorderRadius]); + const animatedShapeStyle = useAnimatedStyle(() => ({ + borderRadius: animatedBorderRadius.value, + })); + const layoutTransition = shouldAnimate ? COMPOSER_LAYOUT_TRANSITION : undefined; const shadowStyle: ViewStyle = { - borderRadius: props.style.borderRadius, shadowColor, shadowOpacity: isDarkMode ? 0.35 : 0.12, shadowRadius: 14, @@ -286,21 +318,26 @@ export function ComposerSurface(props: { }; return ( - - + + {null} + + {props.children} - + ); } @@ -1539,7 +1576,10 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer paddingTop: 14, } : { - borderRadius: 999, + // Bounded so the radius morph interpolates instead of + // travelling from 999; still renders as a capsule at this + // pill height. + borderRadius: 27, overflow: "hidden" as const, flexDirection: "row" as const, alignItems: "center" as const, From b56adfd501bc8b35a0311e9963c5999e7f1f6e65 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 06:39:09 -0600 Subject: [PATCH 3/5] feat(mobile): wire dictation controls into Pylon's composer toolbar Completes the #8614 port's UI half. Upstream places the dictation controls inside its restructured collapsed row and fixed toolbar; Pylon declined that restructure, so they are placed into Pylon's own structure instead. The toolbar now shows whenever isToolbarVisible rather than only when expanded, so dictation stays reachable from the collapsed pill, and it is wrapped in ComposerDictationToolbar. The cancel action leads the row; while dictating, ComposerDictationStatus replaces the toolbar scroller rather than upstream's fixed left group, so Pylon's thirteen ControlPillMenu controls keep their scroller when not dictating. The mic sits beside send in both the collapsed row and the toolbar, and send is hidden while dictation owns the row. Scroller, ControlPillMenu, QuickQuestionTrigger and ContextWindowIndicator are all at unchanged counts. --- .../src/features/threads/ThreadComposer.tsx | 409 ++++++++++-------- 1 file changed, 229 insertions(+), 180 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 693f372da..02328e9d6 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -1684,8 +1684,22 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer {!isExpanded && props.contextWindow ? ( ) : null} - {!isExpanded ? ( - + {!isExpanded && !voiceInput.isBusy ? ( + + {voiceInput.isAvailable ? ( + + ) : null} {showStopAction ? ( ) : null} - {isExpanded ? ( - - - { - if (props.serverConfig?.environment.capabilities.fileAttachments) { - Alert.alert("Add attachment", undefined, [ - { text: "Photos", onPress: () => void props.onPickDraftImages() }, - { text: "Files", onPress: () => void props.onPickDraftFiles() }, - { text: "Cancel", style: "cancel" }, - ]); - return; - } - void props.onPickDraftImages(); - }} - showChevron={false} + {isToolbarVisible ? ( + + + - {quickQuestionAvailable ? ( - setQuickQuestionOpenScopeKey(quickQuestionScopeKey)} + {isVoiceInputPresented ? ( + - ) : null} - - } - label={currentModelOption?.label ?? currentModelSelection.model} - maxWidth={152} - onPress={openSettings} - /> - {sessionHarnessRefinementActions.length > 0 ? ( - { - if ( - parseSessionHarnessRefinementAction( - nativeEvent.event, - sessionHarnessRefinementScopeKey, - ) === "refine" - ) { - confirmSessionHarnessRefinement(sessionHarnessRefinementScopeKey); - } - }} - > + ) : ( + { + if (props.serverConfig?.environment.capabilities.fileAttachments) { + Alert.alert("Add attachment", undefined, [ + { text: "Photos", onPress: () => void props.onPickDraftImages() }, + { text: "Files", onPress: () => void props.onPickDraftFiles() }, + { text: "Cancel", style: "cancel" }, + ]); + return; + } + void props.onPickDraftImages(); + }} + showChevron={false} /> - - ) : null} - {props.sessionGoal ? ( - - setQuickQuestionOpenScopeKey(quickQuestionScopeKey)} + /> + ) : null} + } + label={currentModelOption?.label ?? currentModelSelection.model} + maxWidth={152} + onPress={openSettings} /> - - ) : null} - {props.contextWindow || - (props.sessionCompaction?.available && sessionCompactionScopeKey) ? ( - props.sessionCompaction?.available && sessionCompactionScopeKey ? ( - - handleSessionCompactionAction(nativeEvent.event) - } - > + {sessionHarnessRefinementActions.length > 0 ? ( + { + if ( + parseSessionHarnessRefinementAction( + nativeEvent.event, + sessionHarnessRefinementScopeKey, + ) === "refine" + ) { + confirmSessionHarnessRefinement(sessionHarnessRefinementScopeKey); + } + }} + > + + + ) : null} + {props.sessionGoal ? ( + + + + ) : null} + {props.contextWindow || + (props.sessionCompaction?.available && sessionCompactionScopeKey) ? ( + props.sessionCompaction?.available && sessionCompactionScopeKey ? ( + + handleSessionCompactionAction(nativeEvent.event) + } + > + + + ) : props.contextWindow ? ( + + ) : null + ) : null} + {sessionAgentActions.length > 0 ? ( + + handleSessionAgentAction(nativeEvent.event) + } + > + + + ) : null} + {showSessionInputQueueModes && props.sessionInputQueue ? ( + + handleSessionInputQueueAction(nativeEvent.event) + } + > + 0 ? `Inputs ${sessionQueueCount}` : "Inputs"} + /> + + ) : null} + {showSessionAgentDepth && props.sessionAgentDepth !== null ? ( + + void setSessionAgentDepth(nativeEvent.event) + } + > + + + ) : null} + {sessionResourceInventory !== null ? ( setIsSessionResourcesOpen(true)} + showChevron={false} /> - - ) : props.contextWindow ? ( - - ) : null - ) : null} - {sessionAgentActions.length > 0 ? ( - handleSessionAgentAction(nativeEvent.event)} - > - - - ) : null} - {showSessionInputQueueModes && props.sessionInputQueue ? ( - - handleSessionInputQueueAction(nativeEvent.event) - } - > - 0 ? `Inputs ${sessionQueueCount}` : "Inputs"} + ) : showSessionResourceReload ? ( + void reloadSessionResources()} + showChevron={false} + /> + ) : null} + {showStopAction ? ( + + ) : null} + + )} + + {voiceInput.isAvailable ? ( + - - ) : null} - {showSessionAgentDepth && props.sessionAgentDepth !== null ? ( - - void setSessionAgentDepth(nativeEvent.event) - } - > + ) : null} + {isVoiceInputPresented ? null : ( - - ) : null} - {sessionResourceInventory !== null ? ( - setIsSessionResourcesOpen(true)} - showChevron={false} - /> - ) : showSessionResourceReload ? ( - void reloadSessionResources()} - showChevron={false} - /> - ) : null} - {showStopAction ? ( - - ) : null} - - - + )} + + + ) : null} From 7202195a53f84de1308c43f8008a3b2e2555fe77 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 06:41:01 -0600 Subject: [PATCH 4/5] fix(mobile): keep Pylon branding in the microphone permission prompt The #8614 port carried upstream's string verbatim: "Allow T3 Code to use your microphone for voice input." iOS shows that text in the permission dialog, so it is product copy, not a compatibility identifier. The camera permission two lines below already reads "Allow Pylon to access your camera", so this was purely adoption drift. Verified while checking permissions that no speech-recognition key is needed: @react-native-ai/apple uses SpeechAnalyzer and SpeechTranscriber, Apple's on-device Speech framework, rather than SFSpeechRecognizer. Only NSMicrophoneUsageDescription applies, and it is present. --- apps/mobile/app.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 5bd798577..807e603d0 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -341,7 +341,7 @@ const config: ExpoConfig = { [ "expo-audio", { - microphonePermission: "Allow T3 Code to use your microphone for voice input.", + microphonePermission: "Allow Pylon to use your microphone for voice input.", recordAudioAndroid: false, enableBackgroundPlayback: false, enableBackgroundRecording: false, From b21cb8fcb4d6f6ab45d5b2cd0701824b36db4cb0 Mon Sep 17 00:00:00 2001 From: Trevor Walker Date: Tue, 1 Sep 2026 06:55:29 -0600 Subject: [PATCH 5/5] fix(mobile): repair dictation placement defects found in review Adversarial review of the hand-placed dictation UI found three ways the composer strands the user. All three are in code written by hand rather than ported, and none was caught by typecheck, lint, or 1042 tests. The editor was never frozen. NewTaskDraftScreen passes readOnly={voiceInput.freezesEditor}; the thread composer did not, so the keyboard stayed live during recording. One keystroke makes resolveTranscriptCommit see a changed draft and discard the entire transcript as stale - up to five minutes of speech, silently. It also made every native read-only guard this branch adds dead code on this surface. Send was gated on isVoiceInputPresented rather than voicePresentation.showsSend. Those look equivalent but diverge in exactly one phase: error shows a status label AND keeps send. Any dictation failure - denied permission, no speech detected, the stale-draft error above - left the composer with no send control until the user found the dismiss button. Worse on entry, since dispose() no-ops in the error phase, so a stale error survives navigating away and back. Stop sat inside ComposerToolbarScroller, which is the else branch of the dictation ternary, so an agent was unstoppable for the whole recording and transcription window. Moved to the always-rendered right cluster. Also: guard both submission entry points on blocksSubmission, since canSend is derived above voiceInput and cannot include it; move the 4px spacer outside ComposerDictationToolbar's fixed 44px box, where it was overflowing and clipping the collapsed dictation strip; restore pointerEvents="none" on the glass layer with a comment matching the new sibling structure; and rebrand two T3 Code strings in docs. The showsSend divergence now has a regression test. It is the one defect here that is a pure predicate rather than JSX placement, and it is the one most likely to be reintroduced. --- .../src/features/threads/ThreadComposer.tsx | 59 ++++++++++++------- .../voiceInputPresentation.test.ts | 33 +++++++++++ docs/internals/voice-input.md | 2 +- docs/user/composer.md | 2 +- pnpm-workspace.yaml | 2 +- 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 02328e9d6..7656756ed 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -323,8 +323,9 @@ export function ComposerSurface(props: { chrome="none" fallbackClassName="border border-border bg-card-translucent" glassEffectStyle="regular" - // The composer is a passive material containing interactive controls. - // Expo GlassView defaults to non-interactive and both layouts share it. + // Keep native glass out of the interactive content's layout path: the + // content is now a sibling of this layer, not a child of it. + pointerEvents="none" tintColor="transparent" layout={layoutTransition} style={[{ position: "absolute", inset: 0 }, animatedShapeStyle]} @@ -1261,6 +1262,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer const { onSendMessage } = props; const handleSend = useCallback(async () => { + // canSend is derived above voiceInput, so the block lives here. + // Reachable via a hardware-keyboard Return while recording. + if (voiceInput.blocksSubmission) return; if (!canSend) return; const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id); if (inFlightThreadIdsRef.current.has(threadKey)) return; @@ -1292,6 +1296,9 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer voiceInput.blocksSubmission, ]); const handleQueueFollowUp = useCallback(async () => { + // canSend is derived above voiceInput, so the block lives here. + // Reachable via a hardware-keyboard Return while recording. + if (voiceInput.blocksSubmission) return; if (!canSend) return; const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id); if (inFlightThreadIdsRef.current.has(threadKey) || isMutatingSessionInputQueue) return; @@ -1612,6 +1619,10 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer ref={inputRef} multiline value={props.draftMessage} + // Without this the keyboard stays live during dictation, and any + // keystroke makes resolveTranscriptCommit see a changed draft and + // discard the whole transcript as stale. + readOnly={voiceInput.freezesEditor} skills={selectedProviderStatus?.skills ?? []} selection={composerMenu.selection} onChangeText={props.onChangeDraftMessage} @@ -1729,13 +1740,14 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer )} ) : null} + {isExpanded ? : null} {isToolbarVisible ? ( - + ) : null} - {showStopAction ? ( - - ) : null} )} - {voiceInput.isAvailable ? ( - ) : null} - {isVoiceInputPresented ? null : ( + + {/* showsSend, not isVoiceInputPresented: the error phase shows a + status label AND keeps send, so gating on the label strands + the user with no way to send until they dismiss the error. */} + {voicePresentation.showsSend ? ( - )} + ) : null} diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts index caf160937..d497a3e14 100644 --- a/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts @@ -67,3 +67,36 @@ describe("resolveVoiceComposerPresentation", () => { expect(voiceInputFreezesEditor({ phase: "idle", error: null, errorAction: null })).toBe(false); }); }); + +describe("showsSend is not the same predicate as having a status label", () => { + // The composer must gate its send control on showsSend. Gating on the + // presence of a status label instead looks equivalent — both are non-null + // through recording and transcribing — but diverges exactly in the error + // phase, which shows a label AND keeps send. Getting this wrong strands the + // user with no send control until they find the dismiss affordance. + it("keeps send available in the error phase, which also shows a status label", () => { + const presentation = resolveVoiceComposerPresentation( + { phase: "error", error: "No speech was detected.", errorAction: null }, + 0, + ); + expect(presentation.showsSend).toBe(true); + expect(presentation.statusLabel).not.toBeNull(); + }); + + it("hides send only while dictation actually owns the row", () => { + for (const phase of ["preparing", "recording", "transcribing"] as const) { + const presentation = resolveVoiceComposerPresentation( + { phase, error: null, errorAction: null }, + 0, + ); + expect(presentation.showsSend).toBe(false); + expect(presentation.statusLabel).not.toBeNull(); + } + const idle = resolveVoiceComposerPresentation( + { phase: "idle", error: null, errorAction: null }, + 0, + ); + expect(idle.showsSend).toBe(true); + expect(idle.statusLabel).toBeNull(); + }); +}); diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md index dd37f2a86..66f736bff 100644 --- a/docs/internals/voice-input.md +++ b/docs/internals/voice-input.md @@ -1,6 +1,6 @@ # Voice input -> For maintainers. Using T3 Code? See [voice input on iPhone](../user/composer.md#voice-input-on-iphone). +> For maintainers. Using Pylon? See [voice input on iPhone](../user/composer.md#voice-input-on-iphone). Voice input produces editable composer text. The current implementation records on the client and transcribes locally with Apple's `SpeechAnalyzer` and `SpeechTranscriber` on supported iOS 26+ diff --git a/docs/user/composer.md b/docs/user/composer.md index 5932e5e28..6d6121ca5 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -62,7 +62,7 @@ you send it. The first use can download Apple's speech model and needs a network connection. Later transcription works offline for that language. A recording can be up to five minutes long. Canceling voice input, leaving the screen, or an audio interruption discards the new recording and keeps the existing draft -and attachments. T3 Code deletes the local audio file after transcription or cancellation. It sends +and attachments. Pylon deletes the local audio file after transcription or cancellation. It sends only the normal message text when you submit the draft. ## Commands and skills diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a616b04a3..040ee7af7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -157,8 +157,8 @@ patchedDependencies: "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch effect@4.0.0-beta.103: patches/effect@4.0.0-beta.103.patch - expo-modules-jsi@57.0.6: patches/expo-modules-jsi@57.0.6.patch expo-audio@57.0.4: patches/expo-audio@57.0.4.patch + expo-modules-jsi@57.0.6: patches/expo-modules-jsi@57.0.6.patch expo-sharing@57.0.16: patches/expo-sharing@57.0.16.patch react-native-gesture-handler@2.32.0: patches/react-native-gesture-handler@2.32.0.patch react-native-keyboard-controller@1.21.13: patches/react-native-keyboard-controller@1.21.13.patch