diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 4bf265e24..92cf34e86 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -178,6 +178,7 @@ const sharingPlugin: NonNullable[number] = [ supportsText: true, supportsWebUrlWithMaxCount: 1, supportsImageWithMaxCount: 8, + supportsMovieWithMaxCount: 8, supportsFileWithMaxCount: 8, }, }, @@ -338,6 +339,15 @@ const config: ExpoConfig = { }, }, ], + [ + "expo-audio", + { + microphonePermission: "Allow Pylon to use your microphone for voice input.", + recordAudioAndroid: false, + enableBackgroundPlayback: false, + enableBackgroundRecording: false, + }, + ], [ "expo-camera", { diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift index a56619b7d..06dab5e07 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorModule.swift @@ -29,6 +29,9 @@ public class T3ComposerEditorModule: Module { Prop("editable") { (view: T3ComposerEditorView, editable: Bool) in view.setEditable(editable) } + Prop("readOnly") { (view: T3ComposerEditorView, readOnly: Bool) in + view.setReadOnly(readOnly) + } Prop("scrollEnabled") { (view: T3ComposerEditorView, scrollEnabled: Bool) in view.setScrollEnabled(scrollEnabled) } diff --git a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift index 2a8fb8c4e..fe63acc8e 100644 --- a/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift +++ b/apps/mobile/modules/t3-composer-editor/ios/T3ComposerEditorView.swift @@ -60,10 +60,21 @@ private final class ComposerTextAttachment: NSTextAttachment { private final class ComposerTextView: UITextView { private static let pastedImageDirectoryName = "t3-composer-paste" private static let stalePastedImageAge: TimeInterval = 60 * 60 + private static let readOnlyActions = Set([ + "cut:", + "delete:", + "paste:", + "redo:", + "toggleBoldface:", + "toggleItalics:", + "toggleUnderline:", + "undo:", + ]) var onPasteImages: (([String]) -> Void)? var onAttributedMutation: (() -> Void)? var onSubmit: (() -> Void)? + var isReadOnly = false override var keyCommands: [UIKeyCommand]? { var commands = super.keyCommands ?? [] @@ -83,6 +94,9 @@ private final class ComposerTextView: UITextView { } override func canPerformAction(_ action: Selector, withSender sender: Any?) -> Bool { + if isReadOnly && Self.readOnlyActions.contains(NSStringFromSelector(action)) { + return false + } if action == #selector(paste(_:)) { let pasteboard = UIPasteboard.general if pasteboard.hasImages || @@ -96,6 +110,9 @@ private final class ComposerTextView: UITextView { } override func paste(_ sender: Any?) { + guard !isReadOnly else { + return + } let pasteboard = UIPasteboard.general let imageProviders = pasteboard.itemProviders.filter { $0.canLoadObject(ofClass: UIImage.self) @@ -117,6 +134,9 @@ private final class ComposerTextView: UITextView { } override func deleteBackward() { + guard !isReadOnly else { + return + } guard selectedRange.length == 0, selectedRange.location > 0 else { super.deleteBackward() return @@ -160,9 +180,12 @@ private final class ComposerTextView: UITextView { } group.notify(queue: .main) { [weak self] in + guard let self, !self.isReadOnly else { + return + } let urls = images.compactMap { $0 }.compactMap(Self.writeTemporaryImage) if !urls.isEmpty { - self?.onPasteImages?(urls) + self.onPasteImages?(urls) } } } @@ -175,6 +198,9 @@ private final class ComposerTextView: UITextView { } override func cut(_ sender: Any?) { + guard !isReadOnly else { + return + } guard isEditable, selectedRange.length > 0 else { return super.cut(sender) } @@ -306,6 +332,7 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro private var contentInsetVertical: CGFloat = 0 private var shouldAutoFocus = false private var didAutoFocus = false + private var isReadOnly = false private var isApplyingControlledValue = false private var nativeEventCount = 0 private var lastContentSize = CGSize.zero @@ -451,6 +478,11 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro textView.isEditable = editable } + func setReadOnly(_ readOnly: Bool) { + isReadOnly = readOnly + textView.isReadOnly = readOnly + } + func setScrollEnabled(_ scrollEnabled: Bool) { textView.isScrollEnabled = scrollEnabled } @@ -504,13 +536,16 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro replacementText text: String ) -> Bool { restoreBaseTypingAttributes() - return true + return !isReadOnly } public func textDroppableView( _ textDroppableView: UIView & UITextDroppable, proposalForDrop drop: UITextDropRequest ) -> UITextDropProposal { + guard !isReadOnly else { + return UITextDropProposal(operation: .cancel) + } guard droppedImageProviders(in: drop) != nil else { return drop.suggestedProposal } @@ -527,6 +562,9 @@ public final class T3ComposerEditorView: ExpoView, UITextViewDelegate, UITextDro _ textDroppableView: UIView & UITextDroppable, willPerformDrop drop: UITextDropRequest ) { + guard !isReadOnly else { + return + } guard let imageProviders = droppedImageProviders(in: drop) else { return } diff --git a/apps/mobile/package.json b/apps/mobile/package.json index b540c5186..7ae9b69e8 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -53,6 +53,7 @@ "@noble/curves": "catalog:", "@noble/hashes": "catalog:", "@pierre/diffs": "catalog:", + "@react-native-ai/apple": "0.12.0", "@react-native-menu/menu": "^2.0.0", "@react-navigation/elements": "2.9.26", "@react-navigation/native": "7.3.4", @@ -73,6 +74,7 @@ "effect": "catalog:", "expo": "~57.0.18", "expo-asset": "~57.0.15", + "expo-audio": "~57.0.4", "expo-auth-session": "~57.0.10", "expo-blur": "~57.0.2", "expo-build-properties": "~57.0.15", @@ -141,7 +143,8 @@ "autolinking": { "buildFromSource": [ "react-native-screens", - "@react-native-menu/menu" + "@react-native-menu/menu", + "expo-audio" ] } }, diff --git a/apps/mobile/src/components/AppSymbol.tsx b/apps/mobile/src/components/AppSymbol.tsx index 71590c283..f9f50b0ac 100644 --- a/apps/mobile/src/components/AppSymbol.tsx +++ b/apps/mobile/src/components/AppSymbol.tsx @@ -54,6 +54,7 @@ import { IconMoon, IconNetwork, IconPalette, + IconPhoto, IconPin, IconPinnedOff, IconPlayerPlay, @@ -132,6 +133,7 @@ const ANDROID_ICON_BY_SF_SYMBOL: Partial> = { magnifyingglass: IconSearch, paintbrush: IconPalette, "person.crop.circle": IconUserCircle, + photo: IconPhoto, pin: IconPin, "pin.slash": IconPinnedOff, play: IconPlayerPlay, diff --git a/apps/mobile/src/components/ComposerAttachmentButton.tsx b/apps/mobile/src/components/ComposerAttachmentButton.tsx new file mode 100644 index 000000000..1af72d888 --- /dev/null +++ b/apps/mobile/src/components/ComposerAttachmentButton.tsx @@ -0,0 +1,55 @@ +import type { MenuAction } from "@react-native-menu/menu"; +import { Pressable } from "react-native"; + +import { SymbolView } from "./AppSymbol"; +import { ControlPillMenu } from "./ControlPill"; + +const ATTACHMENT_MENU_ACTIONS: MenuAction[] = [ + { id: "photos", title: "Photo Library", image: "photo" }, + { id: "files", title: "Choose Files", image: "folder" }, +]; + +export function ComposerAttachmentButton(props: { + readonly disabled?: boolean; + readonly supportsFiles: boolean; + readonly onPickMedia: () => Promise; + readonly onPickFiles: () => Promise; +}) { + const button = ( + void props.onPickMedia()} + > + + + ); + + if (props.disabled || !props.supportsFiles) { + return button; + } + + return ( + { + if (nativeEvent.event === "photos") { + void props.onPickMedia(); + } else if (nativeEvent.event === "files") { + void props.onPickFiles(); + } + }} + > + {button} + + ); +} diff --git a/apps/mobile/src/components/ComposerToolbar.tsx b/apps/mobile/src/components/ComposerToolbar.tsx index e13fc9c80..65a92f7ff 100644 --- a/apps/mobile/src/components/ComposerToolbar.tsx +++ b/apps/mobile/src/components/ComposerToolbar.tsx @@ -113,6 +113,7 @@ export function ComposerToolbarRow(props: { export function ComposerToolbarScroller(props: { readonly children: ReactNode; + readonly align?: "start" | "end"; /** Only for non-Uniwind surfaces such as the native terminal palette. */ readonly fadeOpaque?: string; /** Only for non-Uniwind surfaces such as the native terminal palette. */ @@ -167,6 +168,8 @@ export function ComposerToolbarScroller(props: { showsHorizontalScrollIndicator={false} contentContainerStyle={{ alignItems: "center", + flexGrow: props.align === "end" ? 1 : undefined, + justifyContent: props.align === "end" ? "flex-end" : undefined, gap: COMPOSER_TOOLBAR_GAP, paddingLeft: 0, paddingRight: props.contentPaddingRight ?? 1, @@ -214,6 +217,46 @@ export function ComposerToolbarScroller(props: { ); } +export function ComposerActionButton(props: { + readonly accessibilityLabel: string; + readonly disabled?: boolean; + readonly icon: ComponentProps["name"]; + readonly onPress: () => void; + readonly variant?: "primary" | "danger"; +}) { + return ( + + + + + + ); +} + export function ComposerToolbarButton(props: { readonly icon?: ComponentProps["name"]; readonly iconNode?: ReactNode; diff --git a/apps/mobile/src/components/ControlPill.tsx b/apps/mobile/src/components/ControlPill.tsx index 1809a6547..c2b73ee3c 100644 --- a/apps/mobile/src/components/ControlPill.tsx +++ b/apps/mobile/src/components/ControlPill.tsx @@ -6,16 +6,47 @@ import { type ComponentProps, type ReactElement, type ReactNode, + useMemo, useRef, } from "react"; -import { Platform, Pressable, View } from "react-native"; +import { Platform, Pressable, View, type ColorValue } from "react-native"; +import { withUniwind } from "uniwind"; import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider"; import { cn } from "../lib/cn"; +import { withMenuActionIconColors } from "../lib/menu-action-colors"; import { AndroidAnchoredMenu } from "./AndroidAnchoredMenu"; import { SymbolView } from "./AppSymbol"; import { AppText as Text } from "./AppText"; +const ThemedMenuView = withUniwind( + function NativeMenuView({ + iconColor, + destructiveIconColor, + ...props + }: ComponentProps & { + readonly iconColor?: ColorValue; + readonly destructiveIconColor?: ColorValue; + }) { + const actions = useMemo( + () => + withMenuActionIconColors(props.actions, { + icon: iconColor, + destructiveIcon: destructiveIconColor, + }), + [props.actions, iconColor, destructiveIconColor], + ); + return ; + }, + { + iconColor: { fromClassName: "iconColorClassName", styleProperty: "accentColor" }, + destructiveIconColor: { + fromClassName: "destructiveIconColorClassName", + styleProperty: "accentColor", + }, + }, +); + export function ControlPill(props: { readonly icon?: ComponentProps["name"]; readonly iconNode?: ReactNode; @@ -177,8 +208,13 @@ export function ControlPillMenu( }); } return ( - + {children} - + ); } diff --git a/apps/mobile/src/components/GlassSurface.tsx b/apps/mobile/src/components/GlassSurface.tsx index c194d6b55..577c43aa8 100644 --- a/apps/mobile/src/components/GlassSurface.tsx +++ b/apps/mobile/src/components/GlassSurface.tsx @@ -1,5 +1,5 @@ import { GlassView, isGlassEffectAPIAvailable } from "expo-glass-effect"; -import type { ReactNode } from "react"; +import type { ReactNode, Ref } from "react"; import { Platform, useColorScheme, @@ -20,6 +20,7 @@ const ThemedGlassView = withUniwind(GlassView, { }); interface GlassSurfaceProps extends ViewProps { + readonly ref?: Ref; readonly children: ReactNode; readonly glassEffectStyle?: "clear" | "regular" | "none"; readonly tintColor?: ColorValue; @@ -32,6 +33,7 @@ interface GlassSurfaceProps extends ViewProps { } export function GlassSurface({ + ref, children, glassEffectStyle = "regular", chrome = "default", @@ -68,6 +70,7 @@ export function GlassSurface({ return ( { expect(removeOwnedFile).toHaveBeenCalledWith(file.value); }); + it.each([ + { value: "file:///shared/clip.MOV", mimeType: "video/quicktime", originalName: "clip.MOV" }, + { value: "content://media/videos/12", mimeType: "video/mp4", originalName: "clip.mp4" }, + ])("imports a shared video from $value without reading it as an image", async (video) => { + const sizeBytes = 20 * 1024 * 1024; + const fileUri = `file:///documents/${video.originalName}`; + const readBase64 = vi.fn(async () => "unused"); + const persistFile = vi.fn(async () => fileUri); + const removeOwnedFile = vi.fn(async () => undefined); + + const result = await buildIncomingShareDraft({ + id: "share-video", + createdAt: "2026-08-30T10:00:00.000Z", + payloads: [{ ...video, shareType: "video" }], + resolvedPayloads: [], + fileReader: { readBase64, persistFile, readSize: async () => sizeBytes, removeOwnedFile }, + }); + + expect(result.warnings).toEqual([]); + expect(result.attachments).toEqual([ + { + id: "share-video:file:0", + type: "file", + name: video.originalName, + mimeType: video.mimeType, + sizeBytes, + fileUri, + }, + ]); + expect(readBase64).not.toHaveBeenCalled(); + expect(removeOwnedFile).toHaveBeenCalledWith(video.value); + expect( + selectIncomingShareAttachments({ + attachments: result.attachments, + maxFileAttachmentBytes: 50 * 1024 * 1024, + }), + ).toEqual({ attachments: result.attachments, warnings: [] }); + expect( + selectIncomingShareAttachments({ + attachments: result.attachments, + maxFileAttachmentBytes: 10 * 1024 * 1024, + }), + ).toEqual({ + attachments: [], + warnings: [`'${video.originalName}' exceeds the 10 MB attachment limit.`], + }); + }); + it("reports an unreadable shared file without calling it oversized", async () => { const file: SharePayload = { shareType: "file", diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index f3f371227..e9b621c6f 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -12,6 +12,7 @@ import { KeyboardStickyView, useKeyboardState, } from "react-native-keyboard-controller"; +import Animated from "react-native-reanimated"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { useFontFamily } from "../../lib/useFontFamily"; @@ -24,19 +25,28 @@ import { PROVIDER_SEND_TURN_MAX_ATTACHMENTS } from "@t3tools/contracts"; import { ComposerEditor, type ComposerEditorHandle } from "../../components/ComposerEditor"; import { + ComposerActionButton, ComposerInlineControl, - ComposerToolbarButton, ComposerToolbarRow, ComposerToolbarScroller, } from "../../components/ComposerToolbar"; import { AndroidScreenHeader } from "../../components/AndroidScreenHeader"; +import { ComposerAttachmentButton } from "../../components/ComposerAttachmentButton"; import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStrip"; import { ProviderIcon } from "../../components/ProviderIcon"; import { SymbolView } from "../../components/AppSymbol"; import { AppText as Text } from "../../components/AppText"; -import { ComposerSurface } from "./ThreadComposer"; +import { COMPOSER_LAYOUT_TRANSITION, ComposerSurface } from "./ThreadComposer"; import { ComposerCommandPopover } from "./ComposerCommandPopover"; import { useComposerCommandMenu } from "./use-composer-command-menu"; +import { + ComposerDictationCancelAction, + ComposerDictationPrimaryAction, + ComposerDictationStatus, + ComposerDictationToolbar, +} from "../voice-input/ComposerDictationControl"; +import { useVoiceInputController } from "../voice-input/useVoiceInputController"; +import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; import { useThreadSettingsSheetPresentation, type NavigationWithFinishTransitioning, @@ -46,7 +56,7 @@ import { makeTurnCommandMetadata } from "../../lib/commandMetadata"; import { convertPastedImagesToAttachments, pickComposerFiles, - pickComposerImages, + pickComposerMedia, } from "../../lib/composerImages"; import { useScaledTextRole } from "../settings/appearance/useScaledTextRole"; import { @@ -242,6 +252,7 @@ export function NewTaskDraftScreen(props: { // snapshot and thread-only commands stay hidden until the thread exists. const composerMenu = useComposerCommandMenu({ draftMessage: flow.prompt, + ownerKey: flow.draftKey, environmentId: selectedProject?.environmentId ?? null, projectCwd: (flow.workspaceMode === "worktree" @@ -255,6 +266,19 @@ export function NewTaskDraftScreen(props: { onChangeDraftMessage: flow.setPrompt, onUpdateInteractionMode: flow.setInteractionMode, }); + const voiceInput = useVoiceInputController({ + ownerKey: flow.draftKey, + draftMessage: flow.prompt, + selection: composerMenu.selection, + disabled: isIncomingShareTransferPending || isImportingShare || flow.submitting, + onChangeDraftMessage: flow.setPrompt, + onChangeSelection: composerMenu.onSelectionChange, + }); + const voicePresentation = resolveVoiceComposerPresentation( + voiceInput.state, + voiceInput.elapsedSeconds, + ); + const isVoiceInputPresented = voicePresentation.statusLabel !== null; usePreventRemove( (isIncomingShareTransferPending && !isProjectPickerReturnActive) || isCancellingShareImport || @@ -696,12 +720,20 @@ export function NewTaskDraftScreen(props: { }); const showBranchLoading = flow.branchesLoading && flow.availableBranches.length === 0; - async function handlePickImages(): Promise { - if (isComposerInteractionLocked) { + async function handlePickMedia(): Promise { + if (isComposerInteractionLocked || voiceInput.isBusy) { return; } - const result = await pickComposerImages({ existingCount: flow.attachments.length }); - const rejectedCount = result.images.length > 0 ? flow.appendAttachments(result.images) : 0; + const capabilities = selectedEnvironmentServerConfig?.environment.capabilities; + const result = await pickComposerMedia({ + existingCount: flow.attachments.length, + maxVideoBytes: + capabilities?.attachmentUploads === true + ? capabilities.fileAttachments?.maxUploadBytes + : undefined, + }); + const rejectedCount = + result.attachments.length > 0 ? flow.appendAttachments(result.attachments) : 0; const problems = [ ...(result.error ? [result.error] : []), ...(rejectedCount > 0 @@ -709,12 +741,12 @@ export function NewTaskDraftScreen(props: { : []), ]; if (problems.length > 0) { - Alert.alert("Could not attach photo", problems.join("\n\n")); + Alert.alert("Could not attach photo or video", problems.join("\n\n")); } } async function handlePickFiles(): Promise { - if (isComposerInteractionLocked) { + if (isComposerInteractionLocked || voiceInput.isBusy) { return; } const maxBytes = @@ -759,6 +791,7 @@ export function NewTaskDraftScreen(props: { ); async function handleStart(): Promise { + if (voiceInput.blocksSubmission) return; const selectedProject = flow.selectedProject; const draftKey = flow.draftKey; if (!selectedProject || !draftKey) { @@ -945,6 +978,7 @@ export function NewTaskDraftScreen(props: { isIncomingShareReady && !isImportingShare && !flow.submitting && + !voiceInput.blocksSubmission && !(flow.workspaceMode === "worktree" && !flow.selectedBranchName); const promptEditor = ( - {composerMenu.trigger && composerMenu.items.length > 0 ? ( + + {!voiceInput.isBusy && composerMenu.trigger && composerMenu.items.length > 0 ? ( {workspaceControls} {flow.attachments.length > 0 ? ( - + undefined : flow.removeAttachment} + onRemove={ + isComposerInteractionLocked || voiceInput.isBusy + ? () => undefined + : flow.removeAttachment + } /> ) : null} - {promptEditor} + {promptEditor} + - - - { - if (selectedEnvironmentServerConfig?.environment.capabilities.fileAttachments) { - Alert.alert("Add attachment", undefined, [ - { text: "Photos", onPress: () => void handlePickImages() }, - { text: "Files", onPress: () => void handlePickFiles() }, - { text: "Cancel", style: "cancel" }, - ]); - return; - } - void handlePickImages(); - }} - showChevron={false} - /> - - } - label={flow.selectedModelOption?.label ?? "Choose model"} - maxWidth={152} - onPress={settingsSheetPresentation.open} - /> - {flow.planModeEnabled ? ( - - flow.setInteractionMode(flow.interactionMode === "plan" ? "default" : "plan") - } - showChevron={false} + + + + - ) : null} - - void handleStart()} - showChevron={false} - variant="primary" - /> - + {isVoiceInputPresented ? ( + + ) : ( + <> + + + + } + label={flow.selectedModelOption?.label ?? "Choose model"} + maxWidth={152} + onPress={settingsSheetPresentation.open} + /> + {flow.planModeEnabled ? ( + + flow.setInteractionMode( + flow.interactionMode === "plan" ? "default" : "plan", + ) + } + showChevron={false} + /> + ) : null} + + + )} + + {voicePresentation.showsSend ? ( + void handleStart()} + variant="primary" + /> + ) : null} + + + ); @@ -1230,10 +1302,17 @@ export function NewTaskDraftScreen(props: { {heroViewport} - {composerDock} + + {composerDock} + ); diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 8c89e64ec..d3cc019fb 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -59,7 +59,16 @@ import type { } from "@t3tools/contracts"; import { StackActions, useFocusEffect, useNavigation } from "@react-navigation/native"; import type { ReactNode } from "react"; -import { memo, useCallback, useEffect, useMemo, useRef, useState, type RefObject } from "react"; +import { + memo, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, + type RefObject, +} from "react"; import { ActivityIndicator, Alert, @@ -79,6 +88,10 @@ import Animated, { FadeOut, FadeOutDown, LinearTransition, + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, } from "react-native-reanimated"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { presentMobileContextWindow } from "../../lib/contextWindow"; @@ -87,6 +100,7 @@ import { scopedThreadKey } from "../../lib/scopedEntities"; import { AppText as Text, AppTextInput as TextInput } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; +import { ComposerAttachmentButton } from "../../components/ComposerAttachmentButton"; import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStrip"; import { GlassSurface } from "../../components/GlassSurface"; import { ComposerEditor, type ComposerEditorHandle } from "../../components/ComposerEditor"; @@ -111,6 +125,15 @@ import type { RemoteClientConnectionState } from "../../lib/connection"; import { resolveProviderOptionDescriptors } from "../../lib/providerOptions"; import { ComposerCommandPopover } from "./ComposerCommandPopover"; import { useComposerCommandMenu } from "./use-composer-command-menu"; +import { + ComposerDictationCancelAction, + ComposerDictationDraftContent, + ComposerDictationPrimaryAction, + ComposerDictationStatus, + ComposerDictationToolbar, +} from "../voice-input/ComposerDictationControl"; +import { useVoiceInputController } from "../voice-input/useVoiceInputController"; +import { resolveVoiceComposerPresentation } from "../voice-input/voiceInputPresentation"; import { type ExistingThreadSettingsRouteSession, useExistingThreadSettingsRoutePresentation, @@ -193,7 +216,7 @@ export interface ThreadComposerProps { readonly projectCwd: string | null; readonly editorRef?: RefObject; readonly onChangeDraftMessage: (value: string) => void; - readonly onPickDraftImages: () => Promise; + readonly onPickDraftMedia: () => Promise; readonly onPickDraftFiles: () => Promise; readonly onNativePasteImages: (uris: ReadonlyArray) => Promise; readonly onRemoveDraftImage: (imageId: string) => void; @@ -246,12 +269,14 @@ export const COMPOSER_TRANSITION_DURATION_MS = 220; export const COMPOSER_LAYOUT_TRANSITION = Platform.OS === "android" ? undefined - : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS); + : LinearTransition.duration(COMPOSER_TRANSITION_DURATION_MS).reduceMotion(ReduceMotion.System); + +const AnimatedGlassSurface = Animated.createAnimatedComponent(GlassSurface); export function ComposerSurface(props: { readonly children: ReactNode; readonly style: ViewStyle; - /** Existing thread composers morph between pill and card layouts. */ + /** Morphs between the compact and expanded composer layouts. */ readonly animateLayout?: boolean; }) { // Drop shadow lives on a wrapper: `overflow: "hidden"` on the surface itself @@ -264,8 +289,28 @@ export function ComposerSurface(props: { // style and `shadowOpacity: 1` would fall back to RN's default opaque black. const shadowColor = useUniwindTheme()["--color-primary-shadow"]; const isDarkMode = useColorScheme() === "dark"; + // #8793's shape morph, adopted without its toolbar restructure. Animating the + // radius on a shared value keeps the pill/card corners interpolating with the + // layout instead of snapping on the first frame. Every native frame carries + // the same transition: animating only the outer clip leaves the glass and the + // content at their final height immediately. + const targetBorderRadius = + typeof props.style.borderRadius === "number" ? props.style.borderRadius : 0; + const animatedBorderRadius = useSharedValue(targetBorderRadius); + const shouldAnimate = props.animateLayout !== false && Platform.OS !== "android"; + useLayoutEffect(() => { + animatedBorderRadius.value = shouldAnimate + ? withTiming(targetBorderRadius, { + duration: COMPOSER_TRANSITION_DURATION_MS, + reduceMotion: ReduceMotion.System, + }) + : targetBorderRadius; + }, [animatedBorderRadius, shouldAnimate, targetBorderRadius]); + const animatedShapeStyle = useAnimatedStyle(() => ({ + borderRadius: animatedBorderRadius.value, + })); + const layoutTransition = shouldAnimate ? COMPOSER_LAYOUT_TRANSITION : undefined; const shadowStyle: ViewStyle = { - borderRadius: props.style.borderRadius, shadowColor, shadowOpacity: isDarkMode ? 0.35 : 0.12, shadowRadius: 14, @@ -274,21 +319,27 @@ export function ComposerSurface(props: { }; return ( - - + + {null} + + {props.children} - + ); } @@ -474,9 +525,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer setIsFocused(false); onEditorFocusChange?.(false); }, [onEditorFocusChange]); + // #8843: an empty composer shows the interrupt button while the agent works; + // adding text or an attachment swaps it for send. const showStopAction = - props.selectedThread.session?.status === "running" || - props.selectedThread.session?.status === "starting"; + !hasContent && + (props.selectedThread.session?.status === "running" || + props.selectedThread.session?.status === "starting"); const currentModelSelection = props.selectedThread.modelSelection; const currentRuntimeMode = resolveModelSelectionRuntimeMode( @@ -1179,8 +1233,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer ); // ── Composer command menu ──────────────────────────────── + const composerOwnerKey = scopedThreadKey(props.environmentId, props.selectedThread.id); + const composerMenu = useComposerCommandMenu({ draftMessage: props.draftMessage, + ownerKey: composerOwnerKey, environmentId: props.environmentId, projectCwd: props.projectCwd, selectedProviderStatus, @@ -1190,10 +1247,28 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer onChangeDraftMessage: props.onChangeDraftMessage, onUpdateInteractionMode: props.onUpdateInteractionMode, }); + const voiceInput = useVoiceInputController({ + ownerKey: composerOwnerKey, + draftMessage: props.draftMessage, + selection: composerMenu.selection, + onChangeDraftMessage: props.onChangeDraftMessage, + onChangeSelection: composerMenu.onSelectionChange, + }); + const voicePresentation = resolveVoiceComposerPresentation( + voiceInput.state, + voiceInput.elapsedSeconds, + ); + const isVoiceInputPresented = voicePresentation.statusLabel !== null; + // An open draft stays visible; only a collapsed composer becomes a voice strip. + const showsCompactDictation = isVoiceInputPresented && !isExpanded; + const isToolbarVisible = isExpanded || isVoiceInputPresented; const { onSendMessage } = props; const handleSend = useCallback(async () => { + // canSend is derived above voiceInput, so the block lives here. + // Reachable via a hardware-keyboard Return while recording. + if (voiceInput.blocksSubmission) return; if (!canSend) return; const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id); if (inFlightThreadIdsRef.current.has(threadKey)) return; @@ -1222,8 +1297,12 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer props.environmentLabel, props.selectedThread.id, props.selectedThread.title, + voiceInput.blocksSubmission, ]); const handleQueueFollowUp = useCallback(async () => { + // canSend is derived above voiceInput, so the block lives here. + // Reachable via a hardware-keyboard Return while recording. + if (voiceInput.blocksSubmission) return; if (!canSend) return; const threadKey = scopedThreadKey(props.environmentId, props.selectedThread.id); if (inFlightThreadIdsRef.current.has(threadKey) || isMutatingSessionInputQueue) return; @@ -1387,7 +1466,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer }), [currentModelOption?.capabilities, currentModelSelection.options], ); - const settingsOwnerId = scopedThreadKey(props.environmentId, props.selectedThread.id); + const settingsOwnerId = composerOwnerKey; const settingsRouteSession = useMemo( () => ({ ownerId: settingsOwnerId, @@ -1478,7 +1557,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer layout={COMPOSER_LAYOUT_TRANSITION} style={{ maxWidth: props.contentMaxWidth }} > - {composerMenu.trigger && composerMenu.items.length > 0 ? ( + {!voiceInput.isBusy && composerMenu.trigger && composerMenu.items.length > 0 ? ( - {/* Attachment strip — inside the card, above the text input */} - {isExpanded ? ( - 0 ? "pb-2.5" : undefined} - entering={FadeIn.duration(160)} - exiting={FadeOut.duration(120)} - > - - ) : null} - - - void props.onNativePasteImages(uris)} - placeholder={props.placeholder} - onFocus={handleFocus} - onBlur={handleBlur} - onSubmit={handleSend} - scrollEnabled={isExpanded} - // Android: collapsed single line centers natively (gravity) in - // a pill-height box matching the send button; iOS keeps insets. - singleLineCentered={!isExpanded} - contentInsetVertical={isExpanded || Platform.OS === "android" ? 0 : 6} - style={ - isExpanded - ? { - minHeight: 72, - maxHeight: 160, - paddingHorizontal: 4, - paddingVertical: 4, - } - : { - height: 36, - } - } - textStyle={{ - ...bodyText, - color: foregroundColor, - }} - /> - - {!isExpanded && props.draftAttachments.length > 0 ? ( - - {props.draftAttachments.slice(0, 3).map((attachment) => - attachment.type === "image" ? ( - onPressImage(attachment.previewUri)} - > - - - ) : ( - - - - ), - )} - {props.draftAttachments.length > 3 ? ( - - - +{props.draftAttachments.length - 3} - - - ) : null} - ) : null} - {!isExpanded && props.contextWindow ? ( - - ) : null} - {!isExpanded ? ( - - {showStopAction ? ( - - 0 ? ( + + {props.draftAttachments.slice(0, 3).map((attachment) => + attachment.type === "image" ? ( + onPressImage(attachment.previewUri)} + > + + + ) : ( + + + + ), + )} + {props.draftAttachments.length > 3 ? ( + + + +{props.draftAttachments.length - 3} + + + ) : null} + + ) : null} + {!isExpanded && props.contextWindow ? ( + + ) : null} + {!isExpanded && !voiceInput.isBusy ? ( + + {voiceInput.isAvailable ? ( + - {canQueueFollowUp ? ( + ) : null} + {showStopAction ? ( + - ) : null} - - ) : ( - - )} - - ) : null} - {isExpanded ? ( - - - { - if (props.serverConfig?.environment.capabilities.fileAttachments) { - Alert.alert("Add attachment", undefined, [ - { text: "Photos", onPress: () => void props.onPickDraftImages() }, - { text: "Files", onPress: () => void props.onPickDraftFiles() }, - { text: "Cancel", style: "cancel" }, - ]); - return; - } - void props.onPickDraftImages(); - }} - showChevron={false} - /> - {quickQuestionAvailable ? ( - setQuickQuestionOpenScopeKey(quickQuestionScopeKey)} + {canQueueFollowUp ? ( + + ) : null} + + ) : ( + - ) : null} - - } - label={currentModelOption?.label ?? currentModelSelection.model} - maxWidth={152} - onPress={openSettings} + )} + + ) : null} + {isExpanded ? : null} + + {isToolbarVisible ? ( + + + - {sessionHarnessRefinementActions.length > 0 ? ( - { - if ( - parseSessionHarnessRefinementAction( - nativeEvent.event, - sessionHarnessRefinementScopeKey, - ) === "refine" - ) { - confirmSessionHarnessRefinement(sessionHarnessRefinementScopeKey); - } - }} - > - + ) : ( + + {/* #8843 replaces the Alert with a native menu anchored to + the button. Kept inside Pylon's scroller rather than + upstream's fixed left group. */} + - - ) : null} - {props.sessionGoal ? ( - - setQuickQuestionOpenScopeKey(quickQuestionScopeKey)} + /> + ) : null} + } + label={currentModelOption?.label ?? currentModelSelection.model} + maxWidth={152} + onPress={openSettings} /> - - ) : null} - {props.contextWindow || - (props.sessionCompaction?.available && sessionCompactionScopeKey) ? ( - props.sessionCompaction?.available && sessionCompactionScopeKey ? ( - - handleSessionCompactionAction(nativeEvent.event) - } - > + {sessionHarnessRefinementActions.length > 0 ? ( + { + if ( + parseSessionHarnessRefinementAction( + nativeEvent.event, + sessionHarnessRefinementScopeKey, + ) === "refine" + ) { + confirmSessionHarnessRefinement(sessionHarnessRefinementScopeKey); + } + }} + > + + + ) : null} + {props.sessionGoal ? ( + + + + ) : null} + {props.contextWindow || + (props.sessionCompaction?.available && sessionCompactionScopeKey) ? ( + props.sessionCompaction?.available && sessionCompactionScopeKey ? ( + + handleSessionCompactionAction(nativeEvent.event) + } + > + + + ) : props.contextWindow ? ( + + ) : null + ) : null} + {sessionAgentActions.length > 0 ? ( + + handleSessionAgentAction(nativeEvent.event) + } + > + + + ) : null} + {showSessionInputQueueModes && props.sessionInputQueue ? ( + + handleSessionInputQueueAction(nativeEvent.event) + } + > + 0 ? `Inputs ${sessionQueueCount}` : "Inputs"} + /> + + ) : null} + {showSessionAgentDepth && props.sessionAgentDepth !== null ? ( + + void setSessionAgentDepth(nativeEvent.event) + } + > + + + ) : null} + {sessionResourceInventory !== null ? ( setIsSessionResourcesOpen(true)} + showChevron={false} /> - - ) : props.contextWindow ? ( - - ) : null - ) : null} - {sessionAgentActions.length > 0 ? ( - handleSessionAgentAction(nativeEvent.event)} - > - - - ) : null} - {showSessionInputQueueModes && props.sessionInputQueue ? ( - - handleSessionInputQueueAction(nativeEvent.event) - } - > + ) : showSessionResourceReload ? ( + void reloadSessionResources()} + showChevron={false} + /> + ) : null} + + )} + + {/* Stop lives outside the dictation ternary: an agent must stay + stoppable for the whole recording and transcription window. */} + {showStopAction ? ( 0 ? `Inputs ${sessionQueueCount}` : "Inputs"} + accessibilityLabel="Stop" + icon="stop.fill" + variant="danger" + onPress={props.onStopThread} + showChevron={false} /> - - ) : null} - {showSessionAgentDepth && props.sessionAgentDepth !== null ? ( - - void setSessionAgentDepth(nativeEvent.event) - } - > + ) : null} + + {/* showsSend, not isVoiceInputPresented: the error phase shows a + status label AND keeps send, so gating on the label strands + the user with no way to send until they dismiss the error. */} + {voicePresentation.showsSend ? ( - - ) : null} - {sessionResourceInventory !== null ? ( - setIsSessionResourcesOpen(true)} - showChevron={false} - /> - ) : showSessionResourceReload ? ( - void reloadSessionResources()} - showChevron={false} - /> - ) : null} - {showStopAction ? ( - - ) : null} - - - + ) : null} + + + ) : null} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index aa9cae842..c18358a76 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -155,7 +155,7 @@ export interface ThreadDetailScreenProps { readonly onHeaderMaterialVisibilityChange?: (visible: boolean) => void; readonly onOpenConnectionEditor: () => void; readonly onChangeDraftMessage: (value: string) => void; - readonly onPickDraftImages: () => Promise; + readonly onPickDraftMedia: () => Promise; readonly onPickDraftFiles: () => Promise; readonly onNativePasteImages: (uris: ReadonlyArray) => Promise; readonly onRemoveDraftImage: (imageId: string) => void; @@ -334,6 +334,14 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread const selectedThreadKeyRef = useRef(selectedThreadKey); const lastScrolledSubmittedMessageIdRef = useRef(null); const [composerExpanded, setComposerExpanded] = useState(false); + const [composerFocused, setComposerFocused] = useState(false); + const handleComposerFocusChange = useCallback( + (focused: boolean) => { + setComposerFocused(focused); + handleOwnedInputFocusChange(focused); + }, + [handleOwnedInputFocusChange], + ); const [anchorMessageId, setAnchorMessageId] = useState(null); const [submittedMessageId, setSubmittedMessageId] = useState(null); const [endFollowEnabled, setEndFollowEnabled] = useState(true); @@ -344,7 +352,10 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread // animation, so the composer would ride down flush to the screen edge and // then snap up into the inset. On iOS blur precedes the hide, so the // focus-keyed inset is already in place while the composer rides down. - const composerBottomInset = (Platform.OS === "android" ? isKeyboardVisible : composerExpanded) + // Dictation keeps that focus while the composer switches to its compact pill. + const composerBottomInset = ( + Platform.OS === "android" ? isKeyboardVisible : composerExpanded || composerFocused + ) ? 0 : Math.max(insets.bottom, 12); const contentPresentationKind = props.contentPresentation.kind; @@ -596,7 +607,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread useLayoutEffect(() => { selectedThreadKeyRef.current = selectedThreadKey; - }, [selectedThreadKey]); + // A replaced or unmounted native editor may not emit a blur event. + setComposerFocused(false); + }, [selectedThreadKey, showContent]); useEffect(() => { setAnchorMessageId(null); @@ -937,7 +950,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread projectCwd={props.projectWorkspaceRoot} bottomInset={hasBelowEditorWidgets ? 0 : composerBottomInset} onChangeDraftMessage={props.onChangeDraftMessage} - onPickDraftImages={props.onPickDraftImages} + onPickDraftMedia={props.onPickDraftMedia} onPickDraftFiles={props.onPickDraftFiles} onNativePasteImages={props.onNativePasteImages} onRemoveDraftImage={props.onRemoveDraftImage} @@ -960,7 +973,7 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread onUpdateRuntimeMode={props.onUpdateThreadRuntimeMode} onUpdateInteractionMode={props.onUpdateThreadInteractionMode} onExpandedChange={setComposerExpanded} - onEditorFocusChange={handleOwnedInputFocusChange} + onEditorFocusChange={handleComposerFocusChange} /> diff --git a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx index 9abaeff2e..fd8a46419 100644 --- a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx @@ -901,7 +901,7 @@ function ThreadRouteContent( usesAutomaticContentInsets={usesNativeHeaderGlass} onOpenConnectionEditor={handleOpenConnectionEditor} onChangeDraftMessage={composer.onChangeDraftMessage} - onPickDraftImages={composer.onPickDraftImages} + onPickDraftMedia={composer.onPickDraftMedia} onPickDraftFiles={composer.onPickDraftFiles} onNativePasteImages={composer.onNativePasteImages} onRemoveDraftImage={composer.onRemoveDraftImage} diff --git a/apps/mobile/src/features/threads/use-composer-command-menu.test.ts b/apps/mobile/src/features/threads/use-composer-command-menu.test.ts index 3895983f4..22e792673 100644 --- a/apps/mobile/src/features/threads/use-composer-command-menu.test.ts +++ b/apps/mobile/src/features/threads/use-composer-command-menu.test.ts @@ -14,7 +14,7 @@ vi.mock("../../state/use-composer-path-search", () => ({ import type { ComposerCommandMenuProvider } from "./use-composer-command-menu"; -const { buildComposerCommandItems, composerCommandReplacement } = +const { buildComposerCommandItems, composerCommandReplacement, composerSelectionAtEnd } = await import("./use-composer-command-menu"); function skill(overrides: Partial & { name: string }): ServerProviderSkill { @@ -231,3 +231,9 @@ describe("composerCommandReplacement", () => { ).toBe("/model "); }); }); + +describe("composerSelectionAtEnd", () => { + it("resets a changed draft owner to the new draft end", () => { + expect(composerSelectionAtEnd("queued task 🧪")).toEqual({ start: 14, end: 14 }); + }); +}); diff --git a/apps/mobile/src/features/threads/use-composer-command-menu.ts b/apps/mobile/src/features/threads/use-composer-command-menu.ts index 5bb4a10dd..54af71103 100644 --- a/apps/mobile/src/features/threads/use-composer-command-menu.ts +++ b/apps/mobile/src/features/threads/use-composer-command-menu.ts @@ -25,7 +25,7 @@ import { normalizeSearchQuery, scoreQueryMatch, } from "@t3tools/shared/searchRanking"; -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { ComposerEditorSelection } from "../../components/ComposerEditor"; import { useComposerPathSearch } from "../../state/use-composer-path-search"; @@ -271,6 +271,10 @@ export function composerCommandReplacement(item: ComposerCommandItem): string | } } +export function composerSelectionAtEnd(draftMessage: string): ComposerEditorSelection { + return { start: draftMessage.length, end: draftMessage.length }; +} + /** * Composer autocomplete shared by the thread composer and the unsent New task * draft. Owns the caret selection it needs for trigger detection, so callers @@ -278,6 +282,7 @@ export function composerCommandReplacement(item: ComposerCommandItem): string | */ export function useComposerCommandMenu({ draftMessage, + ownerKey, environmentId, projectCwd, selectedProviderStatus, @@ -289,6 +294,7 @@ export function useComposerCommandMenu({ onUpdateInteractionMode, }: { readonly draftMessage: string; + readonly ownerKey: string | null; readonly environmentId: EnvironmentId | null; readonly projectCwd: string | null; readonly selectedProviderStatus: ServerProvider | null; @@ -299,10 +305,8 @@ export function useComposerCommandMenu({ readonly onChangeDraftMessage: (value: string) => void; readonly onUpdateInteractionMode?: (mode: ProviderInteractionMode) => void; }) { - const [selection, setSelection] = useState(() => ({ - start: draftMessage.length, - end: draftMessage.length, - })); + const [selection, setSelection] = useState(() => composerSelectionAtEnd(draftMessage)); + const previousOwnerKeyRef = useRef(ownerKey); const onSelectionChange = useCallback((nextSelection: ComposerEditorSelection) => { setSelection(nextSelection); @@ -318,6 +322,11 @@ export function useComposerCommandMenu({ return { start, end: selectionEnd }; }); }, [draftMessage.length]); + useEffect(() => { + if (previousOwnerKeyRef.current === ownerKey) return; + previousOwnerKeyRef.current = ownerKey; + setSelection(composerSelectionAtEnd(draftMessage)); + }, [draftMessage, ownerKey]); const trigger = useMemo(() => { if (!enabled || selection.start !== selection.end) { diff --git a/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx new file mode 100644 index 000000000..93838440a --- /dev/null +++ b/apps/mobile/src/features/voice-input/ComposerDictationControl.tsx @@ -0,0 +1,419 @@ +import type { VoiceInputPhase, VoiceInputState } from "@t3tools/client-runtime/voice-input"; +import { memo, useCallback, useLayoutEffect, useState, type ReactNode } from "react"; +import { + ActivityIndicator, + Linking, + Platform, + Pressable, + View, + type LayoutChangeEvent, +} from "react-native"; +import Animated, { + Easing, + LinearTransition, + ReduceMotion, + useAnimatedStyle, + useSharedValue, + withTiming, + type EntryExitAnimationFunction, + type SharedValue, +} from "react-native-reanimated"; + +import { AppText as Text } from "../../components/AppText"; +import { SymbolView, type AppSymbolName } from "../../components/AppSymbol"; +import { cn } from "../../lib/cn"; +import type { VoiceComposerPresentation } from "./voiceInputPresentation"; +import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; + +const DICTATION_TIMING = { + duration: 220, + easing: Easing.out(Easing.cubic), + reduceMotion: ReduceMotion.System, +} as const; +const DICTATION_LAYOUT = + Platform.OS === "android" + ? undefined + : LinearTransition.duration(DICTATION_TIMING.duration).reduceMotion(ReduceMotion.System); +const TOOLBAR_FLIP_TIMING = { + duration: 260, + easing: Easing.inOut(Easing.cubic), + reduceMotion: ReduceMotion.System, +} as const; +const TOOLBAR_HALF_HEIGHT = 22; +const TOOLBAR_PERSPECTIVE = 600; + +/** Moves each face around the same horizontal axis, keeping their edges together. */ +function toolbarFlip(fromDegrees: number, toDegrees: number): EntryExitAnimationFunction { + return () => { + "worklet"; + const fromRadians = (fromDegrees * Math.PI) / 180; + const toRadians = (toDegrees * Math.PI) / 180; + const fromSine = Math.sin(fromRadians); + const toSine = Math.sin(toRadians); + return { + initialValues: { + opacity: fromDegrees === 0 ? 1 : 0, + transform: [ + { perspective: TOOLBAR_PERSPECTIVE }, + { translateY: -TOOLBAR_HALF_HEIGHT * fromSine }, + { rotateX: `${fromDegrees}deg` }, + ], + }, + animations: { + opacity: withTiming(toDegrees === 0 ? 1 : 0, TOOLBAR_FLIP_TIMING), + transform: [ + { perspective: TOOLBAR_PERSPECTIVE }, + { + translateY: withTiming(-TOOLBAR_HALF_HEIGHT * toSine, { + ...TOOLBAR_FLIP_TIMING, + easing: (time) => { + const angle = + fromRadians + (toRadians - fromRadians) * TOOLBAR_FLIP_TIMING.easing(time); + return (Math.sin(angle) - fromSine) / (toSine - fromSine); + }, + }), + }, + { rotateX: withTiming(`${toDegrees}deg`, TOOLBAR_FLIP_TIMING) }, + ], + }, + }; + }; +} + +const DRAFT_TOOLBAR_ENTERING = toolbarFlip(90, 0); +const DRAFT_TOOLBAR_EXITING = toolbarFlip(0, 90); +const DICTATION_TOOLBAR_ENTERING = toolbarFlip(-90, 0); +const DICTATION_TOOLBAR_EXITING = toolbarFlip(0, -90); +const WAVEFORM_BAR_HEIGHT = 32; +const WAVEFORM_MIN_BAR_HEIGHT = 2; +const WAVEFORM_BAR_SPACING = 5; +const WAVEFORM_TIMING = { + duration: 100, + easing: Easing.out(Easing.quad), + reduceMotion: ReduceMotion.System, +} as const; + +/** Rotates the compact draft away without unmounting or resizing its native editor. */ +export function ComposerDictationDraftContent(props: { + readonly children: ReactNode; + readonly className?: string; + readonly compact: boolean; + readonly hidden: boolean; +}) { + const rotation = useSharedValue(props.hidden ? 1 : 0); + useLayoutEffect(() => { + rotation.value = withTiming(props.hidden ? 1 : 0, TOOLBAR_FLIP_TIMING); + }, [props.hidden, rotation]); + const compact = props.compact; + const animatedStyle = useAnimatedStyle(() => ({ + opacity: compact ? 1 - rotation.value : 1, + transform: compact + ? [ + { perspective: TOOLBAR_PERSPECTIVE }, + { translateY: -TOOLBAR_HALF_HEIGHT * Math.sin((rotation.value * Math.PI) / 2) }, + { rotateX: `${rotation.value * 90}deg` }, + ] + : [], + })); + + return ( + + {props.children} + + ); +} + +/** Flips the entire row while keeping the outgoing controls intact until it leaves. */ +export function ComposerDictationToolbar(props: { + readonly children: ReactNode; + readonly showsDictation: boolean; + readonly visible?: boolean; +}) { + return ( + + {props.visible !== false ? ( + + {props.children} + + ) : null} + + ); +} + +const WaveformBar = memo(function WaveformBar(props: { + readonly audioLevels: SharedValue; + readonly sampleIndex: number; +}) { + const { audioLevels, sampleIndex } = props; + const animatedStyle = useAnimatedStyle(() => { + const level = audioLevels.value[sampleIndex] ?? 0; + return { + opacity: withTiming(0.22 + level * 0.78, WAVEFORM_TIMING), + transform: [ + { + scaleY: withTiming( + (WAVEFORM_MIN_BAR_HEIGHT + level * (WAVEFORM_BAR_HEIGHT - WAVEFORM_MIN_BAR_HEIGHT)) / + WAVEFORM_BAR_HEIGHT, + WAVEFORM_TIMING, + ), + }, + ], + }; + }); + + return ( + + ); +}); + +const VoiceWaveform = memo(function VoiceWaveform(props: { + readonly audioLevels: SharedValue; +}) { + const [barCount, setBarCount] = useState(0); + const handleLayout = useCallback((event: LayoutChangeEvent) => { + setBarCount( + Math.max( + 1, + Math.min( + VOICE_WAVEFORM_SAMPLE_COUNT, + Math.floor(event.nativeEvent.layout.width / WAVEFORM_BAR_SPACING), + ), + ), + ); + }, []); + + return ( + + {Array.from({ length: barCount }, (_, index) => ( + + ))} + + ); +}); + +function VoiceActionButton(props: { + readonly accessibilityLabel: string; + readonly disabled?: boolean; + readonly icon: AppSymbolName; + readonly loading?: boolean; + readonly onPress: () => void; + readonly variant?: "plain" | "primary"; +}) { + const variant = props.variant ?? "plain"; + const loadingVisibility = useSharedValue(props.loading ? 1 : 0); + useLayoutEffect(() => { + loadingVisibility.value = withTiming(props.loading ? 1 : 0, DICTATION_TIMING); + }, [loadingVisibility, props.loading]); + const primaryStyle = useAnimatedStyle(() => ({ opacity: 1 - loadingVisibility.value })); + + return ( + + + {variant === "primary" ? ( + + ) : null} + + {props.loading ? ( + + ) : ( + + )} + + + + ); +} + +export function ComposerDictationStatus(props: { + readonly audioLevels: SharedValue; + readonly elapsedSeconds: number; + readonly phase: VoiceInputPhase; + readonly presentation: VoiceComposerPresentation; + readonly onDismissError: () => void; +}) { + const recordingVisibility = useSharedValue(props.phase === "recording" ? 1 : 0); + useLayoutEffect(() => { + recordingVisibility.value = withTiming(props.phase === "recording" ? 1 : 0, DICTATION_TIMING); + }, [props.phase, recordingVisibility]); + const waveformStyle = useAnimatedStyle(() => ({ + opacity: recordingVisibility.value, + })); + const labelStyle = useAnimatedStyle(() => ({ + opacity: 1 - recordingVisibility.value, + })); + + if (!props.presentation.statusLabel) return null; + const isError = props.presentation.statusKind === "error"; + const elapsedLabel = `${Math.floor(props.elapsedSeconds / 60)}:${String(props.elapsedSeconds % 60).padStart(2, "0")}`; + return ( + + {isError ? ( + + + {props.presentation.statusLabel} + + + + + + ) : ( + + + + + {elapsedLabel} + + + + + {props.presentation.statusLabel} + + + + )} + + ); +} + +export function ComposerDictationCancelAction(props: { + readonly presentation: VoiceComposerPresentation; + readonly onCancel: () => void; +}) { + if (props.presentation.leadingAction !== "cancel") return null; + return ( + + ); +} + +export function ComposerDictationPrimaryAction(props: { + readonly state: VoiceInputState; + readonly presentation: VoiceComposerPresentation; + readonly isAvailable: boolean; + readonly disabled?: boolean; + readonly onStart: () => void; + readonly onConfirm: () => void; + readonly onCancel: () => void; +}) { + if (props.presentation.trailingAction === "confirm") { + return ( + + ); + } + + return ; +} + +export function ComposerDictationStartAction(props: { + readonly state: VoiceInputState; + readonly isAvailable: boolean; + readonly disabled?: boolean; + readonly onStart: () => void; + readonly onCancel: () => void; +}) { + if (!props.isAvailable) return null; + const openSettings = props.state.phase === "error" && props.state.errorAction === "settings"; + return ( + { + props.onCancel(); + void Linking.openSettings(); + } + : props.onStart + } + /> + ); +} diff --git a/apps/mobile/src/features/voice-input/useVoiceInputController.ts b/apps/mobile/src/features/voice-input/useVoiceInputController.ts new file mode 100644 index 000000000..2170ff255 --- /dev/null +++ b/apps/mobile/src/features/voice-input/useVoiceInputController.ts @@ -0,0 +1,217 @@ +import { + RecordingPresets, + requestRecordingPermissionsAsync, + setAudioModeAsync, + setIsAudioActiveAsync, + useAudioRecorder, + type RecordingStatus, +} from "expo-audio"; +import { File } from "expo-file-system"; +import { useFocusEffect } from "@react-navigation/native"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { AppState } from "react-native"; +import { useSharedValue } from "react-native-reanimated"; + +import type { ComposerEditorSelection } from "../../components/ComposerEditor"; +import { getLocalVoiceTranscriber } from "../../native/voiceTranscription"; +import { + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + voiceInputFreezesEditor, + type VoiceDraftSnapshot, + type VoiceInputState, +} from "@t3tools/client-runtime/voice-input"; +import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering"; + +const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; +const VOICE_METERING_INTERVAL_MS = 80; +const VOICE_RECORDING_OPTIONS = { + ...RecordingPresets.HIGH_QUALITY, + isMeteringEnabled: true, +}; + +async function releaseVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ allowsRecording: false }); + } finally { + // Expo does not deactivate AVAudioSession when recording stops or its + // category changes. Explicit deactivation resumes interrupted app audio. + await setIsAudioActiveAsync(false); + } +} + +async function configureVoiceRecordingAudio(): Promise { + try { + await setAudioModeAsync({ + allowsRecording: true, + interruptionMode: "doNotMix", + playsInSilentMode: true, + shouldPlayInBackground: false, + }); + await setIsAudioActiveAsync(true); + } catch (error) { + try { + await releaseVoiceRecordingAudio(); + } catch { + // Keep the setup error. The controller has not started a recorder yet. + } + throw error; + } +} + +export function useVoiceInputController(input: { + readonly ownerKey: string | null; + readonly draftMessage: string; + readonly selection: ComposerEditorSelection; + readonly disabled?: boolean; + readonly onChangeDraftMessage: (value: string) => void; + readonly onChangeSelection: (selection: ComposerEditorSelection) => void; +}) { + const [state, setState] = useState(INITIAL_STATE); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + const elapsedSecondsRef = useRef(0); + const audioLevelsRef = useRef(Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0)); + const audioLevels = useSharedValue(audioLevelsRef.current); + const controllerRef = useRef(null); + const previousDraftRef = useRef({ ownerKey: input.ownerKey, text: input.draftMessage }); + const revisionRef = useRef(0); + if ( + previousDraftRef.current.ownerKey !== input.ownerKey || + previousDraftRef.current.text !== input.draftMessage + ) { + previousDraftRef.current = { ownerKey: input.ownerKey, text: input.draftMessage }; + revisionRef.current += 1; + } + const latestInputRef = useRef(input); + latestInputRef.current = input; + + const handleRecorderStatus = useCallback((status: RecordingStatus) => { + controllerRef.current?.handleRecorderStatus({ + isFinished: status.isFinished, + hasError: status.hasError || status.mediaServicesDidReset === true, + error: status.error, + url: status.url, + }); + }, []); + const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus); + + if (!controllerRef.current) { + controllerRef.current = new VoiceInputController({ + recorder, + getTranscriber: getLocalVoiceTranscriber, + requestPermission: async () => { + const permission = await requestRecordingPermissionsAsync(); + return { granted: permission.granted, canAskAgain: permission.canAskAgain }; + }, + configureRecording: configureVoiceRecordingAudio, + releaseRecording: releaseVoiceRecordingAudio, + deleteRecording: (uri) => new File(uri).delete(), + readDraft: (): VoiceDraftSnapshot | null => { + const current = latestInputRef.current; + if (!current.ownerKey) return null; + return { + ownerKey: current.ownerKey, + text: current.draftMessage, + selection: current.selection, + revision: revisionRef.current, + }; + }, + commitDraft: (text, selection) => { + const current = latestInputRef.current; + current.onChangeSelection(selection); + current.onChangeDraftMessage(text); + }, + onStateChange: setState, + }); + } + + const controller = controllerRef.current; + const previousOwnerRef = useRef(input.ownerKey); + useEffect(() => { + if (previousOwnerRef.current === input.ownerKey) return; + previousOwnerRef.current = input.ownerKey; + controller.ownerChanged(); + }, [controller, input.ownerKey]); + + useFocusEffect( + useCallback( + () => () => { + controller.dispose(); + }, + [controller], + ), + ); + + useEffect(() => { + const subscription = AppState.addEventListener("change", (nextState) => { + // iOS reports `inactive` while its permission dialog is open. Only the + // real background state cancels preparation; recorder status handles + // calls and route interruptions during capture. + if (nextState === "background") controller.appMovedToBackground(); + }); + return () => subscription.remove(); + }, [controller]); + + useEffect(() => () => controller.dispose(), [controller]); + + useEffect(() => { + if (state.phase !== "preparing" && state.phase !== "recording") return; + + if (audioLevelsRef.current.some((level) => level !== 0)) { + audioLevelsRef.current = Array(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0); + audioLevels.value = audioLevelsRef.current; + } + if (elapsedSecondsRef.current !== 0) { + elapsedSecondsRef.current = 0; + setElapsedSeconds(0); + } + if (state.phase !== "recording") return; + + const sampleRecording = () => { + if (controller.currentState.phase !== "recording") return; + const status = recorder.getStatus(); + if (!status.isRecording) return; + + const level = normalizeVoiceInputDecibels(status.metering); + const history = audioLevelsRef.current; + if (level !== 0 || history.some((sample) => sample !== 0)) { + const nextLevels = [...history.slice(1), level]; + audioLevelsRef.current = nextLevels; + audioLevels.value = nextLevels; + } + + const nextElapsedSeconds = Math.min( + VOICE_RECORDING_LIMIT_SECONDS, + Math.max(0, Math.floor(status.durationMillis / 1_000)), + ); + if (nextElapsedSeconds !== elapsedSecondsRef.current) { + elapsedSecondsRef.current = nextElapsedSeconds; + setElapsedSeconds(nextElapsedSeconds); + } + }; + + sampleRecording(); + const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS); + return () => clearInterval(intervalId); + }, [audioLevels, controller, recorder, state.phase]); + + const start = useCallback(() => { + if (!latestInputRef.current.disabled) void controller.start(); + }, [controller]); + const stop = useCallback(() => controller.stop(), [controller]); + const cancel = useCallback(() => controller.cancel(), [controller]); + + return { + isAvailable: getLocalVoiceTranscriber() !== null, + state, + audioLevels, + elapsedSeconds, + isBusy: voiceInputBlocksSubmission(state), + freezesEditor: voiceInputFreezesEditor(state), + blocksSubmission: voiceInputBlocksSubmission(state), + start, + stop, + cancel, + }; +} diff --git a/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts b/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts new file mode 100644 index 000000000..05356eaee --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputMetering.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { normalizeVoiceInputDecibels } from "./voiceInputMetering"; + +describe("normalizeVoiceInputDecibels", () => { + it.each([undefined, Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY])( + "treats a missing or invalid reading %s as silence", + (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(0); + }, + ); + + it.each([-160, -90, -60])("keeps a reading at or below the noise floor %s silent", (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(0); + }); + + it("keeps quiet background readings close to the baseline", () => { + const quiet = normalizeVoiceInputDecibels(-50); + expect(quiet).toBeGreaterThan(0); + expect(quiet).toBeLessThan(0.05); + }); + + it("keeps loud negative speech readings distinct below full height", () => { + const levels = [-20, -18, -12, -6, -3].map(normalizeVoiceInputDecibels); + + for (const level of levels) { + expect(level).toBeGreaterThan(0); + expect(level).toBeLessThan(1); + } + expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true); + }); + + it("makes near-speech changes visible without an early ceiling", () => { + expect(normalizeVoiceInputDecibels(-6) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan( + 0.18, + ); + expect(normalizeVoiceInputDecibels(-3) - normalizeVoiceInputDecibels(-12)).toBeGreaterThan(0.3); + }); + + it("increases throughout the usable microphone range", () => { + const levels = [-60, -55, -50, -40, -30, -20, -12, -6, -3, -0.001, 0].map( + normalizeVoiceInputDecibels, + ); + expect(levels.every((level, index) => index === 0 || level > levels[index - 1]!)).toBe(true); + }); + + it("approaches the noise floor and full scale without a jump", () => { + expect(normalizeVoiceInputDecibels(-59.999)).toBeLessThan(0.001); + expect(normalizeVoiceInputDecibels(-0.001)).toBeGreaterThan(0.999); + expect(normalizeVoiceInputDecibels(-0.001)).toBeLessThan(1); + }); + + it.each([0, 6, 160])("caps only full-scale or higher readings %s at one", (decibels) => { + expect(normalizeVoiceInputDecibels(decibels)).toBe(1); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputMetering.ts b/apps/mobile/src/features/voice-input/voiceInputMetering.ts new file mode 100644 index 000000000..06f62fc24 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputMetering.ts @@ -0,0 +1,14 @@ +export const VOICE_WAVEFORM_SAMPLE_COUNT = 64; + +const VOICE_NOISE_FLOOR_DECIBELS = -60; +const VOICE_NOISE_FLOOR_AMPLITUDE = 10 ** (VOICE_NOISE_FLOOR_DECIBELS / 20); + +/** Converts measured decibels to compressed amplitude, reserving full height for 0 dB. */ +export function normalizeVoiceInputDecibels(decibels: number | undefined) { + if (decibels === undefined || !Number.isFinite(decibels)) return 0; + if (decibels <= VOICE_NOISE_FLOOR_DECIBELS) return 0; + if (decibels >= 0) return 1; + + const amplitude = 10 ** (decibels / 20); + return Math.sqrt((amplitude - VOICE_NOISE_FLOOR_AMPLITUDE) / (1 - VOICE_NOISE_FLOOR_AMPLITUDE)); +} diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts new file mode 100644 index 000000000..d497a3e14 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vite-plus/test"; +import { voiceInputFreezesEditor } from "@t3tools/client-runtime/voice-input"; + +import { resolveVoiceComposerPresentation } from "./voiceInputPresentation"; + +describe("resolveVoiceComposerPresentation", () => { + it("maps voice states to stable composer actions and editor read-only state", () => { + expect( + resolveVoiceComposerPresentation({ phase: "idle", error: null, errorAction: null }, 0), + ).toEqual({ + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: null, + statusLabel: null, + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation({ phase: "preparing", error: null, errorAction: null }, 0), + ).toMatchObject({ + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusLabel: "Preparing", + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation({ phase: "recording", error: null, errorAction: null }, 64), + ).toMatchObject({ + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusLabel: "Recording 1:04", + confirmationEnabled: true, + }); + expect( + resolveVoiceComposerPresentation( + { phase: "transcribing", error: null, errorAction: null }, + 0, + ), + ).toMatchObject({ + statusLabel: "Transcribing", + confirmationEnabled: false, + }); + expect( + resolveVoiceComposerPresentation( + { phase: "error", error: "Microphone unavailable", errorAction: "retry" }, + 0, + ), + ).toMatchObject({ + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: "error", + statusLabel: "Microphone unavailable", + }); + + expect(voiceInputFreezesEditor({ phase: "preparing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "recording", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "transcribing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputFreezesEditor({ phase: "idle", error: null, errorAction: null })).toBe(false); + }); +}); + +describe("showsSend is not the same predicate as having a status label", () => { + // The composer must gate its send control on showsSend. Gating on the + // presence of a status label instead looks equivalent — both are non-null + // through recording and transcribing — but diverges exactly in the error + // phase, which shows a label AND keeps send. Getting this wrong strands the + // user with no send control until they find the dismiss affordance. + it("keeps send available in the error phase, which also shows a status label", () => { + const presentation = resolveVoiceComposerPresentation( + { phase: "error", error: "No speech was detected.", errorAction: null }, + 0, + ); + expect(presentation.showsSend).toBe(true); + expect(presentation.statusLabel).not.toBeNull(); + }); + + it("hides send only while dictation actually owns the row", () => { + for (const phase of ["preparing", "recording", "transcribing"] as const) { + const presentation = resolveVoiceComposerPresentation( + { phase, error: null, errorAction: null }, + 0, + ); + expect(presentation.showsSend).toBe(false); + expect(presentation.statusLabel).not.toBeNull(); + } + const idle = resolveVoiceComposerPresentation( + { phase: "idle", error: null, errorAction: null }, + 0, + ); + expect(idle.showsSend).toBe(true); + expect(idle.statusLabel).toBeNull(); + }); +}); diff --git a/apps/mobile/src/features/voice-input/voiceInputPresentation.ts b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts new file mode 100644 index 000000000..e461e34d6 --- /dev/null +++ b/apps/mobile/src/features/voice-input/voiceInputPresentation.ts @@ -0,0 +1,65 @@ +import type { VoiceInputState } from "@t3tools/client-runtime/voice-input"; + +export type VoiceComposerPresentation = { + readonly leadingAction: "cancel" | null; + readonly trailingAction: "mic" | "confirm"; + readonly showsSend: boolean; + readonly statusKind: "active" | "error" | null; + readonly statusLabel: string | null; + readonly confirmationEnabled: boolean; +}; + +export function resolveVoiceComposerPresentation( + state: VoiceInputState, + elapsedSeconds: number, +): VoiceComposerPresentation { + switch (state.phase) { + case "idle": + return { + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: null, + statusLabel: null, + confirmationEnabled: false, + }; + case "error": + return { + leadingAction: null, + trailingAction: "mic", + showsSend: true, + statusKind: "error", + statusLabel: state.error, + confirmationEnabled: false, + }; + case "preparing": + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: "Preparing", + confirmationEnabled: false, + }; + case "recording": { + const seconds = Math.max(0, Math.floor(elapsedSeconds)); + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: `Recording ${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`, + confirmationEnabled: true, + }; + } + case "transcribing": + return { + leadingAction: "cancel", + trailingAction: "confirm", + showsSend: false, + statusKind: "active", + statusLabel: "Transcribing", + confirmationEnabled: false, + }; + } +} diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index 3f0270ca5..f52bd9276 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -1,8 +1,10 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import type { ImagePickerAsset } from "expo-image-picker"; const mocks = vi.hoisted(() => ({ documentUri: "file:///documents", pickFile: vi.fn(), + pickMedia: vi.fn(), copy: vi.fn(), delete: vi.fn(), open: vi.fn(), @@ -37,6 +39,14 @@ vi.mock("expo-file-system", () => { return mocks.size(this.uri) ?? null; } + get name(): string { + return this.uri.split("/").at(-1) ?? ""; + } + + get type(): string { + return "video/quicktime"; + } + create(): void {} open(mode: string) { @@ -64,18 +74,23 @@ vi.mock("expo-file-system", () => { }; }); +vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia })); vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" })); import { persistComposerAttachmentFile, pickComposerFiles, + pickComposerImages, + pickComposerMedia, removePersistedComposerAttachmentFile, } from "./composerImages"; +import { isForegroundHandoffActive } from "./foreground-handoff"; -describe("pickComposerFiles", () => { +describe("composer file attachments", () => { beforeEach(() => { mocks.documentUri = "file:///documents"; mocks.pickFile.mockReset(); + mocks.pickMedia.mockReset(); mocks.copy.mockReset(); mocks.delete.mockReset(); mocks.open.mockReset(); @@ -83,6 +98,179 @@ describe("pickComposerFiles", () => { mocks.size.mockImplementation((uri: string) => (uri.startsWith("content:") ? null : 42)); }); + describe("photo library videos", () => { + const image: ImagePickerAsset = { + uri: "file:///picker/photo.png", + type: "image", + fileName: "photo.png", + mimeType: "image/png", + fileSize: 3, + base64: "YWJj", + width: 1, + height: 1, + }; + const video: ImagePickerAsset = { + uri: "file:///picker/clip.mov", + type: "video", + fileName: "clip.mov", + mimeType: "video/quicktime", + fileSize: 20 * 1024 * 1024, + base64: null, + width: 1920, + height: 1080, + }; + + it("retains mixed photos and videos, keeping video bytes in durable file storage", async () => { + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image, video] }); + mocks.size.mockReturnValue(video.fileSize); + + const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 }); + + expect(mocks.pickMedia).toHaveBeenCalledWith( + expect.objectContaining({ + mediaTypes: ["images", "videos"], + shouldDownloadFromNetwork: true, + }), + ); + expect(result).toEqual({ + attachments: [ + expect.objectContaining({ type: "image", dataUrl: "data:image/png;base64,YWJj" }), + { + id: "attachment-id", + type: "file", + name: "clip.mov", + mimeType: "video/quicktime", + sizeBytes: video.fileSize, + fileUri: "file:///documents/t3-composer-attachments/attachment-id-clip.mov", + }, + ], + error: null, + }); + expect(mocks.copy).toHaveBeenCalledWith( + video.uri, + "file:///documents/t3-composer-attachments/attachment-id-clip.mov", + ); + expect(mocks.delete).not.toHaveBeenCalled(); + }); + + it("keeps image-only destinations on the image picker path", async () => { + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image] }); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(mocks.pickMedia).toHaveBeenCalledWith( + expect.objectContaining({ mediaTypes: ["images"] }), + ); + expect(result.images).toEqual([ + expect.objectContaining({ type: "image", name: "photo.png" }), + ]); + expect(result.error).toBeNull(); + expect(mocks.copy).not.toHaveBeenCalled(); + }); + + it("does not persist videos when the destination lacks file support", async () => { + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [video, image] }); + + const result = await pickComposerMedia({ existingCount: 0 }); + + expect(result.attachments).toEqual([expect.objectContaining({ type: "image" })]); + expect(result.error).toBe("Video attachments are unavailable here."); + expect(mocks.copy).not.toHaveBeenCalled(); + }); + + it("uses local video metadata when the picker omits its name, MIME type, or size", async () => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...video, fileName: null, mimeType: undefined, fileSize: undefined }], + }); + + const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 1024 }); + + expect(result.error).toBeNull(); + expect(result.attachments).toEqual([ + expect.objectContaining({ + type: "file", + name: "clip.mov", + mimeType: "video/quicktime", + sizeBytes: 42, + }), + ]); + }); + + it.each([ + { + reason: "picker size exceeds the server limit", + reported: 2 * 1024 * 1024, + stored: 42, + limit: 1024 * 1024, + error: "'clip.mov' exceeds the 1 MB attachment limit.", + }, + { + reason: "actual size exceeds the server limit", + reported: 42, + stored: 2 * 1024 * 1024, + limit: 1024 * 1024, + error: "'clip.mov' exceeds the 1 MB attachment limit.", + }, + { + reason: "stored copy is empty", + reported: 42, + stored: 0, + limit: 1024 * 1024, + error: "'clip.mov' is empty or could not be read.", + }, + { + reason: "server advertises more than the contract limit", + reported: 51 * 1024 * 1024, + stored: 42, + limit: 80 * 1024 * 1024, + error: "'clip.mov' exceeds the 50 MB attachment limit.", + }, + ])( + "rejects a video when $reason while retaining the selected photo", + async ({ reported, stored, limit, error }) => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...video, fileSize: reported }, image], + }); + mocks.size.mockReturnValue(stored); + + const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: limit }); + + expect(result).toEqual({ + attachments: [expect.objectContaining({ type: "image" })], + error, + }); + if (stored === 0) { + expect(mocks.delete).toHaveBeenCalledWith( + "file:///documents/t3-composer-attachments/attachment-id-clip.mov", + ); + } + }, + ); + + it("applies the remaining attachment slots to photos and videos together", async () => { + mocks.pickMedia.mockResolvedValue({ canceled: false, assets: [image, video] }); + + const result = await pickComposerMedia({ existingCount: 7, maxVideoBytes: 50 * 1024 * 1024 }); + + expect(result.attachments).toEqual([expect.objectContaining({ type: "image" })]); + expect(result.error).toBe("You can attach up to 8 attachments per message."); + expect(mocks.pickMedia).toHaveBeenCalledWith(expect.objectContaining({ selectionLimit: 1 })); + expect(mocks.copy).not.toHaveBeenCalled(); + }); + + it("reports a native video retrieval error and ends the foreground handoff", async () => { + mocks.pickMedia.mockRejectedValue(new Error("Could not download video from iCloud.")); + + await expect(pickComposerMedia({ existingCount: 0, maxVideoBytes: 1024 })).resolves.toEqual({ + attachments: [], + error: "Could not download video from iCloud.", + }); + expect(isForegroundHandoffActive()).toBe(false); + }); + }); + it("copies picked files into app-owned storage without loading their contents", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index 4e8908b71..c19193150 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -157,6 +157,40 @@ export async function removePersistedComposerAttachmentFile(uri: string): Promis } } +async function createComposerFileAttachment(input: { + readonly uri: string; + readonly name: string; + readonly mimeType: string; + readonly sizeBytes: number | null; + readonly maxBytes: number; +}): Promise { + if (input.sizeBytes !== null && input.sizeBytes > input.maxBytes) { + throw new Error(fileAttachmentTooLargeMessage(input.name, input.maxBytes)); + } + const { File } = await import("expo-file-system"); + const fileUri = await persistComposerAttachmentFile(input.uri, input.name, input.maxBytes); + try { + const sizeBytes = new File(fileUri).size ?? input.sizeBytes ?? 0; + if (sizeBytes <= 0) { + throw new Error(`'${input.name}' is empty or could not be read.`); + } + if (sizeBytes > input.maxBytes) { + throw new Error(fileAttachmentTooLargeMessage(input.name, input.maxBytes)); + } + return { + id: uuidv4(), + type: "file", + name: input.name, + mimeType: input.mimeType, + sizeBytes, + fileUri, + }; + } catch (error) { + await removePersistedComposerAttachmentFile(fileUri); + throw error; + } +} + export async function pickComposerFiles(input: { readonly existingCount: number; readonly maxBytes?: number; @@ -199,32 +233,16 @@ export async function pickComposerFiles(input: { // contract rejects empty names at send time, so fall back before the name // reaches storage, errors, or the attachment itself. const name = file.name.trim().length > 0 ? file.name : "file"; - const sizeBytes = file.size ?? null; - if (sizeBytes !== null && sizeBytes > maxBytes) { - error = fileAttachmentTooLargeMessage(name, maxBytes); - continue; - } try { - const fileUri = await persistComposerAttachmentFile(file.uri, name, maxBytes); - const storedSizeBytes = new File(fileUri).size ?? sizeBytes ?? 0; - if (storedSizeBytes <= 0) { - await removePersistedComposerAttachmentFile(fileUri); - error = `'${name}' is empty or could not be read.`; - continue; - } - if (storedSizeBytes > maxBytes) { - await removePersistedComposerAttachmentFile(fileUri); - error = fileAttachmentTooLargeMessage(name, maxBytes); - continue; - } - attachments.push({ - id: uuidv4(), - type: "file", - name, - mimeType: file.type || "application/octet-stream", - sizeBytes: storedSizeBytes, - fileUri, - }); + attachments.push( + await createComposerFileAttachment({ + uri: file.uri, + name, + mimeType: file.type || "application/octet-stream", + sizeBytes: file.size ?? null, + maxBytes, + }), + ); } catch (cause) { error = cause instanceof Error ? cause.message : `Could not read '${name}'.`; } @@ -239,7 +257,7 @@ async function loadImagePicker() { try { return await import("expo-image-picker"); } catch (error) { - throw new Error("Image attachments are unavailable right now.", { cause: error }); + throw new Error("The photo library is unavailable right now.", { cause: error }); } } @@ -254,12 +272,27 @@ async function loadClipboard() { export async function pickComposerImages(input: { readonly existingCount: number }): Promise<{ readonly images: ReadonlyArray; readonly error: string | null; +}> { + const result = await pickComposerMedia(input); + return { + images: result.attachments.filter((attachment) => attachment.type === "image"), + error: result.error, + }; +} + +/** Videos use file uploads; omit maxVideoBytes for image-only destinations. */ +export async function pickComposerMedia(input: { + readonly existingCount: number; + readonly maxVideoBytes?: number; +}): Promise<{ + readonly attachments: ReadonlyArray; + readonly error: string | null; }> { const remainingSlots = PROVIDER_SEND_TURN_MAX_ATTACHMENTS - input.existingCount; if (remainingSlots <= 0) { return { - images: [], - error: `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} images per message.`, + attachments: [], + error: `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`, }; } @@ -268,9 +301,8 @@ export async function pickComposerImages(input: { readonly existingCount: number imagePicker = await loadImagePicker(); } catch (error) { return { - images: [], - error: - error instanceof Error ? error.message : "Image attachments are unavailable right now.", + attachments: [], + error: error instanceof Error ? error.message : "The photo library is unavailable right now.", }; } @@ -280,28 +312,61 @@ export async function pickComposerImages(input: { readonly existingCount: number let result: Awaited>; try { result = await imagePicker.launchImageLibraryAsync({ - mediaTypes: ["images"], + mediaTypes: input.maxVideoBytes === undefined ? ["images"] : ["images", "videos"], allowsMultipleSelection: true, selectionLimit: remainingSlots, base64: true, quality: 1, + shouldDownloadFromNetwork: true, }); + } catch (error) { + return { + attachments: [], + error: error instanceof Error ? error.message : "Could not open the photo library.", + }; } finally { endHandoff(); } if (result.canceled) { return { - images: [], + attachments: [], error: null, }; } - const nextImages: DraftComposerImageAttachment[] = []; + const attachments: DraftComposerAttachment[] = []; let error: string | null = null; for (const asset of result.assets) { + if (attachments.length >= remainingSlots) { + error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`; + break; + } const mimeType = asset.mimeType?.toLowerCase(); + if (asset.type === "video" || mimeType?.startsWith("video/")) { + if (input.maxVideoBytes === undefined) { + error = "Video attachments are unavailable here."; + continue; + } + try { + const { File } = await import("expo-file-system"); + const file = new File(asset.uri); + attachments.push( + await createComposerFileAttachment({ + uri: asset.uri, + name: asset.fileName?.trim() || file.name || "video", + mimeType: mimeType || file.type || "application/octet-stream", + sizeBytes: asset.fileSize ?? null, + maxBytes: clampFileAttachmentUploadBytes(input.maxVideoBytes), + }), + ); + } catch (cause) { + error = + cause instanceof Error ? cause.message : `Could not read '${asset.fileName ?? "video"}'.`; + } + continue; + } if (!mimeType?.startsWith("image/")) { error = `Unsupported file type for '${asset.fileName ?? "image"}'.`; continue; @@ -323,7 +388,7 @@ export async function pickComposerImages(input: { readonly existingCount: number continue; } - nextImages.push({ + attachments.push({ id: uuidv4(), type: "image", name: asset.fileName ?? "image", @@ -335,7 +400,7 @@ export async function pickComposerImages(input: { readonly existingCount: number } return { - images: nextImages, + attachments, error, }; } diff --git a/apps/mobile/src/lib/menu-action-colors.test.ts b/apps/mobile/src/lib/menu-action-colors.test.ts new file mode 100644 index 000000000..a6ee03e58 --- /dev/null +++ b/apps/mobile/src/lib/menu-action-colors.test.ts @@ -0,0 +1,67 @@ +import type { MenuAction } from "@react-native-menu/menu"; +import { describe, expect, it } from "vite-plus/test"; + +import { withMenuActionIconColors } from "./menu-action-colors"; + +describe("withMenuActionIconColors", () => { + it.each(["#111111", "#eeeeee"])( + "gives icons a visible color at every menu depth for the %s theme", + (icon) => { + const actions: MenuAction[] = [ + { id: "photos", title: "Photos", image: "photo" }, + { + title: "Thread", + subactions: [ + { + title: "Pinned thread", + image: "pin", + subactions: [{ title: "Move up", image: "arrow.up" }], + }, + ], + }, + ]; + + const result = withMenuActionIconColors(actions, { icon, destructiveIcon: "#ff0000" }); + + expect(result[0]?.imageColor).toBe(icon); + expect(result[1]).not.toHaveProperty("imageColor"); + expect(result[1]?.subactions?.[0]?.imageColor).toBe(icon); + expect(result[1]?.subactions?.[0]?.subactions?.[0]?.imageColor).toBe(icon); + expect(actions[0]).not.toHaveProperty("imageColor"); + expect(actions[1]?.subactions?.[0]).not.toHaveProperty("imageColor"); + }, + ); + + it("uses the destructive color while retaining action state and attributes", () => { + const action: MenuAction = { + id: "delete", + title: "Delete", + image: "trash", + state: "off", + attributes: { destructive: true, disabled: true }, + }; + + expect( + withMenuActionIconColors([action], { + icon: "#111111", + destructiveIcon: "#cc0000", + }), + ).toEqual([{ ...action, imageColor: "#cc0000" }]); + }); + + it.each(["#123456", "transparent", 0])("preserves explicit icon color %s", (imageColor) => { + const action: MenuAction = { + title: "Delete", + image: "trash", + imageColor, + attributes: { destructive: true }, + }; + + expect( + withMenuActionIconColors([action], { + icon: "#111111", + destructiveIcon: "#cc0000", + }), + ).toEqual([action]); + }); +}); diff --git a/apps/mobile/src/lib/menu-action-colors.ts b/apps/mobile/src/lib/menu-action-colors.ts new file mode 100644 index 000000000..611784319 --- /dev/null +++ b/apps/mobile/src/lib/menu-action-colors.ts @@ -0,0 +1,24 @@ +import type { MenuAction } from "@react-native-menu/menu"; + +// MenuView's iOS bridge treats an omitted imageColor as transparent. +export function withMenuActionIconColors( + actions: readonly MenuAction[], + colors: { + readonly icon: MenuAction["imageColor"]; + readonly destructiveIcon: MenuAction["imageColor"]; + }, +): MenuAction[] { + return actions.map((action) => ({ + ...action, + ...(action.image + ? { + imageColor: + action.imageColor ?? + (action.attributes?.destructive ? colors.destructiveIcon : colors.icon), + } + : {}), + ...(action.subactions + ? { subactions: withMenuActionIconColors(action.subactions, colors) } + : {}), + })); +} diff --git a/apps/mobile/src/native/T3ComposerEditor.ios.tsx b/apps/mobile/src/native/T3ComposerEditor.ios.tsx index cb818245d..85decebe9 100644 --- a/apps/mobile/src/native/T3ComposerEditor.ios.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.ios.tsx @@ -62,6 +62,7 @@ interface NativeComposerEditorProps extends ViewProps { readonly lineHeight: number; readonly contentInsetVertical: number; readonly editable: boolean; + readonly readOnly: boolean; readonly scrollEnabled: boolean; readonly autoFocus: boolean; readonly autoCorrect: boolean; @@ -243,6 +244,7 @@ export function ComposerEditor({ } contentInsetVertical={contentInsetVertical} editable={props.editable ?? true} + readOnly={props.readOnly ?? false} scrollEnabled={props.scrollEnabled ?? true} autoFocus={props.autoFocus ?? false} autoCorrect={props.autoCorrect ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.native.tsx b/apps/mobile/src/native/T3ComposerEditor.native.tsx index 5f5201f3b..1a488d34f 100644 --- a/apps/mobile/src/native/T3ComposerEditor.native.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.native.tsx @@ -248,7 +248,7 @@ export function ComposerEditor({ } contentInsetVertical={contentInsetVertical} singleLineCentered={props.singleLineCentered ?? false} - editable={props.editable ?? true} + editable={(props.editable ?? true) && !(props.readOnly ?? false)} scrollEnabled={props.scrollEnabled ?? true} autoFocus={props.autoFocus ?? false} autoCorrect={props.autoCorrect ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.tsx b/apps/mobile/src/native/T3ComposerEditor.tsx index 30bc0ccee..07a409c9a 100644 --- a/apps/mobile/src/native/T3ComposerEditor.tsx +++ b/apps/mobile/src/native/T3ComposerEditor.tsx @@ -16,6 +16,7 @@ export function ComposerEditor({ textStyle, contentInsetVertical = 0, singleLineCentered: _singleLineCentered, + readOnly = false, ...props }: ComposerEditorProps) { const inputRef = useRef(null); @@ -39,6 +40,7 @@ export function ComposerEditor({ props.onSelectionChange?.(event.nativeEvent.selection)} multiline={props.multiline ?? true} diff --git a/apps/mobile/src/native/T3ComposerEditor.types.ts b/apps/mobile/src/native/T3ComposerEditor.types.ts index bfc47ed36..c8833bb4c 100644 --- a/apps/mobile/src/native/T3ComposerEditor.types.ts +++ b/apps/mobile/src/native/T3ComposerEditor.types.ts @@ -23,6 +23,8 @@ export interface ComposerEditorProps { readonly placeholder?: string; readonly autoFocus?: boolean; readonly editable?: boolean; + /** Blocks user edits while preserving focus, selection, and the software keyboard on iOS. */ + readonly readOnly?: boolean; readonly scrollEnabled?: boolean; readonly autoCorrect?: boolean; readonly spellCheck?: boolean; diff --git a/apps/mobile/src/native/voiceTranscription.ios.test.ts b/apps/mobile/src/native/voiceTranscription.ios.test.ts new file mode 100644 index 000000000..b08e32cdb --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ios.test.ts @@ -0,0 +1,140 @@ +import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input"; + +const mocks = vi.hoisted(() => ({ + isAvailable: vi.fn<(locale: string) => boolean>(), + prepare: vi.fn<(locale: string) => Promise>(), + transcribe: vi.fn<(audio: ArrayBufferLike, locale: string) => Promise>(), + readAudio: vi.fn<() => Promise>(), +})); + +vi.mock("@react-native-ai/apple/src/NativeAppleTranscription", () => ({ + default: { + isAvailable: mocks.isAvailable, + prepare: mocks.prepare, + transcribe: mocks.transcribe, + }, +})); + +vi.mock("expo-file-system", () => ({ + File: class { + arrayBuffer = mocks.readAudio; + }, +})); + +import { getLocalVoiceTranscriber } from "./voiceTranscription.ios"; + +const audio = new ArrayBuffer(4); +const nativeTranscript: TranscriptionResult = { + duration: 2, + segments: [ + { text: " Hej", startSecond: 0, endSecond: 1 }, + { text: "världen. ", startSecond: 1, endSecond: 2 }, + ], +}; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((next) => { + resolve = next; + }); + return { promise, resolve }; +} + +beforeEach(() => { + vi.resetAllMocks(); + mocks.isAvailable.mockReturnValue(true); + mocks.prepare.mockResolvedValue("sv-SE"); + mocks.readAudio.mockResolvedValue(audio); + mocks.transcribe.mockResolvedValue(nativeTranscript); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("getLocalVoiceTranscriber", () => { + it("keeps the selected language and Apple's resolved locale when the device language changes", async () => { + const resolvedOptions = Intl.DateTimeFormat().resolvedOptions(); + const deviceLocale = vi + .spyOn(Intl.DateTimeFormat.prototype, "resolvedOptions") + .mockReturnValue({ ...resolvedOptions, locale: "sv-FI" }); + const transcriber = getLocalVoiceTranscriber()!; + const options = { signal: new AbortController().signal }; + + deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "de-DE" }); + const prepared = await transcriber.prepare(options); + deviceLocale.mockReturnValue({ ...resolvedOptions, locale: "en-US" }); + + await expect(prepared.transcribe("file:///voice.m4a", options)).resolves.toBe("Hej världen."); + expect(mocks.prepare).toHaveBeenCalledWith("sv-FI"); + expect(prepared.locale).toBe("sv-SE"); + expect(mocks.transcribe).toHaveBeenCalledWith(audio, "sv-SE"); + }); + + it("does not start native transcription after cancellation during a file read", async () => { + const enteredRead = deferred(); + const readResult = deferred(); + mocks.readAudio.mockImplementation(() => { + enteredRead.resolve(); + return readResult.promise; + }); + const controller = new AbortController(); + const options = { signal: controller.signal }; + const prepared = await getLocalVoiceTranscriber()!.prepare(options); + const result = prepared + .transcribe("file:///voice.m4a", options) + .catch((error: unknown) => error); + + await enteredRead.promise; + controller.abort(); + readResult.resolve(audio); + + const error = await result; + expect(error).toBeInstanceOf(VoiceTranscriptionError); + expect(error).toMatchObject({ code: "cancelled" }); + expect(mocks.transcribe).not.toHaveBeenCalled(); + }); + + it.each(["prepare", "transcribe"] as const)( + "waits for native %s to finish before settling cancellation", + async (phase) => { + const enteredNative = deferred(); + const finishNative = deferred(); + if (phase === "prepare") { + mocks.prepare.mockImplementation(async () => { + enteredNative.resolve(); + await finishNative.promise; + return "sv-SE"; + }); + } else { + mocks.transcribe.mockImplementation(async () => { + enteredNative.resolve(); + await finishNative.promise; + return nativeTranscript; + }); + } + const controller = new AbortController(); + const options = { signal: controller.signal }; + const transcriber = getLocalVoiceTranscriber()!; + const operation = + phase === "prepare" + ? transcriber.prepare(options) + : (await transcriber.prepare(options)).transcribe("file:///voice.m4a", options); + const settled = vi.fn((value: unknown) => value); + const result = operation.then(settled, settled); + + await enteredNative.promise; + controller.abort(); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).not.toHaveBeenCalled(); + finishNative.resolve(); + + const error = await result; + expect(error).toBeInstanceOf(VoiceTranscriptionError); + expect(error).toMatchObject({ code: "cancelled" }); + }, + ); +}); diff --git a/apps/mobile/src/native/voiceTranscription.ios.ts b/apps/mobile/src/native/voiceTranscription.ios.ts new file mode 100644 index 000000000..216b9e958 --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ios.ts @@ -0,0 +1,98 @@ +import AppleTranscription from "@react-native-ai/apple/src/NativeAppleTranscription"; +import { File } from "expo-file-system"; + +import { + VoiceTranscriptionError, + throwIfVoiceTranscriptionAborted, + type PreparedVoiceTranscription, + type VoiceTranscriber, + type VoiceTranscriptionOptions, +} from "@t3tools/client-runtime/voice-input"; + +function getDeviceLocale(): string { + return Intl.DateTimeFormat().resolvedOptions().locale; +} + +function wrapError( + code: "preparation-failed" | "transcription-failed", + message: string, + cause: unknown, +): VoiceTranscriptionError { + if (cause instanceof VoiceTranscriptionError) { + return cause; + } + + return new VoiceTranscriptionError(code, message, { cause }); +} + +function getNativeErrorCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) { + return undefined; + } + + return typeof error.code === "string" ? error.code : undefined; +} + +export function getLocalVoiceTranscriber(): VoiceTranscriber | null { + const locale = getDeviceLocale(); + if (!AppleTranscription.isAvailable(locale)) return null; + return { prepare: (options) => prepareVoiceTranscription(locale, options) }; +} + +async function prepareVoiceTranscription( + locale: string, + { signal }: VoiceTranscriptionOptions, +): Promise { + throwIfVoiceTranscriptionAborted(signal); + if (!AppleTranscription.isAvailable(locale)) { + throw new VoiceTranscriptionError( + "unavailable", + "Voice transcription requires a supported device with iOS 26 or later.", + ); + } + + try { + const supportedLocale = await AppleTranscription.prepare(locale); + throwIfVoiceTranscriptionAborted(signal); + return { + locale: supportedLocale, + transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options), + }; + } catch (error) { + throwIfVoiceTranscriptionAborted(signal); + if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") { + throw new VoiceTranscriptionError( + "unsupported-locale", + "Voice transcription does not support this device language.", + { cause: error }, + ); + } + + throw wrapError( + "preparation-failed", + "Voice transcription could not prepare this language.", + error, + ); + } +} + +async function transcribeVoiceRecording( + uri: string, + locale: string, + { signal }: VoiceTranscriptionOptions, +): Promise { + try { + throwIfVoiceTranscriptionAborted(signal); + const audio = await new File(uri).arrayBuffer(); + throwIfVoiceTranscriptionAborted(signal); + const result = await AppleTranscription.transcribe(audio, locale); + throwIfVoiceTranscriptionAborted(signal); + return result.segments + .map((segment) => segment.text) + .join(" ") + .trim(); + } catch (error) { + throwIfVoiceTranscriptionAborted(signal); + throw wrapError("transcription-failed", "Voice transcription failed.", error); + } +} diff --git a/apps/mobile/src/native/voiceTranscription.ts b/apps/mobile/src/native/voiceTranscription.ts new file mode 100644 index 000000000..e003064ae --- /dev/null +++ b/apps/mobile/src/native/voiceTranscription.ts @@ -0,0 +1,5 @@ +import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input"; + +export function getLocalVoiceTranscriber(): VoiceTranscriber | null { + return null; +} diff --git a/apps/mobile/src/state/use-composer-drafts.test.ts b/apps/mobile/src/state/use-composer-drafts.test.ts index c2f69d887..3ab0baa39 100644 --- a/apps/mobile/src/state/use-composer-drafts.test.ts +++ b/apps/mobile/src/state/use-composer-drafts.test.ts @@ -221,40 +221,49 @@ describe("mobile composer drafts", () => { }); }); - it("caps appended attachments at the send limit against the live draft", () => { + it("releases videos rejected by the live draft limit and keeps accepted files", async () => { + const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => outboxLoad.mockRestore()); + const cleanup = Promise.withResolvers(); + composerAttachmentCleanupMocks.remove.mockImplementationOnce(async () => { + cleanup.resolve(); + }); const makeAttachment = (id: string) => ({ id, type: "file" as const, - name: `${id}.pdf`, - mimeType: "application/pdf", + name: `${id}.mov`, + mimeType: "video/quicktime", sizeBytes: 42, - fileUri: `file:///documents/t3-composer-attachments/${id}.pdf`, + fileUri: `file:///documents/t3-composer-attachments/${id}.mov`, }); + const draftKey = "new-task:environment-1:project-cap"; const existing = Array.from({ length: 7 }, (_, index) => makeAttachment(`held-${index}`)); appAtomRegistry.set(composerDraftsAtom, { - "environment-1:thread-cap": { text: "send this", attachments: existing }, + [draftKey]: { text: "send this", attachments: existing }, }); - const rejected = appendComposerDraftAttachments("environment-1:thread-cap", [ + const rejected = appendComposerDraftAttachments(draftKey, [ makeAttachment("incoming-1"), makeAttachment("incoming-2"), ]); expect(rejected).toBe(1); - const draft = appAtomRegistry.get(composerDraftsAtom)["environment-1:thread-cap"]; + const draft = appAtomRegistry.get(composerDraftsAtom)[draftKey]; expect(draft?.attachments).toHaveLength(8); expect(draft?.attachments.at(-1)?.id).toBe("incoming-1"); + await cleanup.promise; + expect(composerAttachmentCleanupMocks.remove).toHaveBeenCalledExactlyOnceWith( + makeAttachment("incoming-2").fileUri, + ); // Restore paths bypass the cap so a failed send never drops its files. const overflowRejected = appendComposerDraftAttachments( - "environment-1:thread-cap", + draftKey, [makeAttachment("restored-1")], { allowOverflow: true }, ); expect(overflowRejected).toBe(0); - expect( - appAtomRegistry.get(composerDraftsAtom)["environment-1:thread-cap"]?.attachments, - ).toHaveLength(9); + expect(appAtomRegistry.get(composerDraftsAtom)[draftKey]?.attachments).toHaveLength(9); }); it("keeps shared attachment files until every draft releases them", async () => { diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 9dfc147d6..fc81dc7b3 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -60,7 +60,7 @@ import { convertPastedImagesToAttachments, pasteComposerClipboard, pickComposerFiles, - pickComposerImages, + pickComposerMedia, } from "../lib/composerImages"; import type { DraftComposerImageAttachment } from "../lib/composerImages"; import { prepareTurnAttachments, validateDraftFileAttachments } from "../lib/attachmentUpload"; @@ -995,26 +995,31 @@ export function useThreadComposerState() { [selectedThreadShell], ); - const onPickDraftImages = useCallback(async () => { + const onPickDraftMedia = useCallback(async () => { if (!selectedThreadShell) { return; } const threadKey = scopedThreadKey(selectedThreadShell.environmentId, selectedThreadShell.id); - const result = await pickComposerImages({ + const capabilities = selectedEnvironmentRuntime?.serverConfig?.environment.capabilities; + const result = await pickComposerMedia({ existingCount: composerDrafts[threadKey]?.attachments.length ?? 0, + maxVideoBytes: + capabilities?.attachmentUploads === true + ? capabilities.fileAttachments?.maxUploadBytes + : undefined, }); - const rejectedImageCount = appendComposerDraftAttachments(threadKey, result.images); + const rejectedCount = appendComposerDraftAttachments(threadKey, result.attachments); const problems = [ ...(result.error ? [result.error] : []), - ...(rejectedImageCount > 0 + ...(rejectedCount > 0 ? [`You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`] : []), ]; if (problems.length > 0) { - Alert.alert("Could not attach image", problems.join("\n\n")); + Alert.alert("Could not attach photo or video", problems.join("\n\n")); } - }, [composerDrafts, selectedThreadShell]); + }, [composerDrafts, selectedEnvironmentRuntime?.serverConfig, selectedThreadShell]); const onPickDraftFiles = useCallback(async () => { if (!selectedThreadShell) { @@ -1188,7 +1193,7 @@ export function useThreadComposerState() { interactionMode, activeThreadBusy, onChangeDraftMessage, - onPickDraftImages, + onPickDraftMedia, onPickDraftFiles, onPasteIntoDraft, onNativePasteImages, diff --git a/docs/README.md b/docs/README.md index a16a7ce61..c3ec64036 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ policy in [CONTRIBUTING.md](../CONTRIBUTING.md); agent rules in [AGENTS.md](../A - [Glossary](./internals/glossary.md) - [Scripts](./internals/scripts.md) - [Connection runtime](./internals/connection-runtime.md) +- [Voice input](./internals/voice-input.md) - [Providers](./internals/providers.md) - [Remote environments](./internals/remote.md) - [Server updates](./internals/server-updates.md) diff --git a/docs/internals/mobile-navigation.md b/docs/internals/mobile-navigation.md index b559f39bc..86c61921f 100644 --- a/docs/internals/mobile-navigation.md +++ b/docs/internals/mobile-navigation.md @@ -41,3 +41,10 @@ After changing a dependency patch, refresh CocoaPods before rebuilding an existing iOS project. pnpm installs each patch hash in a different directory; an old Pods project can keep compiling the previous directory even though Metro and `apps/mobile/node_modules` resolve to the new patch. + +`ControlPillMenu` resolves semantic icon colors through `withUniwind` and supplies them to every +iOS `MenuView` action, including nested actions. The menu library's Fabric bridge converts a missing +`imageColor` to transparent, so callers should use this wrapper instead of +rendering `MenuView` directly. Explicit colors are preserved, and destructive +actions default to the theme's danger foreground color. Native stack header menus +use a separate implementation and do not need this workaround. diff --git a/docs/internals/voice-input.md b/docs/internals/voice-input.md new file mode 100644 index 000000000..66f736bff --- /dev/null +++ b/docs/internals/voice-input.md @@ -0,0 +1,102 @@ +# Voice input + +> For maintainers. Using Pylon? See [voice input on iPhone](../user/composer.md#voice-input-on-iphone). + +Voice input produces editable composer text. The current implementation records on the client and +transcribes locally with Apple's `SpeechAnalyzer` and `SpeechTranscriber` on supported iOS 26+ +devices. Environment-provided transcription and transcription on web and desktop are not implemented. + +## Current boundaries + +The shared [`VoiceInputController`][controller] in `packages/client-runtime` owns preparation, +recording, transcription, cancellation, temporary-file cleanup, and insertion into the captured +draft selection. Applications import it through the [voice-input entry point][voice-input] as +`@t3tools/client-runtime/voice-input`. Its dependencies separate capture from transcription; the +controller imports neither React Native nor an Apple transcription API. + +The shared [transcription contract][transcription] defines `VoiceTranscriber`, +`PreparedVoiceTranscription`, and transcription errors. The controller calls `getTranscriber()` once +at the start of an operation, before asking for microphone permission. Preparation returns a resolved +locale and a bound `transcribe` function. The controller retains that result for the recording, so a +selection change cannot prepare with one implementation and transcribe with another. + +[`useVoiceInputController`][hook] supplies Expo audio capture, microphone permissions, audio-session +management, waveform samples, and app and navigation lifecycle handling. It normalizes Expo's +`mediaServicesDidReset` into a generic recorder error. [`voiceTranscription.ios.ts`][ios] adapts +`@react-native-ai/apple` through `getLocalVoiceTranscriber()`, capturing the requested device locale +and binding the prepared transcriber to Apple's resolved locale. The other-platform binding returns +no local transcriber. That result describes the local implementation, not whether a client could use +an environment's transcription service. + +Mobile's [`voiceInputPresentation.ts`][presentation] maps shared state to toolbar labels and actions. +Waveform and toolbar rendering stay in mobile. The composer edits draft text without selecting a +speech vendor. +Recording captures the draft owner, revision, text, and selection. A late transcript cannot overwrite +a different or edited draft. Only normal message submission sends the resulting text to an agent. + +Each operation passes one `AbortSignal` through preparation and transcription. Cancellation +invalidates the operation and aborts that signal immediately. Implementations settle their promises +only after their underlying work stops. The Apple binding checks cancellation between asynchronous +steps but cannot interrupt an in-flight native request. The controller retains its session until +that work settles, ignores its result, and cleans up the recording. + +## Ownership decisions + +The extension boundary distinguishes transcription on the client device from transcription through +the composer's environment. These constraints apply when adding selectable transcription services: + +- Local means the client device, regardless of which machine hosts the environment. A device's lack + of local recognition does not prevent it from recording audio for an environment service. +- Remote service configuration and API keys belong to the environment. The environment calls the + external service. Clients receive service identifiers, labels, and availability information, never + credential values. Transcription services are independent of coding-agent `providerInstances`; + selecting OpenAI for transcription does not select Codex for the thread. +- The client owns its transcription preference, scoped by stable `environmentId`. Its choices are + supported local recognition and the services exposed by the composer's environment. A service ID + is meaningful only within that environment. Different clients can make different choices. +- Resolve and capture the environment, selected service, and locale when an operation starts. + Preparation and transcription use the same selection; preference changes affect the next + recording. Capture environment identity explicitly rather than recovering it from a draft key. + Keep the existing draft-owner and revision checks before inserting text. +- If the selected option is unavailable, report that state and let the user choose another option. + A local failure must not silently upload audio, and a disconnected environment must not redirect + a recording to another environment or service. +- Transcription audio is temporary input, separate from durable chat attachments and messages. + Remote adapters need cancellation of upload and transcription where supported, cleanup after + success, failure, or cancellation, and the same protection against late results as local transcription. + +## Existing integration points + +[`ServerSettingsService`][settings] and [`ServerSecretStore`][secrets] provide environment-owned +configuration and secret persistence. Existing settings redaction handles coding-provider environment +variables only. Any transcription credential fields need their own explicit separation and redaction +before settings responses or subscriptions reach client caches. + +[`ExecutionEnvironmentCapabilities`][capabilities] handles version skew. Remote transcription must be +opt-in: a missing transcription capability means unsupported. The authenticated server-config +subscription and [shared environment state][server-state] already distribute configuration per +environment. A transcription service catalog belongs behind that capability and authenticated +boundary. Older servers expose no remote transcription choices. + +The [attachment upload contracts][uploads] and [shared upload operations][attachment-state] provide a +pattern for authorized binary uploads through an environment, including remote connections. Their +existing chat-attachment retention is not a transcription cleanup policy. + +Future service selection and environment requests belong alongside the controller in +`packages/client-runtime`, with wire contracts in `packages/contracts`. Capture and native local +recognition remain client-specific. An environment-backed transcriber implements the same shared +contract, with its environment and service bound when selected. The controller does not own service +credentials, provider SDKs, or transport selection. + +[controller]: ../../packages/client-runtime/src/voice-input/controller.ts +[voice-input]: ../../packages/client-runtime/src/voice-input/index.ts +[transcription]: ../../packages/client-runtime/src/voice-input/transcription.ts +[hook]: ../../apps/mobile/src/features/voice-input/useVoiceInputController.ts +[presentation]: ../../apps/mobile/src/features/voice-input/voiceInputPresentation.ts +[ios]: ../../apps/mobile/src/native/voiceTranscription.ios.ts +[settings]: ../../apps/server/src/serverSettings.ts +[secrets]: ../../apps/server/src/auth/ServerSecretStore.ts +[capabilities]: ../../packages/contracts/src/environment.ts +[server-state]: ../../packages/client-runtime/src/state/server.ts +[uploads]: ../../packages/contracts/src/assets.ts +[attachment-state]: ../../packages/client-runtime/src/state/attachments.ts diff --git a/docs/user/composer.md b/docs/user/composer.md index 00c72c658..6d5236f46 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -4,16 +4,22 @@ Messages can contain up to 120,000 characters. If a draft is longer, Pylon keeps composer and shows how many characters need to be removed. Shorten the draft or split it into multiple messages, then send again in the same thread. -You can attach images up to 10 MB. On servers that support file uploads, you can also attach text -files, PDFs, ZIP archives, and other files. Each file can be up to the limit advertised by the -server, capped at 50 MB. Each message can carry up to eight attachments in total. Files upload -directly to the environment, where your agent can read, copy, or edit them by their file path. +On mobile, an empty composer shows an interrupt button while the agent is working. Adding text +or an attachment replaces it with the send button. This applies to both compact and expanded +composers. + +You can attach images up to 10 MB. On servers that support file uploads, you can also attach +videos, text files, PDFs, ZIP archives, and other files. Each file can be up to the limit advertised +by the server, capped at 50 MB. Each message can carry up to eight attachments in total. Files +upload directly to the environment, where your agent can read, copy, or edit them by their file path. In the web and desktop apps, attachments upload as soon as you add them. The send button becomes available after every upload finishes. Failed uploads can be retried or removed. In the mobile app, -the **+** control offers Photos, and adds Files when the connected server supports file uploads. You -can share a file into Pylon from any app through the system share sheet. Mobile uploads happen when -the message sends, so queued messages keep their files until they deliver. Select a received file on +tap **+** to open the photo library from either the compact or expanded composer. When the connected +server supports file uploads, **+** opens a menu beside the button with **Photo Library** and +**Choose Files**. Videos use the server's file upload limit. You can also share photos, videos, and +files into Pylon from other apps through the system share sheet. Mobile uploads happen when the +message sends, so queued messages keep their files until they deliver. Select a received file on mobile to save it or open it in another app through the system share sheet. On web and desktop, select a video attachment before or after sending to play it with the browser's @@ -49,6 +55,23 @@ uploaded. Stashed files stay uploaded on the server for 24 hours. If you restore that, the file comes back with **Attach again** next to it. Attach the file again or remove it, then send. +## Voice input on iPhone + +On supported iPhones with iOS 26 or later, tap the microphone in the composer to record a message. +An expanded composer keeps your draft visible and flips its bottom toolbar into recording controls +with waves that respond to your voice. A collapsed composer flips into a compact recording strip +without changing height. Tap the checkmark to finish and transcribe on your device. The waves fade +into a transcription status, then the usual +controls return with the text inserted at the selection where recording started. If the keyboard +is open when you start, it stays open during voice input. You can review and edit the text before +you send it. + +The first use can download Apple's speech model and needs a network connection. Later transcription +works offline for that language. A recording can be up to five minutes long. Canceling voice input, +leaving the screen, or an audio interruption discards the new recording and keeps the existing draft +and attachments. Pylon deletes the local audio file after transcription or cancellation. It sends +only the normal message text when you submit the draft. + ## Commands and skills Type `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source, diff --git a/packages/client-runtime/README.md b/packages/client-runtime/README.md index 722d6f6d3..f2864a2db 100644 --- a/packages/client-runtime/README.md +++ b/packages/client-runtime/README.md @@ -5,18 +5,19 @@ subpath. The package intentionally has no root export. ## Public subpaths -| Subpath | Responsibility | -| --------------------- | ---------------------------------------------------------------- | -| `authorization` | Bearer and DPoP authorization plus token persistence contracts | -| `connection` | Targets, catalog, supervision, retries, registry, and onboarding | -| `environment` | Environment identity, descriptors, endpoints, and scoped keys | -| `errors` | Shared client error inspection | -| `operations` | Multi-step application workflows | -| `operations/projects` | Multi-step project creation workflows | -| `platform` | Platform capability and persistence service contracts | -| `relay` | Managed relay API and environment discovery | -| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions | -| `state/` | Focused shared state, retention, reducers, and Atom constructors | +| Subpath | Responsibility | +| --------------------- | ----------------------------------------------------------------- | +| `authorization` | Bearer and DPoP authorization plus token persistence contracts | +| `connection` | Targets, catalog, supervision, retries, registry, and onboarding | +| `environment` | Environment identity, descriptors, endpoints, and scoped keys | +| `errors` | Shared client error inspection | +| `operations` | Multi-step application workflows | +| `operations/projects` | Multi-step project creation workflows | +| `platform` | Platform capability and persistence service contracts | +| `relay` | Managed relay API and environment discovery | +| `rpc` | HTTP/RPC clients, protocol, sessions, and subscriptions | +| `state/` | Focused shared state, retention, reducers, and Atom constructors | +| `voice-input` | Recording lifecycle, transcription contracts, and draft insertion | ## Dependency direction @@ -25,6 +26,11 @@ capabilities with `authorization`, `relay`, and `rpc` to supervise environment sessions. Independent `state` modules consume the connection registry and expose focused state or Atom constructors to application-owned runtimes. +The `voice-input` controller accepts capture callbacks and a selected `VoiceTranscriber`. +Preparation binds transcription to its implementation and resolved locale; one cancellation +signal covers both operations. Applications provide recorder events, permissions, native +transcription implementations, and presentation. + Applications should import the narrowest relevant subpath. There is no broad `state` export: use domain paths such as `state/shell`, `state/threads`, `state/terminal`, or `state/vcs`. Subpath indices and explicitly exported domain diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 0171dcd4d..cff49256e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -55,6 +55,10 @@ "types": "./src/providerSkills.ts", "default": "./src/providerSkills.ts" }, + "./voice-input": { + "types": "./src/voice-input/index.ts", + "default": "./src/voice-input/index.ts" + }, "./relay": { "types": "./src/relay/index.ts", "default": "./src/relay/index.ts" diff --git a/packages/client-runtime/src/voice-input/controller.test.ts b/packages/client-runtime/src/voice-input/controller.test.ts new file mode 100644 index 000000000..5f26b882b --- /dev/null +++ b/packages/client-runtime/src/voice-input/controller.test.ts @@ -0,0 +1,535 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + resetVoiceInputGlobalsForTests, + resolveTranscriptCommit, + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + voiceInputBlocksSubmission, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, + type VoiceRecorder, +} from "./controller.ts"; +import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +class TestRecorder implements VoiceRecorder { + uri: string | null = "file:///voice.m4a"; + readonly prepareToRecordAsync = vi.fn(async () => undefined); + readonly record = vi.fn(); + readonly stop = vi.fn(async () => undefined); +} + +function preparedTranscription( + transcribe: PreparedVoiceTranscription["transcribe"] = async () => "new text", +): PreparedVoiceTranscription { + return { locale: "en-US", transcribe }; +} + +function draft(overrides: Partial = {}): VoiceDraftSnapshot { + return { + ownerKey: "environment:thread", + text: "hello world", + selection: { start: 6, end: 11 }, + revision: 1, + ...overrides, + }; +} + +function createHarness( + overrides: Partial = {}, + initialDraft = draft(), +) { + const recorder = new TestRecorder(); + let currentDraft: VoiceDraftSnapshot | null = initialDraft; + const commits: Array<{ text: string; selection: { start: number; end: number } }> = []; + const deleted: string[] = []; + const dependencies: VoiceInputControllerDependencies = { + recorder, + getTranscriber: () => ({ prepare: async () => preparedTranscription() }), + requestPermission: async () => ({ granted: true, canAskAgain: true }), + configureRecording: async () => undefined, + releaseRecording: async () => undefined, + deleteRecording: (uri) => deleted.push(uri), + readDraft: () => currentDraft, + commitDraft: (text, selection) => commits.push({ text, selection }), + onStateChange: vi.fn(), + ...overrides, + }; + return { + controller: new VoiceInputController(dependencies), + recorder, + commits, + deleted, + setDraft: (next: VoiceDraftSnapshot | null) => { + currentDraft = next; + }, + }; +} + +describe("resolveTranscriptCommit", () => { + it("replaces the recorded UTF-16 selection around emoji and composer tokens", () => { + const text = "Fix 🧪 then $review please"; + const tokenStart = text.indexOf("$review"); + const captured = draft({ + text, + selection: { start: tokenStart, end: tokenStart + "$review".length }, + }); + + expect(resolveTranscriptCommit(captured, captured, "use the mobile skill", "en-US")).toEqual({ + kind: "commit", + text: "Fix 🧪 then use the mobile skill please", + selection: { start: tokenStart + "use the mobile skill".length, end: tokenStart + 20 }, + }); + }); + + it("does not replace text after the owner, text, or revision changes", () => { + const captured = draft(); + expect( + resolveTranscriptCommit(captured, draft({ ownerKey: "other" }), "text", "en-US"), + ).toEqual({ + kind: "stale", + }); + expect(resolveTranscriptCommit(captured, draft({ text: "newer" }), "text", "en-US")).toEqual({ + kind: "stale", + }); + expect(resolveTranscriptCommit(captured, draft({ revision: 2 }), "text", "en-US")).toEqual({ + kind: "stale", + }); + }); + + it("adds English spacing at empty start, middle, and end caret boundaries", () => { + const atEnd = draft({ + text: "Fix cache.", + selection: { start: "Fix cache.".length, end: "Fix cache.".length }, + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en-US")).toMatchObject({ + kind: "commit", + text: "Fix cache. Also fix tests.", + }); + expect(resolveTranscriptCommit(atEnd, atEnd, "Also fix tests.", "en_US")).toMatchObject({ + kind: "commit", + text: "Fix cache. Also fix tests.", + }); + + const atStart = draft({ text: "Fix cache.", selection: { start: 0, end: 0 } }); + expect(resolveTranscriptCommit(atStart, atStart, "First", "en-US")).toMatchObject({ + kind: "commit", + text: "First Fix cache.", + }); + + const inMiddle = draft({ text: "Fix cache.", selection: { start: 4, end: 4 } }); + expect(resolveTranscriptCommit(inMiddle, inMiddle, "also", "en-US")).toMatchObject({ + kind: "commit", + text: "Fix also cache.", + }); + }); + + it("does not add English boundary spaces to CJK or selected inline text", () => { + const cjk = draft({ text: "修正キャッシュ", selection: { start: 8, end: 8 } }); + expect(resolveTranscriptCommit(cjk, cjk, "テストも", "ja-JP")).toMatchObject({ + kind: "commit", + text: "修正キャッシュテストも", + }); + + const selected = draft({ text: "one $skill two", selection: { start: 4, end: 10 } }); + expect(resolveTranscriptCommit(selected, selected, "new", "en-US")).toMatchObject({ + kind: "commit", + text: "one new two", + }); + }); +}); + +describe("VoiceInputController", () => { + beforeEach(() => resetVoiceInputGlobalsForTests()); + + it("checks support and permission before recording", async () => { + const unsupported = createHarness({ getTranscriber: () => null }); + await unsupported.controller.start(); + expect(unsupported.controller.currentState.error).toContain("not available"); + expect(unsupported.recorder.record).not.toHaveBeenCalled(); + + const denied = createHarness({ + requestPermission: async () => ({ granted: false, canAskAgain: false }), + }); + await denied.controller.start(); + expect(denied.controller.currentState.errorAction).toBe("settings"); + expect(denied.recorder.record).not.toHaveBeenCalled(); + }); + + it.each(["permission", "transcription"] as const)( + "clears %s errors when switching to another draft", + async (failure) => { + const harness = createHarness( + failure === "permission" + ? { requestPermission: async () => ({ granted: false, canAskAgain: false }) } + : { + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(async () => { + throw new Error("Transcription failed"); + }), + }), + }, + ); + await harness.controller.start(); + await harness.controller.stop(); + expect(harness.controller.currentState).toMatchObject({ + phase: "error", + error: expect.any(String), + errorAction: failure === "permission" ? "settings" : "retry", + }); + + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + harness.controller.ownerChanged(); + + expect(harness.controller.currentState).toEqual({ + phase: "idle", + error: null, + errorAction: null, + }); + expect(harness.commits).toEqual([]); + }, + ); + + it.each(["permission", "preparation", "recording"] as const)( + "keeps the selected transcriber when preferences change during %s", + async (changeDuring) => { + const permission = deferred<{ granted: boolean; canAskAgain: boolean }>(); + const permissionEntered = deferred(); + const preparation = deferred(); + const preparationEntered = deferred(); + const preparationSignals: AbortSignal[] = []; + const transcriptionSignals: AbortSignal[] = []; + const transcriber = (text: string): VoiceTranscriber => ({ + prepare: async ({ signal }) => { + preparationSignals.push(signal); + preparationEntered.resolve(undefined); + await preparation.promise; + return preparedTranscription(async (_uri, { signal }) => { + transcriptionSignals.push(signal); + return text; + }); + }, + }); + const first = transcriber("first choice"); + const second = transcriber("second choice"); + let selected = first; + const harness = createHarness({ + getTranscriber: () => selected, + requestPermission: () => { + permissionEntered.resolve(undefined); + return permission.promise; + }, + }); + + const starting = harness.controller.start(); + await permissionEntered.promise; + if (changeDuring === "permission") selected = second; + permission.resolve({ granted: true, canAskAgain: true }); + await preparationEntered.promise; + if (changeDuring === "preparation") selected = second; + preparation.resolve(undefined); + await starting; + if (changeDuring === "recording") selected = second; + await harness.controller.stop(); + + expect(harness.commits.map((commit) => commit.text)).toEqual(["hello first choice"]); + + await harness.controller.start(); + await harness.controller.stop(); + + expect(harness.commits.map((commit) => commit.text)).toEqual([ + "hello first choice", + "hello second choice", + ]); + expect(preparationSignals).toHaveLength(2); + expect(transcriptionSignals).toHaveLength(2); + expect(transcriptionSignals[0]).toBe(preparationSignals[0]); + expect(transcriptionSignals[1]).toBe(preparationSignals[1]); + expect(preparationSignals[1]).not.toBe(preparationSignals[0]); + }, + ); + + it("blocks submit while voice input can still change the draft", () => { + expect(voiceInputBlocksSubmission({ phase: "preparing", error: null, errorAction: null })).toBe( + true, + ); + expect(voiceInputBlocksSubmission({ phase: "recording", error: null, errorAction: null })).toBe( + true, + ); + expect( + voiceInputBlocksSubmission({ phase: "transcribing", error: null, errorAction: null }), + ).toBe(true); + expect(voiceInputBlocksSubmission({ phase: "idle", error: null, errorAction: null })).toBe( + false, + ); + }); + + it("uses the native five-minute cap and commits one final transcript", async () => { + const harness = createHarness(); + await harness.controller.start(); + expect(harness.recorder.record).toHaveBeenCalledWith({ + forDuration: VOICE_RECORDING_LIMIT_SECONDS, + }); + + const stopping = harness.controller.stop(); + harness.controller.handleRecorderStatus({ + isFinished: true, + hasError: false, + error: null, + url: "file:///voice.m4a", + }); + await stopping; + + expect(harness.commits).toEqual([ + { text: "hello new text", selection: { start: 14, end: 14 } }, + ]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + }); + + it.each(["cancel", "dispose", "ownerChanged"] as const)( + "holds the session after %s until non-abortable transcription settles", + async (action) => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription((_uri, { signal }) => { + transcriptionEntered.resolve(signal); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + const signal = await transcriptionEntered.promise; + expect(signal.aborted).toBe(false); + if (action === "ownerChanged") { + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + } + harness.controller[action](); + expect(signal.aborted).toBe(true); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.error).toContain("already active"); + expect(next.recorder.record).not.toHaveBeenCalled(); + + transcription.resolve("late text"); + await stopping; + + expect(harness.commits).toEqual([]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.controller.currentState.phase).toBe("idle"); + + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }, + ); + + it("cancels an in-flight transcriber that rejects when its signal aborts", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription((_uri, { signal }) => { + signal.addEventListener("abort", () => transcription.reject(new Error("aborted")), { + once: true, + }); + transcriptionEntered.resolve(signal); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + const signal = await transcriptionEntered.promise; + harness.controller.cancel(); + await stopping; + + expect(signal.aborted).toBe(true); + expect(harness.commits).toEqual([]); + expect(harness.deleted).toEqual(["file:///voice.m4a"]); + expect(harness.controller.currentState.phase).toBe("idle"); + }); + + it("releases the microphone before transcription starts", async () => { + const events: string[] = []; + const harness = createHarness({ + releaseRecording: async () => { + events.push("released"); + }, + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(async () => { + events.push("transcribed"); + return "done"; + }), + }), + }); + await harness.controller.start(); + await harness.controller.stop(); + + expect(events).toEqual(["released", "transcribed"]); + }); + + it("retries audio-session release during final cleanup", async () => { + const releaseRecording = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("busy")) + .mockResolvedValueOnce(undefined); + const harness = createHarness({ releaseRecording }); + await harness.controller.start(); + await harness.controller.stop(); + + expect(releaseRecording).toHaveBeenCalledTimes(2); + }); + + it("leaves transcription with an error when recorder finalization fails", async () => { + const harness = createHarness(); + harness.recorder.stop.mockRejectedValueOnce(new Error("stop failed")); + await harness.controller.start(); + await harness.controller.stop(); + + expect(harness.controller.currentState.phase).toBe("error"); + expect(harness.controller.currentState.error).toContain("finish voice recording"); + }); + + it("ignores a late transcript after the draft owner changes", async () => { + const transcription = deferred(); + const transcriptionEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async () => + preparedTranscription(() => { + transcriptionEntered.resolve(undefined); + return transcription.promise; + }), + }), + }); + await harness.controller.start(); + const stopping = harness.controller.stop(); + await transcriptionEntered.promise; + harness.setDraft(draft({ ownerKey: "environment:other-thread" })); + transcription.resolve("late text"); + await stopping; + + expect(harness.commits).toEqual([]); + expect(harness.controller.currentState.error).toContain("draft changed"); + }); + + it("keeps the app-wide session locked until canceled preparation settles", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const first = createHarness({ + getTranscriber: () => ({ + prepare: ({ signal }) => { + preparationEntered.resolve(signal); + return preparation.promise; + }, + }), + }); + const firstStart = first.controller.start(); + const signal = await preparationEntered.promise; + first.controller.cancel(); + expect(signal.aborted).toBe(true); + + const blocked = createHarness(); + await blocked.controller.start(); + expect(blocked.controller.currentState.error).toContain("already active"); + + preparation.resolve(preparedTranscription()); + await firstStart; + expect(first.recorder.record).not.toHaveBeenCalled(); + blocked.controller.cancel(); + + const next = createHarness(); + await next.controller.start(); + expect(next.controller.currentState.phase).toBe("recording"); + await next.controller.interruptRecording(); + }); + + it("does not start the microphone for an owner that changed during preparation", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: () => { + preparationEntered.resolve(undefined); + return preparation.promise; + }, + }), + }); + const starting = harness.controller.start(); + await preparationEntered.promise; + harness.setDraft(draft({ ownerKey: "environment:other-thread", text: "other draft" })); + preparation.resolve(preparedTranscription()); + await starting; + + expect(harness.recorder.record).not.toHaveBeenCalled(); + expect(harness.controller.currentState.error).toContain("no longer available"); + }); + + it("discards recorder errors and audio interruptions without transcribing", async () => { + const transcribe = vi.fn(async () => "ignored"); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: async ({ signal }) => { + preparationEntered.resolve(signal); + return preparedTranscription(transcribe); + }, + }), + }); + await harness.controller.start(); + const signal = await preparationEntered.promise; + harness.recorder.uri = "file:///reset-empty.m4a"; + await harness.controller.handleRecorderStatus({ + isFinished: true, + hasError: true, + error: "Audio route changed", + url: "file:///voice.m4a", + }); + + expect(harness.commits).toEqual([]); + expect(transcribe).not.toHaveBeenCalled(); + expect(signal.aborted).toBe(true); + expect(harness.controller.currentState.error).toBe("Audio route changed"); + expect(harness.deleted).toEqual(["file:///voice.m4a", "file:///reset-empty.m4a"]); + }); + + it("cancels preparation when the app reaches the background", async () => { + const preparation = deferred(); + const preparationEntered = deferred(); + const harness = createHarness({ + getTranscriber: () => ({ + prepare: ({ signal }) => { + preparationEntered.resolve(signal); + return preparation.promise; + }, + }), + }); + const starting = harness.controller.start(); + const signal = await preparationEntered.promise; + harness.controller.appMovedToBackground(); + expect(signal.aborted).toBe(true); + preparation.resolve(preparedTranscription()); + await starting; + + expect(harness.recorder.record).not.toHaveBeenCalled(); + expect(harness.controller.currentState.error).toContain("background"); + }); +}); diff --git a/packages/client-runtime/src/voice-input/controller.ts b/packages/client-runtime/src/voice-input/controller.ts new file mode 100644 index 000000000..cb284ad9f --- /dev/null +++ b/packages/client-runtime/src/voice-input/controller.ts @@ -0,0 +1,493 @@ +import { replaceTextRange } from "@t3tools/shared/composerTrigger"; + +import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts"; + +export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60; + +export type VoiceInputPhase = "idle" | "preparing" | "recording" | "transcribing" | "error"; + +export type VoiceInputState = { + readonly phase: VoiceInputPhase; + readonly error: string | null; + readonly errorAction: "retry" | "settings" | null; +}; + +export function voiceInputBlocksSubmission(state: VoiceInputState): boolean { + return ( + state.phase === "preparing" || state.phase === "recording" || state.phase === "transcribing" + ); +} + +export function voiceInputFreezesEditor(state: VoiceInputState): boolean { + return voiceInputBlocksSubmission(state); +} + +export type VoiceDraftSnapshot = { + readonly ownerKey: string; + readonly text: string; + readonly selection: { readonly start: number; readonly end: number }; + readonly revision: number; +}; + +export type VoiceRecorderStatus = { + readonly isFinished: boolean; + readonly hasError: boolean; + readonly error: string | null; + readonly url: string | null; +}; + +export interface VoiceRecorder { + readonly uri: string | null; + prepareToRecordAsync(): Promise; + record(options: { readonly forDuration: number }): void; + stop(): Promise; +} + +export type VoiceInputControllerDependencies = { + readonly recorder: VoiceRecorder; + readonly getTranscriber: () => VoiceTranscriber | null; + readonly requestPermission: () => Promise<{ + readonly granted: boolean; + readonly canAskAgain: boolean; + }>; + readonly configureRecording: () => Promise; + readonly releaseRecording: () => Promise; + readonly deleteRecording: (uri: string) => void; + readonly readDraft: () => VoiceDraftSnapshot | null; + readonly commitDraft: ( + text: string, + selection: { readonly start: number; readonly end: number }, + ) => void; + readonly onStateChange: (state: VoiceInputState) => void; +}; + +type TranscriptCommitResult = + | { + readonly kind: "commit"; + readonly text: string; + readonly selection: { readonly start: number; readonly end: number }; + } + | { readonly kind: "stale" } + | { readonly kind: "empty" }; + +export function resolveTranscriptCommit( + captured: VoiceDraftSnapshot, + current: VoiceDraftSnapshot | null, + transcript: string, + locale: string, +): TranscriptCommitResult { + if ( + !current || + current.ownerKey !== captured.ownerKey || + current.text !== captured.text || + current.revision !== captured.revision + ) { + return { kind: "stale" }; + } + + const replacement = transcript.trim(); + if (replacement.length === 0) { + return { kind: "empty" }; + } + + const isEmptySelection = captured.selection.start === captured.selection.end; + const normalizedLocale = locale.replaceAll("_", "-").toLowerCase(); + const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-"); + let insertion = replacement; + if (isEmptySelection && usesEnglishSpacing) { + const left = captured.text[captured.selection.start - 1]; + const right = captured.text[captured.selection.start]; + const leftNeedsBoundary = + left !== undefined && + /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) && + (right === undefined || /\s/.test(right)); + const rightNeedsBoundary = + right !== undefined && + /[A-Za-z0-9([{'"]/.test(right) && + (left === undefined || /\s/.test(left)); + insertion = `${leftNeedsBoundary ? " " : ""}${replacement}${rightNeedsBoundary ? " " : ""}`; + } + + const result = replaceTextRange( + captured.text, + captured.selection.start, + captured.selection.end, + insertion, + ); + return { + kind: "commit", + text: result.text, + selection: { start: result.cursor, end: result.cursor }, + }; +} + +let activeSession: symbol | null = null; +let activeTranscriptionOperation: Promise | null = null; + +function acquireSession(): symbol | null { + if (activeSession) return null; + const token = Symbol("voice-input-session"); + activeSession = token; + return token; +} + +function releaseSession(token: symbol | null): void { + if (token && activeSession === token) activeSession = null; +} + +async function runTranscriptionOperation(operation: () => Promise): Promise { + if (activeTranscriptionOperation) { + throw new Error("voice-operation-busy"); + } + + const promise = operation(); + activeTranscriptionOperation = promise; + try { + return await promise; + } finally { + if (activeTranscriptionOperation === promise) activeTranscriptionOperation = null; + } +} + +function errorCode(error: unknown): string | null { + if (typeof error !== "object" || error === null || !("code" in error)) return null; + return typeof error.code === "string" ? error.code : null; +} + +function preparationErrorMessage(error: unknown): string { + if (error instanceof Error && error.message === "voice-operation-busy") { + return "Voice transcription is still finishing. Try again shortly."; + } + if (errorCode(error) === "unsupported-locale") { + return "Voice transcription is not available for this language."; + } + return "Could not prepare voice transcription."; +} + +function transcriptionErrorMessage(error: unknown): string { + if (error instanceof Error && error.message === "voice-operation-busy") { + return "Voice transcription is still finishing. Try again shortly."; + } + return "Could not transcribe this recording."; +} + +const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null }; + +export class VoiceInputController { + private readonly dependencies: VoiceInputControllerDependencies; + private state: VoiceInputState = IDLE_STATE; + private operationToken = 0; + private sessionToken: symbol | null = null; + private transcription: PreparedVoiceTranscription | null = null; + private transcriptionAbortController: AbortController | null = null; + private capturedDraft: VoiceDraftSnapshot | null = null; + private recordingUri: string | null = null; + private readonly ownedRecordingUris = new Set(); + private recordingConfigured = false; + private finishing = false; + + constructor(dependencies: VoiceInputControllerDependencies) { + this.dependencies = dependencies; + } + + get currentState(): VoiceInputState { + return this.state; + } + + async start(): Promise { + if (this.state.phase !== "idle" && this.state.phase !== "error") return; + const initiatingDraft = this.dependencies.readDraft(); + if (!initiatingDraft) { + this.setError("This draft is no longer available.", "retry"); + return; + } + const sessionToken = acquireSession(); + if (!sessionToken) { + this.setError("Another voice recording is already active.", "retry"); + return; + } + + this.sessionToken = sessionToken; + const operationToken = ++this.operationToken; + const abortController = new AbortController(); + this.transcriptionAbortController = abortController; + this.setState({ phase: "preparing", error: null, errorAction: null }); + + try { + const transcriber = this.dependencies.getTranscriber(); + if (!transcriber) { + this.setError("Voice transcription is not available.", null); + return; + } + + const permission = await this.dependencies.requestPermission(); + if (!this.isCurrent(operationToken)) return; + if (!permission.granted) { + this.setError( + "Microphone access is required for voice input.", + permission.canAskAgain ? "retry" : "settings", + ); + return; + } + + try { + this.transcription = await runTranscriptionOperation(() => + transcriber.prepare({ signal: abortController.signal }), + ); + } catch (error) { + if (this.isCurrent(operationToken)) this.setError(preparationErrorMessage(error), "retry"); + return; + } + if (!this.isCurrent(operationToken)) return; + + await this.dependencies.configureRecording(); + this.recordingConfigured = true; + if (!this.isCurrent(operationToken)) return; + await this.dependencies.recorder.prepareToRecordAsync(); + if (!this.isCurrent(operationToken)) return; + this.recordingUri = this.dependencies.recorder.uri; + this.rememberRecordingUri(this.recordingUri); + + const capturedDraft = this.dependencies.readDraft(); + if (!capturedDraft || capturedDraft.ownerKey !== initiatingDraft.ownerKey) { + this.setError("This draft is no longer available.", "retry"); + return; + } + this.capturedDraft = capturedDraft; + this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS }); + this.setState({ phase: "recording", error: null, errorAction: null }); + } catch { + if (this.isCurrent(operationToken)) + this.setError("Could not start voice recording.", "retry"); + } finally { + if (this.isCurrent(operationToken) && this.state.phase === "error") { + await this.releaseResources(); + } else if (!this.isCurrent(operationToken) && !this.finishing) { + await this.releaseResources(); + } + } + } + + stop(): Promise { + if (this.state.phase !== "recording") return Promise.resolve(); + return this.finishRecording(false, null); + } + + cancel(): void { + switch (this.state.phase) { + case "idle": + return; + case "error": + this.setState(IDLE_STATE); + return; + case "preparing": + this.invalidateOperation(); + this.setState(IDLE_STATE); + return; + case "recording": + this.discardRecording(null); + return; + case "transcribing": + this.invalidateOperation(); + this.setState(IDLE_STATE); + return; + } + } + + interruptRecording( + message = "Voice recording was interrupted.", + completedUri: string | null = null, + ): Promise | void { + if (this.state.phase !== "recording") return; + this.rememberRecordingUri(completedUri); + this.recordingUri = completedUri ?? this.recordingUri; + return this.discardRecording(message); + } + + appMovedToBackground(): Promise | void { + if (this.state.phase === "preparing") { + this.invalidateOperation(); + this.setError("Voice input stopped when the app moved to the background.", "retry"); + return; + } + return this.interruptRecording(); + } + + handleRecorderStatus(status: VoiceRecorderStatus): Promise | void { + if (this.state.phase !== "recording") return; + if (status.hasError) { + return this.interruptRecording( + status.error ?? "Voice recording was interrupted.", + status.url, + ); + } + if (status.isFinished) { + if (!status.url) { + return this.interruptRecording(); + } + return this.finishRecording(true, status.url); + } + } + + ownerChanged(): void { + if (this.state.phase === "idle") return; + this.cancel(); + } + + dispose(): void { + if (this.state.phase === "recording") { + this.discardRecording(null); + return; + } + if (this.state.phase === "preparing" || this.state.phase === "transcribing") { + this.invalidateOperation(); + this.setState(IDLE_STATE); + } + } + + private async finishRecording( + alreadyStopped: boolean, + completedUri: string | null, + ): Promise { + if (this.finishing || this.state.phase !== "recording") return; + this.finishing = true; + const operationToken = this.operationToken; + this.setState({ phase: "transcribing", error: null, errorAction: null }); + + try { + if (!alreadyStopped) await this.dependencies.recorder.stop(); + await this.releaseAudioSession(); + this.recordingUri = completedUri ?? this.dependencies.recorder.uri ?? this.recordingUri; + this.rememberRecordingUri(this.recordingUri); + if (!this.isCurrent(operationToken)) return; + if ( + !this.recordingUri || + !this.transcription || + !this.transcriptionAbortController || + !this.capturedDraft + ) { + this.setError("Could not finish voice recording.", "retry"); + return; + } + + const recordingUri = this.recordingUri; + const transcription = this.transcription; + const signal = this.transcriptionAbortController.signal; + const capturedDraft = this.capturedDraft; + let transcript: string; + try { + transcript = await runTranscriptionOperation(() => + transcription.transcribe(recordingUri, { signal }), + ); + } catch (error) { + if (this.isCurrent(operationToken)) { + this.setError(transcriptionErrorMessage(error), "retry"); + } + return; + } + if (!this.isCurrent(operationToken)) return; + + const result = resolveTranscriptCommit( + capturedDraft, + this.dependencies.readDraft(), + transcript, + transcription.locale, + ); + if (result.kind === "stale") { + this.setError( + "The draft changed while voice input was running. The transcript was not added.", + "retry", + ); + return; + } + if (result.kind === "empty") { + this.setError("No speech was detected.", "retry"); + return; + } + + this.dependencies.commitDraft(result.text, result.selection); + this.setState(IDLE_STATE); + } catch { + if (this.isCurrent(operationToken)) { + this.setError("Could not finish voice recording.", "retry"); + } + } finally { + this.finishing = false; + await this.releaseResources(); + } + } + + private async discardRecording(error: string | null): Promise { + this.invalidateOperation(); + this.setState( + error + ? { phase: "error", error, errorAction: "retry" } + : { phase: "idle", error: null, errorAction: null }, + ); + try { + await this.dependencies.recorder.stop(); + this.rememberRecordingUri(this.dependencies.recorder.uri); + } catch { + this.rememberRecordingUri(this.dependencies.recorder.uri); + } finally { + await this.releaseResources(); + } + } + + private async releaseResources(): Promise { + this.rememberRecordingUri(this.recordingUri); + this.rememberRecordingUri(this.dependencies.recorder.uri); + this.recordingUri = null; + for (const uri of this.ownedRecordingUris) { + try { + this.dependencies.deleteRecording(uri); + } catch { + // The cache may already have removed a failed or interrupted recording. + } + } + this.ownedRecordingUris.clear(); + await this.releaseAudioSession(); + releaseSession(this.sessionToken); + this.sessionToken = null; + this.capturedDraft = null; + this.transcription = null; + this.transcriptionAbortController = null; + } + + private rememberRecordingUri(uri: string | null): void { + if (uri) this.ownedRecordingUris.add(uri); + } + + private async releaseAudioSession(): Promise { + if (!this.recordingConfigured) return; + try { + await this.dependencies.releaseRecording(); + this.recordingConfigured = false; + } catch { + // Final cleanup retries if the prompt release before transcription fails. + } + } + + private invalidateOperation(): void { + this.operationToken += 1; + this.transcriptionAbortController?.abort(); + } + + private isCurrent(operationToken: number): boolean { + return operationToken === this.operationToken; + } + + private setError(error: string, errorAction: VoiceInputState["errorAction"]): void { + this.setState({ phase: "error", error, errorAction }); + } + + private setState(state: VoiceInputState): void { + this.state = state; + this.dependencies.onStateChange(state); + } +} + +export function resetVoiceInputGlobalsForTests(): void { + activeSession = null; + activeTranscriptionOperation = null; +} diff --git a/packages/client-runtime/src/voice-input/index.ts b/packages/client-runtime/src/voice-input/index.ts new file mode 100644 index 000000000..c8c8da455 --- /dev/null +++ b/packages/client-runtime/src/voice-input/index.ts @@ -0,0 +1,21 @@ +export { + VoiceInputController, + VOICE_RECORDING_LIMIT_SECONDS, + resolveTranscriptCommit, + voiceInputBlocksSubmission, + voiceInputFreezesEditor, + type VoiceDraftSnapshot, + type VoiceInputControllerDependencies, + type VoiceInputPhase, + type VoiceInputState, + type VoiceRecorder, + type VoiceRecorderStatus, +} from "./controller.ts"; +export { + VoiceTranscriptionError, + throwIfVoiceTranscriptionAborted, + type PreparedVoiceTranscription, + type VoiceTranscriber, + type VoiceTranscriptionErrorCode, + type VoiceTranscriptionOptions, +} from "./transcription.ts"; diff --git a/packages/client-runtime/src/voice-input/transcription.ts b/packages/client-runtime/src/voice-input/transcription.ts new file mode 100644 index 000000000..f3ce377ac --- /dev/null +++ b/packages/client-runtime/src/voice-input/transcription.ts @@ -0,0 +1,37 @@ +/** Cancellation is cooperative: settle only after the underlying work has stopped. */ +export type VoiceTranscriptionOptions = { + readonly signal: AbortSignal; +}; + +/** Binds a recording to its selected implementation and resolved locale. */ +export type PreparedVoiceTranscription = { + readonly locale: string; + readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise; +}; + +export type VoiceTranscriber = { + readonly prepare: (options: VoiceTranscriptionOptions) => Promise; +}; + +export type VoiceTranscriptionErrorCode = + | "unavailable" + | "unsupported-locale" + | "preparation-failed" + | "transcription-failed" + | "cancelled"; + +export class VoiceTranscriptionError extends Error { + readonly code: VoiceTranscriptionErrorCode; + + constructor(code: VoiceTranscriptionErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.name = "VoiceTranscriptionError"; + this.code = code; + } +} + +export function throwIfVoiceTranscriptionAborted(signal: AbortSignal): void { + if (signal.aborted) { + throw new VoiceTranscriptionError("cancelled", "Voice transcription was cancelled."); + } +} diff --git a/patches/@react-native-ai__apple@0.12.0.patch b/patches/@react-native-ai__apple@0.12.0.patch new file mode 100644 index 000000000..b2a7aaf22 --- /dev/null +++ b/patches/@react-native-ai__apple@0.12.0.patch @@ -0,0 +1,194 @@ +diff --git a/ios/transcription/AppleTranscriptionImpl.swift b/ios/transcription/AppleTranscriptionImpl.swift +index 188371a5f55fadae19187108c4a688569d0223e3..3e1cdd77b0bb8ae6c97a40d29c1854200d931eee 100644 +--- a/ios/transcription/AppleTranscriptionImpl.swift ++++ b/ios/transcription/AppleTranscriptionImpl.swift +@@ -12,6 +12,11 @@ import UniformTypeIdentifiers + + @objc + public class AppleTranscriptionImpl: NSObject { ++ private struct CollectedSegment: Sendable { ++ let text: String ++ let startSecond: Double ++ let endSecond: Double ++ } + + @available(iOS 26, *) + private func createTranscriber(for locale: Locale) -> SpeechTranscriber { +@@ -41,7 +46,7 @@ public class AppleTranscriptionImpl: NSObject { + let locale = Locale(identifier: language) + + guard let supportedLocale = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { +- reject("AppleTranscription", "Locale not supported: \(language)", nil) ++ reject("AppleTranscriptionUnsupportedLocale", "Locale not supported: \(language)", nil) + return + } + +@@ -51,16 +56,14 @@ public class AppleTranscriptionImpl: NSObject { + + switch status { + case .installed: +- resolve(nil) ++ resolve(supportedLocale.identifier) + case .supported, .downloading: +- if let request = try? await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { ++ if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + try await request.downloadAndInstall() +- resolve(nil) +- } else { +- resolve(nil) + } ++ resolve(supportedLocale.identifier) + case .unsupported: +- reject("AppleTranscription", "Assets not supported for locale: \(supportedLocale.identifier)", nil) ++ reject("AppleTranscriptionUnsupportedLocale", "Assets not supported for locale: \(supportedLocale.identifier)", nil) + @unknown default: + reject ("AppleTranscription", "Unknown asset inventory status", nil) + } +@@ -83,57 +86,77 @@ public class AppleTranscriptionImpl: NSObject { + do { + try audioData.write(to: fileURL) + +- guard let audioFile = try? AVAudioFile(forReading: fileURL) else { +- reject("AppleTranscription", "Invalid audio data", nil) ++ let audioFile: AVAudioFile ++ do { ++ audioFile = try AVAudioFile(forReading: fileURL) ++ } catch { ++ try? FileManager.default.removeItem(at: fileURL) ++ reject("AppleTranscription", "Invalid audio data", error) + return + } + + Task { ++ defer { ++ try? FileManager.default.removeItem(at: fileURL) ++ } ++ + do { + let transcriber = createTranscriber(for: Locale(identifier: language)) + + let analyzer = SpeechAnalyzer(modules: [transcriber]) +- +- defer { +- try? FileManager.default.removeItem(at: fileURL) +- } +- +- var segments: [[String: Any]] = [] +- +- Task { ++ ++ let collectorTask = Task { () throws -> [CollectedSegment] in ++ var segments: [CollectedSegment] = [] ++ + for try await result in transcriber.results { + if result.isFinal { +- let segment: [String: Any] = [ +- "text": String(result.text.characters), +- "startSecond": CMTimeGetSeconds(result.range.start), +- "endSecond": CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) +- ] +- segments.append(segment) ++ segments.append( ++ CollectedSegment( ++ text: String(result.text.characters), ++ startSecond: CMTimeGetSeconds(result.range.start), ++ endSecond: CMTimeGetSeconds(CMTimeRangeGetEnd(result.range)) ++ ) ++ ) + } + } ++ ++ return segments + } +- +- let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) +- +- if let lastSampleTime { +- try await analyzer.finalizeAndFinish(through: lastSampleTime) +- } else { ++ ++ do { ++ let lastSampleTime = try await analyzer.analyzeSequence(from: audioFile) ++ ++ if let lastSampleTime { ++ try await analyzer.finalizeAndFinish(through: lastSampleTime) ++ } else { ++ await analyzer.cancelAndFinishNow() ++ } ++ ++ let segments: [[String: Any]] = try await collectorTask.value.map { segment in ++ [ ++ "text": segment.text, ++ "startSecond": segment.startSecond, ++ "endSecond": segment.endSecond ++ ] ++ } ++ let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } ++ ++ resolve([ ++ "segments": segments, ++ "duration": totalDuration ++ ]) ++ } catch { ++ collectorTask.cancel() + await analyzer.cancelAndFinishNow() ++ _ = try? await collectorTask.value ++ throw error + } +- +- let totalDuration = if let lastSampleTime { CMTimeGetSeconds(lastSampleTime) } else { 0.0 } +- +- let result: [String: Any] = [ +- "segments": segments, +- "duration": totalDuration +- ] +- +- resolve(result) + } catch { + reject("AppleTranscription", "Transcription failed: \(error.localizedDescription)", error) + } + } + } catch { ++ try? FileManager.default.removeItem(at: fileURL) + reject("AppleTranscription", "Failed to write audio data: \(error.localizedDescription)", error) + } + } else { +@@ -141,4 +164,3 @@ public class AppleTranscriptionImpl: NSObject { + } + } + } +- +diff --git a/lib/typescript/NativeAppleTranscription.d.ts b/lib/typescript/NativeAppleTranscription.d.ts +index 985b6b3593a884c41d689346be9d73d86c55eae1..86e7936f23a8673d099f7fe18af9a31abae5a13a 100644 +--- a/lib/typescript/NativeAppleTranscription.d.ts ++++ b/lib/typescript/NativeAppleTranscription.d.ts +@@ -10,14 +10,14 @@ export interface TranscriptionResult { + } + export interface Spec extends TurboModule { + isAvailable(language: string): boolean; +- prepare(language: string): Promise; ++ prepare(language: string): Promise; + } + declare global { + function __apple__llm__transcribe__(data: ArrayBufferLike, language: string): Promise; + } + declare const _default: { + transcribe: (data: ArrayBufferLike, language: string) => Promise; +- prepare: (language: string) => Promise; ++ prepare: (language: string) => Promise; + isAvailable: (language: string) => boolean; + }; + export default _default; +diff --git a/src/NativeAppleTranscription.ts b/src/NativeAppleTranscription.ts +index 13332a0176000b60b6f6f043de681db55a7b4bef..5389fb4b3a53de48721c307d2145b6770624c852 100644 +--- a/src/NativeAppleTranscription.ts ++++ b/src/NativeAppleTranscription.ts +@@ -14,7 +14,7 @@ export interface TranscriptionResult { + + export interface Spec extends TurboModule { + isAvailable(language: string): boolean +- prepare(language: string): Promise ++ prepare(language: string): Promise + } + + declare global { diff --git a/patches/expo-audio@57.0.4.patch b/patches/expo-audio@57.0.4.patch new file mode 100644 index 000000000..1452eca31 --- /dev/null +++ b/patches/expo-audio@57.0.4.patch @@ -0,0 +1,24 @@ +diff --git a/ios/AudioRecorder.swift b/ios/AudioRecorder.swift +index 20020bff9f8ba7bb6e7f61c99b4de5d29eee00db..284c22f12e30c8c2ed16feb141876cbfd8897c90 100644 +--- a/ios/AudioRecorder.swift ++++ b/ios/AudioRecorder.swift +@@ -216,15 +216,15 @@ class AudioRecorder: SharedRef, RecordingResultHandler { + } + + func didFinish(_ recorder: AVAudioRecorder, successfully flag: Bool) { +- // Update internal state when recording finishes automatically (e.g., from recordForDuration) +- currentState = .stopped ++ // Update internal state when AVAudioRecorder finishes or fails. ++ currentState = flag ? .stopped : .error + resetDurationTracking() + + emit(event: recordingStatus, payload: [ + "id": id, + "isFinished": true, +- "hasError": false, +- "error": nil, ++ "hasError": !flag, ++ "error": flag ? nil : "Recording failed", + "url": recorder.url.absoluteString + ]) + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 006f3fc53..7149a07f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -89,9 +89,11 @@ patchedDependencies: '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca368 '@legendapp/list@3.3.5': d14852e52d4bfb128bf8c3c1476cb8ca4dd392999f21f13fd62ac3bf04a62c40 '@pierre/diffs@1.3.0-beta.10': 7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa + '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d '@react-native-menu/menu@2.0.0': c7f66d121c726ade4f5c4e1aed11a691e5711d244c544084e289ac26132a0045 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e2295552 effect@4.0.0-beta.103: af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6 + expo-audio@57.0.4: fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a expo-modules-jsi@57.0.6: 0794db2805abb43f770292fea9afbd80a85726082d33709237182a8d1568f133 expo-sharing@57.0.16: 8d2e3b10eb3f52036a9a086800180ec6cebf3b75bccc5b1775117a7244d4ac45 react-native-gesture-handler@2.32.0: 808eb26f9e57cf4945efd3985af4d9c764da6f91f4c9764433cc868602bbf4d3 @@ -237,6 +239,9 @@ importers: '@pierre/diffs': specifier: 'catalog:' version: 1.3.0-beta.10(patch_hash=7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) + '@react-native-ai/apple': + specifier: 0.12.0 + version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)) '@react-native-menu/menu': specifier: ^2.0.0 version: 2.0.0(patch_hash=c7f66d121c726ade4f5c4e1aed11a691e5711d244c544084e289ac26132a0045)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -297,6 +302,9 @@ importers: expo-asset: specifier: ~57.0.15 version: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + expo-audio: + specifier: ~57.0.4 + version: 57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) expo-auth-session: specifier: ~57.0.10 version: 57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) @@ -997,6 +1005,16 @@ importers: packages: + '@ai-sdk/provider-utils@4.0.49': + resolution: {integrity: sha512-8e7pd+82bobqrFOaD5dG/PiEuvLYr5olaE3I56ch0jipR0H7sGD6ohwTUynv6k8O8QidWiyIbEsZCtr/2dyXIA==} + engines: {node: '>=18.17'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@3.0.15': + resolution: {integrity: sha512-XeZW1CcDF2GMbH4wejW6xBRI2QCOgnkVYUnxoeDadB1mf85riL2bMUeDoh+6gJ/r4mjNfzUPW8OjLjvwTP0u1Q==} + engines: {node: '>=18'} + '@alcalzone/ansi-tokenize@0.2.5': resolution: {integrity: sha512-3NX/MpTdroi0aKz134A6RC2Gb2iXVECN4QaAXnvCIxxIm3C3AVB1mkUe8NaaiyvOpDfsrqWhYtj+Q6a62RrTsw==} engines: {node: '>=18'} @@ -3872,6 +3890,11 @@ packages: resolution: {integrity: sha512-gMDYY2rw6OWajCcDlXSIgs2LC432YJXSb3Lm5yM187uhRgBYddoEVULi36h+IolX3r7jSb3ew7vn9FfI8NSo0A==} hasBin: true + '@react-native-ai/apple@0.12.0': + resolution: {integrity: sha512-BC/kEDbCZprv1xcQCWsgapXm/WEsj5lieBDvU1vJXStrN+BG+hWM01zoqDGTD/j71zvt9DEfHcMvEqbrGY2uAA==} + peerDependencies: + react-native: '>=0.76.0' + '@react-native-masked-view/masked-view@0.3.2': resolution: {integrity: sha512-XwuQoW7/GEgWRMovOQtX3A4PrXhyaZm0lVUiY8qJDvdngjLms9Cpdck6SmGAUNqQwcj2EadHC1HwL0bEyoa/SQ==} peerDependencies: @@ -6627,6 +6650,14 @@ packages: react: '*' react-native: '*' + expo-audio@57.0.4: + resolution: {integrity: sha512-TLP8rt1UvUDzgxGnyQ0TR9hV6tNP/UJQdDu7mSK+dgEydMFoptq55D3hcUoB1gF39f3/3AUuWfOtpGM+4N4X1A==} + peerDependencies: + expo: '*' + expo-asset: '*' + react: '*' + react-native: '*' + expo-auth-session@57.0.10: resolution: {integrity: sha512-i1RY93LouEgal2e/Ly1r2Ytwed7/y6i4AUShVj4Gg49h/Okf8QpZO7XylwL43IHfKRrKkqapRDEN8mWe7z/HgA==} peerDependencies: @@ -7618,6 +7649,9 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stringify-safe@5.0.1: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} @@ -10080,8 +10114,8 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - undici@6.26.0: - resolution: {integrity: sha512-4yqz8a3n5HmGTlsbADNtr/dJlhkh/55Rq798G6ibiULcXbDtaLpTl1pvdqcbFfeoj3iSi52lePFM7h9H21cw/A==} + undici@6.28.0: + resolution: {integrity: sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==} engines: {node: '>=18.17'} undici@7.27.1: @@ -10738,6 +10772,18 @@ packages: snapshots: + '@ai-sdk/provider-utils@4.0.49(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.15 + '@standard-schema/spec': 1.1.0 + eventsource-parser: 3.1.0 + undici: 6.28.0 + zod: 4.4.3 + + '@ai-sdk/provider@3.0.15': + dependencies: + json-schema: 0.4.0 + '@alcalzone/ansi-tokenize@0.2.5': dependencies: ansi-styles: 6.2.3 @@ -14094,6 +14140,13 @@ snapshots: prompts: 2.4.2 tinyexec: 1.2.4 + '@react-native-ai/apple@0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))': + dependencies: + '@ai-sdk/provider': 3.0.15 + '@ai-sdk/provider-utils': 4.0.49(zod@4.4.3) + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + zod: 4.4.3 + '@react-native-masked-view/masked-view@0.3.2(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)': dependencies: react: 19.2.3 @@ -16817,6 +16870,13 @@ snapshots: - typescript optional: true + expo-audio@57.0.4(patch_hash=fa9a3e0442ed395d4071bb406e08c3a471c9a84700bdfa0b9ad7ff144c96041a)(expo-asset@57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): + dependencies: + expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc) + expo-asset: 57.0.15(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)(typescript@6.0.3) + react: 19.2.3 + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + expo-auth-session@57.0.10(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): dependencies: expo-application: 57.0.2(expo@57.0.18) @@ -18100,6 +18160,8 @@ snapshots: json-schema-typed@8.0.2: {} + json-schema@0.4.0: {} + json-stringify-safe@5.0.1: optional: true @@ -19364,7 +19426,7 @@ snapshots: semver: 7.8.5 tar: 7.5.16 tinyglobby: 0.2.17 - undici: 6.26.0 + undici: 6.28.0 which: 6.0.1 node-int64@0.4.0: {} @@ -21239,7 +21301,7 @@ snapshots: undici-types@7.16.0: {} - undici@6.26.0: {} + undici@6.28.0: {} undici@7.27.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 417d11062..040ee7af7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -153,9 +153,11 @@ patchedDependencies: "@ff-labs/fff-node@0.9.4": patches/@ff-labs__fff-node@0.9.4.patch "@legendapp/list@3.3.5": patches/@legendapp__list@3.3.5.patch "@pierre/diffs@1.3.0-beta.10": patches/@pierre%2Fdiffs@1.3.0-beta.10.patch + "@react-native-ai/apple@0.12.0": patches/@react-native-ai__apple@0.12.0.patch "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch effect@4.0.0-beta.103: patches/effect@4.0.0-beta.103.patch + expo-audio@57.0.4: patches/expo-audio@57.0.4.patch expo-modules-jsi@57.0.6: patches/expo-modules-jsi@57.0.6.patch expo-sharing@57.0.16: patches/expo-sharing@57.0.16.patch react-native-gesture-handler@2.32.0: patches/react-native-gesture-handler@2.32.0.patch