diff --git a/apps/mobile/app.config.ts b/apps/mobile/app.config.ts index 92cf34e86..4a0563337 100644 --- a/apps/mobile/app.config.ts +++ b/apps/mobile/app.config.ts @@ -249,6 +249,7 @@ const config: ExpoConfig = { }, NSLocalNetworkUsageDescription: "Allow Pylon to connect to Pylon servers on your local network or tailnet.", + NSPhotoLibraryAddUsageDescription: "Allow Pylon to save images to your photo library.", ITSAppUsesNonExemptEncryption: false, // The App Store screenshot harness rotates the iPad interface from // inside the app (CI denies osascript the Accessibility access that diff --git a/apps/mobile/modules/t3-native-controls/ios/T3NativeControlsModule.swift b/apps/mobile/modules/t3-native-controls/ios/T3NativeControlsModule.swift index 6aa8fa6bb..ddc8a8027 100644 --- a/apps/mobile/modules/t3-native-controls/ios/T3NativeControlsModule.swift +++ b/apps/mobile/modules/t3-native-controls/ios/T3NativeControlsModule.swift @@ -3,9 +3,57 @@ import Security import UIKit public final class T3NativeControlsModule: Module { + private let presentationSources = T3PresentationSources() + private var videoPresentation: T3NativeVideoPresentation? + private var filePresentation: T3NativeFilePresentation? + public func definition() -> ModuleDefinition { Name("T3NativeControls") + AsyncFunction("presentVideo") { (url: URL, title: String, sourceIdentifier: String, identifier: String, promise: Promise) in + try self.presentVideo( + url: url, + title: title, + sourceIdentifier: sourceIdentifier, + identifier: identifier, + promise: promise + ) + }.runOnQueue(.main) + + AsyncFunction("dismissVideo") { (identifier: String) in + self.dismissVideo(identifier: identifier) + }.runOnQueue(.main) + + AsyncFunction("presentFile") { (url: URL, title: String, sourceIdentifier: String, identifier: String, promise: Promise) in + try self.presentFile(url: url, title: title, sourceIdentifier: sourceIdentifier, + identifier: identifier, promise: promise) + }.runOnQueue(.main) + + AsyncFunction("dismissFile") { (identifier: String) in + self.dismissFile(identifier: identifier) + }.runOnQueue(.main) + + OnDestroy { + let presentation = self.videoPresentation + let file = self.filePresentation + DispatchQueue.main.async { + presentation?.dismiss() + file?.dismiss() + } + } + + View(T3PresentationSourceView.self) { + ViewName("PresentationSource") + Prop("identifier") { (view: T3PresentationSourceView, identifier: String) in + view.sources = self.presentationSources + view.identifier = identifier + } + } + + AsyncFunction("shareFileFromSource") { (url: URL, title: String, identifier: String, promise: Promise) in + try self.shareFile(url: url, title: title, sourceIdentifier: identifier, promise: promise) + }.runOnQueue(.main) + Function("getShowcasePairingUrl") { let arguments = ProcessInfo.processInfo.arguments guard @@ -101,4 +149,65 @@ public final class T3NativeControlsModule: Module { try? scene.write(toFile: readyPath, atomically: true, encoding: .utf8) } } + + private func presentVideo(url: URL, title: String, sourceIdentifier: String, identifier: String, promise: Promise) throws { + let isPlayableURL = url.isFileURL + ? FileManager.default.isReadableFile(atPath: url.path) + : (["https", "http"].contains(url.scheme?.lowercased() ?? "") && url.host != nil) + guard videoPresentation == nil, filePresentation == nil, + let presenter = appContext?.utilities?.currentViewController(), + isPlayableURL + else { + throw NSError( + domain: "T3NativeVideo", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "The video preview is no longer available."] + ) + } + let presentation = T3NativeVideoPresentation(identifier: identifier, url: url, title: title) { [weak self] error in + self?.videoPresentation = nil + if let error { promise.reject(error) } else { promise.resolve(nil) } + } + videoPresentation = presentation + presentation.present(from: presenter, sources: presentationSources, sourceIdentifier: sourceIdentifier) + } + + private func dismissVideo(identifier: String) { + if videoPresentation?.identifier == identifier { videoPresentation?.dismiss() } + } + + private func presentFile(url: URL, title: String, sourceIdentifier: String, + identifier: String, promise: Promise) throws { + guard filePresentation == nil, videoPresentation == nil, + let presenter = appContext?.utilities?.currentViewController() + else { throw URLError(.cannotLoadFromNetwork) } + let file = T3NativeFilePresentation(identifier: identifier, sources: presentationSources, + sourceIdentifier: sourceIdentifier) { [weak self] error in + self?.filePresentation = nil + if let error { promise.reject(error) } else { promise.resolve(nil) } + } + filePresentation = file + file.present(url: url, title: title, from: presenter) + } + + private func dismissFile(identifier: String) { + if filePresentation?.identifier == identifier { filePresentation?.dismiss() } + } + + private func shareFile(url: URL, title: String, sourceIdentifier: String, promise: Promise) throws { + guard let presenter = appContext?.utilities?.currentViewController() else { + throw NSError( + domain: "T3NativePresentation", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "The presenting screen is no longer open."] + ) + } + try presentFileShare( + url: url, + title: title, + source: presentationSources.view(for: sourceIdentifier), + presenter: presenter, + promise: promise + ) + } } diff --git a/apps/mobile/modules/t3-native-controls/ios/T3NativeFilePresentation.swift b/apps/mobile/modules/t3-native-controls/ios/T3NativeFilePresentation.swift new file mode 100644 index 000000000..1a7009c38 --- /dev/null +++ b/apps/mobile/modules/t3-native-controls/ios/T3NativeFilePresentation.swift @@ -0,0 +1,165 @@ +import ImageIO +import QuickLook +import UIKit +import UniformTypeIdentifiers + +private final class FilePreviewItem: NSObject, QLPreviewItem { + var previewItemURL: URL? + var previewItemTitle: String? +} + +private final class FilePreviewController: QLPreviewController { + var onAppear: (() -> Void)? + + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated) + onAppear?() + } +} + +/// Quick Look owns image and document controls, zooming, and source-view transitions. +final class T3NativeFilePresentation: NSObject, QLPreviewControllerDataSource, + QLPreviewControllerDelegate, UIAdaptivePresentationControllerDelegate { + let identifier: String + private var controller: UIViewController? + private let completion: (Error?) -> Void + private weak var sources: T3PresentationSources? + private let sourceIdentifier: String + private let item = FilePreviewItem() + private var loading: Task? + private var dismissRequested = false + private var finished = false + + init(identifier: String, sources: T3PresentationSources, sourceIdentifier: String, completion: @escaping (Error?) -> Void) { + self.identifier = identifier + self.sources = sources + self.sourceIdentifier = sourceIdentifier + self.completion = completion + super.init() + } + + func present(url: URL, title: String, from presenter: UIViewController) { + loading = Task { @MainActor [self] in + do { + let file = try await Self.prepareFile(url: url, title: title) + guard !finished, !Task.isCancelled else { + try? FileManager.default.removeItem(at: file.deletingLastPathComponent()) + return + } + item.previewItemURL = file + item.previewItemTitle = title + let preview = FilePreviewController() + preview.delegate = self + preview.dataSource = self + preview.onAppear = { [weak self] in self?.resumePendingDismissal() } + controller = preview + presenter.present(preview, animated: !UIAccessibility.isReduceMotionEnabled) { [self] in + resumePendingDismissal() + } + preview.presentationController?.delegate = self + } catch { + finish(error: error) + } + } + } + + func dismiss() { + dismissRequested = true + loading?.cancel() + guard !finished else { return } + guard let controller else { finish(); return } + // Drain Close from viewDidAppear after opening or cancelling an interactive dismissal. + // Starting a second modal transition while UIKit is settling the first can strand it. + guard !controller.isBeingPresented, !controller.isBeingDismissed else { return } + controller.dismiss(animated: !UIAccessibility.isReduceMotionEnabled) { [self] in finish() } + } + + private func resumePendingDismissal() { + // Appearance callbacks run before UIKit has cleared the current transition. + DispatchQueue.main.async { [weak self] in + if self?.dismissRequested == true { self?.dismiss() } + } + } + + func numberOfPreviewItems(in controller: QLPreviewController) -> Int { item.previewItemURL == nil ? 0 : 1 } + + func previewController(_ controller: QLPreviewController, previewItemAt index: Int) -> QLPreviewItem { + item + } + + func previewController(_ controller: QLPreviewController, transitionViewFor item: QLPreviewItem) -> UIView? { + guard !UIAccessibility.isReduceMotionEnabled else { return nil } + return sources?.view(for: sourceIdentifier) + } + + func previewController(_ controller: QLPreviewController, frameFor item: QLPreviewItem, + inSourceView view: AutoreleasingUnsafeMutablePointer) -> CGRect { + guard !UIAccessibility.isReduceMotionEnabled, let source = sources?.view(for: sourceIdentifier) else { return .zero } + view.pointee = source + return source.bounds + } + + func previewControllerDidDismiss(_ controller: QLPreviewController) { finish() } + + func presentationControllerDidDismiss(_ presentationController: UIPresentationController) { finish() } + + private func finish(error: Error? = nil) { + guard !finished else { return } + finished = true + loading?.cancel() + loading = nil + if let file = item.previewItemURL { + try? FileManager.default.removeItem(at: file.deletingLastPathComponent()) + } + item.previewItemURL = nil + DispatchQueue.main.async { [completion] in completion(error) } + } + + /// Copy original bytes so preview and sharing do not mutate a draft or workspace file. + nonisolated private static func prepareFile(url: URL, title: String) async throws -> URL { + try Task.checkCancellation() + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("t3-preview-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + do { + let download = directory.appendingPathComponent("original") + if url.isFileURL { + try FileManager.default.copyItem(at: url, to: download) + } else if url.scheme == "data" { + try Data(contentsOf: url).write(to: download, options: .atomic) + } else { + guard ["https", "http"].contains(url.scheme?.lowercased() ?? "") else { + throw URLError(.unsupportedURL) + } + let (temporaryFile, response) = try await URLSession.shared.download(from: url) + guard let response = response as? HTTPURLResponse, (200..<300).contains(response.statusCode) else { + throw URLError(.badServerResponse) + } + try FileManager.default.moveItem(at: temporaryFile, to: download) + } + try Task.checkCancellation() + let type: UTType + if let image = CGImageSourceCreateWithURL(download as CFURL, nil), + CGImageSourceGetCount(image) > 0, let imageType = CGImageSourceGetType(image), + let detectedType = UTType(imageType as String) { + type = detectedType + } else if CGPDFDocument(download as CFURL) != nil { + type = .pdf + } else { + throw URLError(.cannotDecodeContentData) + } + let filename = URL(fileURLWithPath: title).lastPathComponent as NSString + let originalExtension = filename.pathExtension + let fileExtension = UTType(filenameExtension: originalExtension) == type + ? originalExtension : type.preferredFilenameExtension ?? "png" + let stem = filename.deletingPathExtension + var name = String(stem.prefix(60)).components(separatedBy: .controlCharacters).joined(separator: "_") + while name.utf8.count > 200 { name.removeLast() } + let file = directory.appendingPathComponent("\(name.isEmpty ? "Preview" : name).\(fileExtension)") + try FileManager.default.moveItem(at: download, to: file) + return file + } catch { + try? FileManager.default.removeItem(at: directory) + throw error + } + } +} diff --git a/apps/mobile/modules/t3-native-controls/ios/T3NativePresentation.swift b/apps/mobile/modules/t3-native-controls/ios/T3NativePresentation.swift new file mode 100644 index 000000000..f537e8704 --- /dev/null +++ b/apps/mobile/modules/t3-native-controls/ios/T3NativePresentation.swift @@ -0,0 +1,75 @@ +import ExpoModulesCore +import UIKit + +final class T3PresentationSources { + private class Entry { + weak var view: UIView? + init(_ view: UIView) { self.view = view } + } + + private var entries: [String: Entry] = [:] + + func register(_ view: UIView, identifier: String) { + entries[identifier] = Entry(view) + } + + func remove(_ view: UIView, identifier: String) { + if entries[identifier]?.view == nil || entries[identifier]?.view === view { + entries.removeValue(forKey: identifier) + } + } + + func view(for identifier: String) -> UIView? { + // Use the child bounds, not the wrapper's potentially stretched layout bounds. + entries[identifier]?.view?.subviews.first + } +} + +final class T3PresentationSourceView: ExpoView { + weak var sources: T3PresentationSources? + var identifier = "" { + didSet { + sources?.remove(self, identifier: oldValue) + if !identifier.isEmpty { sources?.register(self, identifier: identifier) } + } + } + + deinit { + sources?.remove(self, identifier: identifier) + } +} + +func presentFileShare( + url: URL, + title: String, + source: UIView?, + presenter: UIViewController, + promise: Promise +) throws { + guard url.isFileURL, FileManager.default.isReadableFile(atPath: url.path) else { + throw NSError( + domain: "T3NativePresentation", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "The file is no longer available."] + ) + } + + guard let origin = source ?? presenter.view else { + throw NSError( + domain: "T3NativePresentation", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "The presenting screen is no longer open."] + ) + } + + let activity = UIActivityViewController(activityItems: [url], applicationActivities: nil) + activity.title = title + activity.overrideUserInterfaceStyle = source?.traitCollection.userInterfaceStyle + ?? presenter.traitCollection.userInterfaceStyle + activity.completionWithItemsHandler = { _, _, _, _ in promise.resolve(nil) } + activity.modalPresentationStyle = .popover + activity.popoverPresentationController?.sourceView = origin + activity.popoverPresentationController?.sourceRect = source?.bounds + ?? CGRect(x: origin.bounds.midX, y: origin.bounds.maxY, width: 0, height: 0) + presenter.present(activity, animated: true) +} diff --git a/apps/mobile/modules/t3-native-controls/ios/T3NativeVideoPresentation.swift b/apps/mobile/modules/t3-native-controls/ios/T3NativeVideoPresentation.swift new file mode 100644 index 000000000..74d2f1c75 --- /dev/null +++ b/apps/mobile/modules/t3-native-controls/ios/T3NativeVideoPresentation.swift @@ -0,0 +1,167 @@ +import AVKit +import UIKit + +final class T3NativeVideoPresentation: NSObject, AVPlayerViewControllerDelegate, + UIAdaptivePresentationControllerDelegate { + let identifier: String + private let controller = AVPlayerViewController() + private let completion: (Error?) -> Void + private var itemObservation: NSKeyValueObservation? + private var backgroundObserver: NSObjectProtocol? + private var playbackError: Error? + private var presented = false + private var dismissRequested = false + private var finished = false + private struct AudioSessionConfiguration { + let category: AVAudioSession.Category + let mode: AVAudioSession.Mode + let options: AVAudioSession.CategoryOptions + + init(_ session: AVAudioSession) { + category = session.category + mode = session.mode + options = session.categoryOptions + } + } + private var previousAudioSession: AudioSessionConfiguration? + private weak var fullScreenController: UIViewController? + private var embedded = false + + init(identifier: String, url: URL, title: String, completion: @escaping (Error?) -> Void) { + self.identifier = identifier + self.completion = completion + super.init() + + let item = AVPlayerItem(url: url) + let metadata = AVMutableMetadataItem() + metadata.identifier = .commonIdentifierTitle + metadata.value = title as NSString + item.externalMetadata = [metadata] + controller.player = AVPlayer(playerItem: item) + controller.delegate = self + controller.overrideUserInterfaceStyle = .dark + controller.allowsPictureInPicturePlayback = false + + itemObservation = item.observe(\.status, options: [.initial, .new]) { [weak self] item, _ in + guard item.status == .failed else { return } + DispatchQueue.main.async { + guard let self else { return } + self.playbackError = item.error ?? NSError( + domain: "T3NativeVideo", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "This video couldn't be played on this device."] + ) + self.dismiss() + } + } + backgroundObserver = NotificationCenter.default.addObserver( + forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: .main + ) { [weak self] _ in self?.controller.player?.pause() } + } + + func present(from presenter: UIViewController, sources: T3PresentationSources, sourceIdentifier: String) { + let audioSession = AVAudioSession.sharedInstance() + previousAudioSession = AudioSessionConfiguration(audioSession) + do { + try audioSession.setCategory(.playback, mode: .moviePlayback) + } catch { + NSLog("T3 video audio session: %@", error.localizedDescription) + } + // AVKit exposes programmatic inline-to-full-screen entry through this selector. + // This is the same guarded entry point used by expo-video's enterFullscreen(). + let enterFullScreen = NSSelectorFromString("enterFullScreenAnimated:completionHandler:") + if let source = sources.view(for: sourceIdentifier), source.window != nil, + controller.responds(to: enterFullScreen) { + // AVKit owns the transition from its inline view to full screen. Using a + // separate UIKit zoom transition prevents its native Close action from exiting. + var responder: UIResponder? = source + while let current = responder, !(current is UIViewController) { responder = current.next } + let parent = responder as? UIViewController ?? presenter + embedded = true + parent.addChild(controller) + controller.view.frame = source.bounds + controller.view.autoresizingMask = [.flexibleWidth, .flexibleHeight] + source.addSubview(controller.view) + controller.didMove(toParent: parent) + controller.view.layoutIfNeeded() + controller.perform(enterFullScreen, with: true, with: nil) + controller.player?.play() + } else { + presenter.present(controller, animated: true) { [self] in + presented = true + if dismissRequested { + dismiss() + } else if UIApplication.shared.applicationState == .active { + controller.player?.play() + } + } + controller.presentationController?.delegate = self + } + } + + func dismiss() { + dismissRequested = true + guard !finished else { return } + guard presented else { + if embedded && fullScreenController == nil { finish() } + return + } + (fullScreenController ?? controller).dismiss(animated: true) { [self] in finish() } + } + + func playerViewController( + _ playerViewController: AVPlayerViewController, + willBeginFullScreenPresentationWithAnimationCoordinator coordinator: UIViewControllerTransitionCoordinator + ) { + fullScreenController = coordinator.viewController(forKey: .to) + coordinator.animate(alongsideTransition: nil) { [weak self] context in + guard let self else { return } + if context.isCancelled { + finish() + } else { + presented = true + if dismissRequested { dismiss() } + } + } + } + + func playerViewController( + _ playerViewController: AVPlayerViewController, + willEndFullScreenPresentationWithAnimationCoordinator coordinator: UIViewControllerTransitionCoordinator + ) { + coordinator.animate(alongsideTransition: nil) { [weak self] context in + if !context.isCancelled { self?.finish() } + } + } + + func presentationControllerDidDismiss(_ presentationController: UIPresentationController) { + finish() + } + + private func finish() { + guard !finished else { return } + finished = true + controller.player?.pause() + if embedded { + controller.willMove(toParent: nil) + controller.view.removeFromSuperview() + controller.removeFromParent() + } + itemObservation = nil + controller.player = nil + if let backgroundObserver { NotificationCenter.default.removeObserver(backgroundObserver) } + backgroundObserver = nil + let audioSession = AVAudioSession.sharedInstance() + if let previousAudioSession, audioSession.category == .playback, + audioSession.mode == .moviePlayback, audioSession.categoryOptions.isEmpty { + // AVPlayer owns activation. Deactivating the shared session here could + // stop another player or recorder that was active before this preview. + try? audioSession.setCategory( + previousAudioSession.category, + mode: previousAudioSession.mode, + options: previousAudioSession.options + ) + } + completion(playbackError) + } +} diff --git a/apps/mobile/package.json b/apps/mobile/package.json index c139ae6a5..57d8b4a2d 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -84,6 +84,7 @@ "expo-crypto": "~57.0.2", "expo-dev-client": "~57.0.16", "expo-device": "~57.0.1", + "expo-document-picker": "~57.0.1", "expo-file-system": "~57.0.6", "expo-font": "~57.0.2", "expo-glass-effect": "~57.0.1", @@ -101,6 +102,7 @@ "expo-sqlite": "~57.0.2", "expo-symbols": "~57.0.2", "expo-updates": "~57.0.19", + "expo-video": "~57.0.3", "expo-web-browser": "~57.0.2", "expo-widgets": "~57.0.15", "punycode": "^2.3.1", diff --git a/apps/mobile/src/Stack.tsx b/apps/mobile/src/Stack.tsx index c1f2cef48..5f407726d 100644 --- a/apps/mobile/src/Stack.tsx +++ b/apps/mobile/src/Stack.tsx @@ -73,6 +73,7 @@ import { NATIVE_LIQUID_GLASS_SUPPORTED } from "./native/native-glass"; import { nativeHeaderScrollEdgeEffects } from "./native/StackHeader"; import { FORM_SHEET_PRESENTATION_OPTIONS } from "./native/sheet-surface"; import { useThreadOutboxDrain } from "./state/use-thread-outbox-drain"; +import { useComposerAttachmentUploadWorker } from "./state/composer-attachment-uploads"; const HEADER_SCROLL_EDGE_EFFECTS = nativeHeaderScrollEdgeEffects(Platform.OS, Platform.Version); @@ -355,6 +356,7 @@ function workspacePathFromState(state: NavigationState): string { // each enqueue, shell change, or reconnect. function ThreadOutboxDrainWorker() { useThreadOutboxDrain(); + useComposerAttachmentUploadWorker(); return null; } diff --git a/apps/mobile/src/components/ComposerAttachmentStrip.tsx b/apps/mobile/src/components/ComposerAttachmentStrip.tsx index 2f3f6cbe4..ee89f940a 100644 --- a/apps/mobile/src/components/ComposerAttachmentStrip.tsx +++ b/apps/mobile/src/components/ComposerAttachmentStrip.tsx @@ -1,16 +1,33 @@ import { SymbolView } from "../components/AppSymbol"; -import { Image, Pressable, ScrollView, View } from "react-native"; +import { videoMimeType } from "@t3tools/shared/video"; +import { useEffect, useRef, useState } from "react"; +import { Alert, Image, Pressable, ScrollView, View } from "react-native"; import { AppText as Text } from "./AppText"; -import type { DraftComposerAttachment } from "../lib/composerImages"; +import type { DraftComposerAttachment, DraftComposerFileAttachment } from "../lib/composerImages"; +import { VideoAttachmentTile } from "./VideoAttachmentTile"; +import { loadLocalAttachmentPreview } from "../lib/localAttachmentPreview"; +import { PresentationSource } from "./NativePresentation"; +import type { FilePreviewSource } from "./FilePreviewModal"; +import { isPdfFile } from "../lib/filePreview"; +import type { EnvironmentId } from "@t3tools/contracts"; +import { + retryComposerAttachmentUpload, + useComposerAttachmentUploadState, +} from "../state/composer-attachment-uploads"; export interface ComposerAttachmentStripProps { + readonly environmentId?: EnvironmentId; /** Attachments to display. */ readonly attachments: ReadonlyArray; /** Called when the user removes an attachment. */ readonly onRemove: (imageId: string) => void; - /** Called when the user taps on an image thumbnail to preview it. */ - readonly onPressImage?: (previewUri: string) => void; + /** Called when the user taps an image or PDF to preview it. */ + readonly onPressPreview?: (source: FilePreviewSource) => void; + readonly onPressVideo?: ( + attachment: DraftComposerFileAttachment, + sourceIdentifier: string, + ) => void; /** Image thumbnail size in points. Defaults to 72. */ readonly imageSize?: number; /** Border radius of each image thumbnail. Defaults to 16. */ @@ -19,6 +36,199 @@ export interface ComposerAttachmentStripProps { readonly removeButtonPlacement?: "overlay" | "gutter"; } +type ComposerAttachmentThumbnailProps = { + readonly environmentId?: EnvironmentId; + readonly attachment: DraftComposerAttachment; + readonly size: number; + readonly borderRadius: number; + readonly compact?: boolean; + readonly onPressPreview?: (source: FilePreviewSource) => void; + readonly onPressVideo?: ( + attachment: DraftComposerFileAttachment, + sourceIdentifier: string, + ) => void; +}; + +export function ComposerAttachmentThumbnail(props: ComposerAttachmentThumbnailProps) { + const upload = useComposerAttachmentUploadState(props.environmentId, props.attachment.id); + return ( + + + {upload && upload.status !== "ready" ? ( + + props.environmentId && + retryComposerAttachmentUpload(props.environmentId, props.attachment.id) + } + className="absolute bottom-0.5 left-0.5 flex-row items-center gap-0.5 rounded-full bg-black/70 px-1 py-0.5" + > + + {!props.compact ? ( + + {upload.status === "failed" ? "Retry" : `${Math.floor(upload.progress * 100)}%`} + + ) : null} + + ) : null} + + ); +} + +function ComposerAttachmentContent(props: ComposerAttachmentThumbnailProps) { + const { attachment } = props; + const style = { width: props.size, height: props.size, borderRadius: props.borderRadius }; + if (attachment.type === "image") { + const sourceIdentifier = `draft-image:${attachment.id}`; + return ( + + + props.onPressPreview?.({ + kind: "image", + uri: attachment.dataUrl, + name: attachment.name, + sourceIdentifier, + }) + } + > + + + + ); + } + const onPressVideo = props.onPressVideo; + if (onPressVideo && videoMimeType(attachment) !== null) { + return ( + + ); + } + const canPreview = isPdfFile(attachment) && props.onPressPreview !== undefined; + const sourceIdentifier = `draft-file:${attachment.id}`; + return ( + + + props.onPressPreview?.({ + kind: "pdf", + name: attachment.name, + attachment, + sourceIdentifier, + }) + } + className={ + props.compact + ? "items-center justify-center bg-subtle" + : "items-center justify-center gap-1 bg-subtle px-2" + } + style={style} + > + + {!props.compact ? ( + + {attachment.name} + + ) : null} + + + ); +} + +function ComposerVideoAttachment(props: { + readonly attachment: DraftComposerFileAttachment; + readonly size: number; + readonly borderRadius: number; + readonly compact?: boolean; + readonly onPressVideo: ( + attachment: DraftComposerFileAttachment, + sourceIdentifier: string, + ) => void; +}) { + const { attachment } = props; + const sourceIdentifier = `draft:${attachment.id}`; + const style = { width: props.size, height: props.size, borderRadius: props.borderRadius }; + const shareRef = useRef(null); + const [sharing, setSharing] = useState(false); + useEffect( + () => () => { + shareRef.current?.abort(); + shareRef.current = null; + }, + [], + ); + + const onShare = () => { + if (shareRef.current) return; + const controller = new AbortController(); + shareRef.current = controller; + setSharing(true); + void (async () => { + const preview = await loadLocalAttachmentPreview(attachment, controller.signal); + if (!preview) return; + try { + await preview.share(controller.signal, sourceIdentifier); + } finally { + preview.dispose(); + } + })() + .catch((error: unknown) => { + if (!controller.signal.aborted) { + Alert.alert( + "Could not share video", + error instanceof Error ? error.message : "Try again.", + ); + } + }) + .finally(() => { + if (shareRef.current === controller) { + shareRef.current = null; + setSharing(false); + } + }); + }; + + return ( + props.onPressVideo(attachment, sourceIdentifier)} + onShare={onShare} + disabled={sharing} + style={style} + /> + ); +} + /** * Attachment thumbnails used by the thread composer and the new-task draft screen. */ @@ -49,43 +259,14 @@ export function ComposerAttachmentStrip(props: ComposerAttachmentStripProps) { paddingRight: removeButtonGutter, }} > - {attachment.type === "image" ? ( - props.onPressImage!(attachment.previewUri) : undefined - } - > - - - ) : ( - - - - {attachment.name} - - - )} + ; + dismissFile(identifier: string): Promise; +}>("T3NativeControls"); + +export function FilePreview(props: { + readonly source: ResolvedFilePreviewSource; + readonly onRequestClose: () => void; +}) { + const { uri, name, sourceIdentifier } = props.source; + const identifier = useId(); + const onRequestClose = useEffectEvent(props.onRequestClose); + + useEffect(() => { + let canceled = false; + void NativeControls.presentFile(uri, name ?? "Preview", sourceIdentifier ?? "", identifier) + .catch(() => { + if (!canceled) { + Alert.alert("Could not open preview", "The file could not be loaded. Please try again."); + } + }) + .finally(() => { + if (!canceled) onRequestClose(); + }); + return () => { + canceled = true; + void NativeControls.dismissFile(identifier).catch(() => undefined); + }; + }, [uri, name, sourceIdentifier, identifier]); + + return null; +} diff --git a/apps/mobile/src/components/FilePreview.tsx b/apps/mobile/src/components/FilePreview.tsx new file mode 100644 index 000000000..f10bfb8b3 --- /dev/null +++ b/apps/mobile/src/components/FilePreview.tsx @@ -0,0 +1,52 @@ +import { useEffect, useEffectEvent } from "react"; +import { Alert } from "react-native"; +import ImageViewing from "react-native-image-viewing"; + +import { downloadAndShareAttachment, shareLocalAttachment } from "../lib/attachmentDownload"; +import type { ResolvedFilePreviewSource } from "./FilePreviewModal"; + +function PdfPreview(props: { + readonly source: ResolvedFilePreviewSource; + readonly onRequestClose: () => void; +}) { + const { uri, name } = props.source; + const onRequestClose = useEffectEvent(props.onRequestClose); + useEffect(() => { + const controller = new AbortController(); + const input = { + attachment: { name: name ?? "Document.pdf", mimeType: "application/pdf" }, + signal: controller.signal, + }; + // Android's system chooser supplies the installed PDF apps. + const opened = + uri.startsWith("file:") || uri.startsWith("content:") + ? shareLocalAttachment({ ...input, uri }) + : downloadAndShareAttachment({ ...input, url: uri }); + void opened + .catch(() => { + if (!controller.signal.aborted) Alert.alert("Could not open PDF", "Please try again."); + }) + .finally(() => { + if (!controller.signal.aborted) onRequestClose(); + }); + return () => controller.abort(); + }, [uri, name]); + return null; +} + +export function FilePreview(props: { + readonly source: ResolvedFilePreviewSource; + readonly onRequestClose: () => void; +}) { + if (props.source.kind === "pdf") return ; + return ( + + ); +} diff --git a/apps/mobile/src/components/FilePreviewModal.tsx b/apps/mobile/src/components/FilePreviewModal.tsx new file mode 100644 index 000000000..c9df7e892 --- /dev/null +++ b/apps/mobile/src/components/FilePreviewModal.tsx @@ -0,0 +1,93 @@ +import { useIsFocused } from "@react-navigation/native"; +import type { AssetResource, EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useEffectEvent, useState } from "react"; +import { Alert, Keyboard } from "react-native"; + +import type { DraftComposerFileAttachment } from "../lib/composerImages"; +import { loadLocalAttachmentPreview } from "../lib/localAttachmentPreview"; +import { useAssetUrlState } from "../state/assets"; +import { usePreparedConnection } from "../state/session"; +import { FilePreview } from "./FilePreview"; + +export interface ResolvedFilePreviewSource { + readonly kind: "image" | "pdf"; + readonly uri: string; + readonly name?: string; + readonly sourceIdentifier?: string; +} + +export type FilePreviewSource = Omit & + ( + | { readonly uri: string } + | { readonly attachment: DraftComposerFileAttachment } + | { readonly environmentId: EnvironmentId; readonly resource: AssetResource } + ); + +function ResolvedFilePreview(props: { + readonly source: FilePreviewSource; + readonly onRequestClose: () => void; +}) { + const { source } = props; + const environmentId = "environmentId" in source ? source.environmentId : null; + const connection = usePreparedConnection(environmentId); + const asset = useAssetUrlState(environmentId, "resource" in source ? source.resource : null); + // Keep the original URL through dismissal; a refreshed signature must not reopen the viewer. + const [uri, setUri] = useState("uri" in source ? source.uri : null); + const onRequestClose = useEffectEvent(props.onRequestClose); + const failed = + environmentId !== null && + uri === null && + (connection._tag === "None" || asset._tag === "Failure"); + useEffect(() => Keyboard.dismiss(), []); + useEffect(() => { + if (uri === null && asset._tag === "Success") setUri(asset.url); + }, [uri, asset]); + useEffect(() => { + if (!failed) return; + Alert.alert("Could not open preview", "Reconnect to this environment and try again."); + onRequestClose(); + }, [failed]); + useEffect(() => { + if (!("attachment" in source)) return; + const controller = new AbortController(); + let release: (() => void) | undefined; + void loadLocalAttachmentPreview(source.attachment, controller.signal) + .then((file) => { + if (!file) return; + if (controller.signal.aborted) { + file.dispose(); + return; + } + release = file.dispose; + setUri(file.uri); + }) + .catch(() => { + if (controller.signal.aborted) return; + Alert.alert("Could not open preview", "Attach the file again and retry."); + onRequestClose(); + }); + return () => { + controller.abort(); + release?.(); + }; + }, [source]); + + return uri === null ? null : ( + + ); +} + +export function FilePreviewModal(props: { + readonly source: FilePreviewSource | null; + readonly onRequestClose: () => void; +}) { + const isFocused = useIsFocused(); + const hasSource = props.source !== null; + const onRequestClose = useEffectEvent(props.onRequestClose); + useEffect(() => { + if (!isFocused && hasSource) onRequestClose(); + }, [isFocused, hasSource]); + + if (!props.source || !isFocused) return null; + return ; +} diff --git a/apps/mobile/src/components/NativePresentation.ios.tsx b/apps/mobile/src/components/NativePresentation.ios.tsx new file mode 100644 index 000000000..b93578dde --- /dev/null +++ b/apps/mobile/src/components/NativePresentation.ios.tsx @@ -0,0 +1,12 @@ +import { requireNativeView } from "expo"; +import type { ComponentType } from "react"; +import type { PresentationSourceProps } from "./NativePresentation"; + +const NativeSource: ComponentType = requireNativeView( + "T3NativeControls", + "PresentationSource", +); + +export function PresentationSource(props: PresentationSourceProps) { + return ; +} diff --git a/apps/mobile/src/components/NativePresentation.tsx b/apps/mobile/src/components/NativePresentation.tsx new file mode 100644 index 000000000..d48b88395 --- /dev/null +++ b/apps/mobile/src/components/NativePresentation.tsx @@ -0,0 +1,13 @@ +import type { ReactElement } from "react"; +import { View, type ViewProps } from "react-native"; + +export interface PresentationSourceProps extends ViewProps { + readonly children: ReactElement; + /** Stable across remounts so dismissal can find a recycled attachment thumbnail. */ + readonly identifier: string; +} + +/** Registers the view as an iOS zoom or share-sheet origin. */ +export function PresentationSource({ identifier: _identifier, ...props }: PresentationSourceProps) { + return ; +} diff --git a/apps/mobile/src/components/VideoAttachmentMenu.tsx b/apps/mobile/src/components/VideoAttachmentMenu.tsx new file mode 100644 index 000000000..301d6503a --- /dev/null +++ b/apps/mobile/src/components/VideoAttachmentMenu.tsx @@ -0,0 +1,53 @@ +import type { ReactElement } from "react"; +import { Platform, type PressableProps } from "react-native"; + +import { ControlPillMenu } from "./ControlPill"; +import { PresentationSource } from "./NativePresentation"; + +export function VideoAttachmentMenu(props: { + readonly sourceIdentifier: string; + readonly onOpen: () => void; + readonly onShare?: () => void; + readonly disabled?: boolean; + readonly children: ReactElement; +}) { + return ( + { + if (!props.disabled) props.onOpen(); + }} + accessibilityActions={props.onShare ? [{ name: "share", label: "Save or share video" }] : []} + onAccessibilityAction={({ nativeEvent }) => { + if (nativeEvent.actionName === "share" && !props.disabled) props.onShare?.(); + }} + > + {Platform.OS === "ios" && props.onShare ? ( + { + if (nativeEvent.event === "share") props.onShare?.(); + }} + > + {props.children} + + ) : ( + props.children + )} + + ); +} diff --git a/apps/mobile/src/components/VideoAttachmentTile.tsx b/apps/mobile/src/components/VideoAttachmentTile.tsx new file mode 100644 index 000000000..6f582ac5f --- /dev/null +++ b/apps/mobile/src/components/VideoAttachmentTile.tsx @@ -0,0 +1,66 @@ +import { Platform, Pressable, View, type StyleProp, type ViewStyle } from "react-native"; + +import { cn } from "../lib/cn"; +import type { DraftComposerFileAttachment } from "../lib/composerImages"; +import { SymbolView } from "./AppSymbol"; +import { AppText } from "./AppText"; +import { VideoAttachmentMenu } from "./VideoAttachmentMenu"; +import { VideoThumbnailImage } from "./VideoThumbnailImage"; + +export function VideoAttachmentTile(props: { + readonly name: string; + readonly sourceIdentifier: string; + readonly thumbnailSource: string | DraftComposerFileAttachment | null; + readonly compact?: boolean; + readonly onPress: (sourceIdentifier: string) => void; + readonly onShare?: () => void; + readonly disabled?: boolean; + readonly className?: string; + readonly style?: StyleProp; +}) { + return ( + props.onPress(props.sourceIdentifier)} + onShare={props.onShare} + disabled={props.disabled} + > + props.onPress(props.sourceIdentifier)} + className={cn("items-center justify-center overflow-hidden bg-black/80", props.className)} + style={props.style} + > + + + + + {!props.compact ? ( + + + {props.name} + + + ) : null} + + + ); +} diff --git a/apps/mobile/src/components/VideoPreviewModal.ios.tsx b/apps/mobile/src/components/VideoPreviewModal.ios.tsx new file mode 100644 index 000000000..a947d8d2e --- /dev/null +++ b/apps/mobile/src/components/VideoPreviewModal.ios.tsx @@ -0,0 +1,121 @@ +import { useIsFocused } from "@react-navigation/native"; +import { videoMimeType } from "@t3tools/shared/video"; +import { requireNativeModule } from "expo"; +import { useEffect, useEffectEvent, useId, useState } from "react"; +import { Alert, Keyboard } from "react-native"; + +import { loadLocalAttachmentPreview } from "../lib/localAttachmentPreview"; +import { useAssetUrlState } from "../state/assets"; +import { usePreparedConnection } from "../state/session"; +import type { VideoPreviewSource } from "./VideoPreviewModal"; + +export type { VideoPreviewSource } from "./VideoPreviewModal"; + +const NativeControls = requireNativeModule<{ + presentVideo( + uri: string, + title: string, + sourceIdentifier: string, + identifier: string, + ): Promise; + dismissVideo(identifier: string): Promise; +}>("T3NativeControls"); + +function NativeVideoPreview(props: { + readonly source: VideoPreviewSource; + readonly onRequestClose: () => void; +}) { + const { source } = props; + const { attachment } = source; + const identifier = useId(); + const onRequestClose = useEffectEvent(props.onRequestClose); + const environmentId = source.type === "remote" ? source.environmentId : null; + const preparedConnection = usePreparedConnection(environmentId); + const mimeType = videoMimeType(attachment) ?? attachment.mimeType; + const assetUrl = useAssetUrlState( + environmentId, + source.type === "remote" + ? { _tag: "attachment", attachmentId: attachment.id, fileName: attachment.name, mimeType } + : null, + ); + const [playbackUrl, setPlaybackUrl] = useState(() => + assetUrl._tag === "Success" ? assetUrl.url : null, + ); + const loadError = + source.type === "remote" && playbackUrl === null + ? preparedConnection._tag === "None" + ? "Reconnect to this environment and open the video again." + : assetUrl._tag === "Failure" + ? "Could not load this video. Check the connection and try again." + : null + : null; + + useEffect(() => Keyboard.dismiss(), []); + useEffect(() => { + if (playbackUrl === null && assetUrl._tag === "Success") setPlaybackUrl(assetUrl.url); + }, [playbackUrl, assetUrl]); + useEffect(() => { + if (!loadError) return; + Alert.alert("Could not open video", loadError); + onRequestClose(); + }, [loadError]); + + useEffect(() => { + if (source.type === "remote" && playbackUrl === null) return; + const controller = new AbortController(); + let ready = false; + void (async () => { + const file = + source.type === "local" + ? await loadLocalAttachmentPreview(source.attachment, controller.signal) + : null; + if (source.type === "local" && !file) return; + try { + if (controller.signal.aborted) return; + ready = true; + await NativeControls.presentVideo( + file?.uri ?? playbackUrl!, + attachment.name, + source.sourceIdentifier ?? "", + identifier, + ); + if (!controller.signal.aborted) onRequestClose(); + } finally { + // Native completion follows dismissal, so local playback keeps its file lease. + file?.dispose(); + } + })().catch((error: unknown) => { + if (controller.signal.aborted) return; + Alert.alert( + "Could not open video", + ready + ? "This video couldn't be loaded or played. Check the connection, or touch and hold the attachment to save or share the original." + : error instanceof Error + ? error.message + : "Could not load this video.", + ); + onRequestClose(); + }); + return () => { + controller.abort(); + void NativeControls.dismissVideo(identifier).catch(() => undefined); + }; + }, [source, attachment.name, playbackUrl, identifier]); + + return null; +} + +export function VideoPreviewModal(props: { + readonly source: VideoPreviewSource | null; + readonly onRequestClose: () => void; +}) { + const isFocused = useIsFocused(); + const hasSource = props.source !== null; + const onRequestClose = useEffectEvent(props.onRequestClose); + useEffect(() => { + if (!isFocused && hasSource) onRequestClose(); + }, [isFocused, hasSource]); + + if (!props.source || !isFocused) return null; + return ; +} diff --git a/apps/mobile/src/components/VideoPreviewModal.tsx b/apps/mobile/src/components/VideoPreviewModal.tsx new file mode 100644 index 000000000..eaa01c5d1 --- /dev/null +++ b/apps/mobile/src/components/VideoPreviewModal.tsx @@ -0,0 +1,258 @@ +import { useIsFocused } from "@react-navigation/native"; +import type { ChatFileAttachment, EnvironmentId } from "@t3tools/contracts"; +import { videoMimeType } from "@t3tools/shared/video"; +import { useEvent } from "expo"; +import { useVideoPlayer, VideoView } from "expo-video"; +import { useEffect, useRef, useState } from "react"; +import { + ActivityIndicator, + AppState, + Keyboard, + Modal, + Pressable, + StyleSheet, + View, +} from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; + +import { + downloadAttachmentForPreview, + type AttachmentPreviewFile, +} from "../lib/attachmentDownload"; +import type { DraftComposerFileAttachment } from "../lib/composerImages"; +import { loadLocalAttachmentPreview } from "../lib/localAttachmentPreview"; +import { useAssetUrlState } from "../state/assets"; +import { usePreparedConnection } from "../state/session"; +import { SymbolView } from "./AppSymbol"; +import { AppText } from "./AppText"; + +export type VideoPreviewSource = ( + | { readonly type: "local"; readonly attachment: DraftComposerFileAttachment } + | { + readonly type: "remote"; + readonly environmentId: EnvironmentId; + readonly attachment: ChatFileAttachment; + } +) & { readonly sourceIdentifier?: string }; + +function VideoPlayback(props: { readonly file: AttachmentPreviewFile }) { + const player = useVideoPlayer(props.file.uri, (player) => { + player.staysActiveInBackground = false; + if (AppState.currentState === "active") player.play(); + }); + const { status } = useEvent(player, "statusChange", { status: player.status }); + const shareControllerRef = useRef(null); + const [sharing, setSharing] = useState(false); + const [shareError, setShareError] = useState(null); + + useEffect( + () => () => { + shareControllerRef.current?.abort(); + shareControllerRef.current = null; + }, + [], + ); + + const onShare = () => { + if (shareControllerRef.current) return; + player.pause(); + const controller = new AbortController(); + shareControllerRef.current = controller; + setSharing(true); + setShareError(null); + void props.file + .share(controller.signal) + .catch((error: unknown) => { + if (!controller.signal.aborted) { + setShareError(error instanceof Error ? error.message : "Could not share this video."); + } + }) + .finally(() => { + if (shareControllerRef.current === controller) { + shareControllerRef.current = null; + setSharing(false); + } + }); + }; + + return ( + <> + + {status === "error" ? ( + + This video couldn't be played on this device. You can save or share the original file. + + ) : ( + <> + + {status === "loading" ? ( + + ) : null} + + )} + + + + {sharing ? "Opening share sheet..." : "Save or share video"} + + + {shareError ? ( + + {shareError} + + ) : null} + + ); +} + +function OpenVideoPreviewModal(props: { + readonly source: VideoPreviewSource; + readonly onRequestClose: () => void; +}) { + const { source } = props; + const { attachment } = source; + const insets = useSafeAreaInsets(); + const environmentId = source.type === "remote" ? source.environmentId : null; + const preparedConnection = usePreparedConnection(environmentId); + const fileUri = source.type === "local" ? source.attachment.fileUri : null; + const mimeType = videoMimeType(attachment) ?? attachment.mimeType; + const assetUrl = useAssetUrlState( + environmentId, + source.type === "remote" + ? { _tag: "attachment", attachmentId: attachment.id, fileName: attachment.name, mimeType } + : null, + ); + const [downloadUrl, setDownloadUrl] = useState(null); + const [file, setFile] = useState(null); + const [failure, setFailure] = useState(null); + + useEffect(() => Keyboard.dismiss(), []); + useEffect(() => { + if (environmentId !== null && downloadUrl === null && assetUrl._tag === "Success") { + setDownloadUrl(assetUrl.url); + } + }, [environmentId, downloadUrl, assetUrl]); + + useEffect(() => { + if (source.type === "remote" && downloadUrl === null) return; + const controller = new AbortController(); + let preview: AttachmentPreviewFile | null = null; + setFile(null); + setFailure(null); + const loading = + source.type === "local" + ? loadLocalAttachmentPreview(source.attachment, controller.signal) + : downloadAttachmentForPreview({ + url: downloadUrl!, + attachment: { name: attachment.name, mimeType }, + signal: controller.signal, + }); + void loading.then( + (loaded) => { + if (controller.signal.aborted) { + loaded?.dispose(); + return; + } + preview = loaded; + setFile(loaded); + }, + (error: unknown) => { + if (!controller.signal.aborted) { + setFailure(error instanceof Error ? error.message : "Could not load this video."); + } + }, + ); + return () => { + controller.abort(); + preview?.dispose(); + }; + }, [source.type, environmentId, attachment.id, attachment.name, mimeType, fileUri, downloadUrl]); + + const loadError = + failure ?? + (environmentId !== null && downloadUrl === null + ? preparedConnection._tag === "None" + ? "This environment is disconnected. Reconnect and open the video again." + : assetUrl._tag === "Failure" + ? "Could not load this video. Check the connection to this environment and try again." + : null + : null); + + return ( + + + + + {attachment.name} + + + + + + {file ? ( + + ) : ( + + {loadError ? ( + + {loadError} + + ) : ( + <> + + Loading video... + + )} + + )} + + + ); +} + +export function VideoPreviewModal(props: { + readonly source: VideoPreviewSource | null; + readonly onRequestClose: () => void; +}) { + const isFocused = useIsFocused(); + const hasSource = props.source !== null; + useEffect(() => { + if (!isFocused && hasSource) props.onRequestClose(); + }, [isFocused, hasSource, props.onRequestClose]); + const { source } = props; + if (source === null || !isFocused) return null; + const key = + source.type === "local" + ? `local:${source.attachment.id}:${source.attachment.fileUri}` + : `remote:${source.environmentId}:${source.attachment.id}`; + return ; +} diff --git a/apps/mobile/src/components/VideoThumbnailImage.tsx b/apps/mobile/src/components/VideoThumbnailImage.tsx new file mode 100644 index 000000000..0be94c700 --- /dev/null +++ b/apps/mobile/src/components/VideoThumbnailImage.tsx @@ -0,0 +1,45 @@ +import { Image } from "expo-image"; +import { useIsFocused } from "@react-navigation/native"; +import type { VideoThumbnail } from "expo-video"; +import { useEffect, useState } from "react"; +import { StyleSheet } from "react-native"; + +import type { DraftComposerFileAttachment } from "../lib/composerImages"; +import { loadLocalAttachmentPreview } from "../lib/localAttachmentPreview"; +import { cachedVideoThumbnail, loadVideoThumbnail } from "../lib/videoThumbnails"; + +export function VideoThumbnailImage(props: { + readonly cacheKey: string; + readonly source: string | DraftComposerFileAttachment | null; +}) { + const { cacheKey, source } = props; + const isFocused = useIsFocused(); + const [loaded, setLoaded] = useState<{ key: string; thumbnail: VideoThumbnail } | null>(null); + const thumbnail = loaded?.key === cacheKey ? loaded.thumbnail : cachedVideoThumbnail(cacheKey); + + useEffect(() => { + if (!source || !isFocused) return; + const controller = new AbortController(); + void loadVideoThumbnail( + cacheKey, + async (signal) => + typeof source === "string" + ? { uri: source, dispose: () => undefined } + : loadLocalAttachmentPreview(source, signal), + controller.signal, + ).then((thumbnail) => { + if (thumbnail && !controller.signal.aborted) setLoaded({ key: cacheKey, thumbnail }); + }); + return () => controller.abort(); + }, [cacheKey, source, isFocused]); + + return thumbnail ? ( + + ) : null; +} diff --git a/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts b/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts index 2bc62d2a3..5fe74f673 100644 --- a/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts +++ b/apps/mobile/src/features/cloud/CloudAuthProvider.test.ts @@ -26,6 +26,12 @@ vi.mock("../../connection/catalog", () => ({ }, })); +vi.mock("./cloud-drafts", () => ({ removeCloudEnvironments: {} })); +vi.mock("../../state/use-composer-drafts", () => ({ + getComposerCloudAccountId: vi.fn(async () => null), + restoreCloudComposerDrafts: vi.fn(async () => undefined), +})); + vi.mock("./publicConfig", () => ({ resolveCloudPublicConfig: vi.fn(() => ({ clerk: { publishableKey: null }, diff --git a/apps/mobile/src/features/cloud/CloudAuthProvider.tsx b/apps/mobile/src/features/cloud/CloudAuthProvider.tsx index cf92067b3..f8f034804 100644 --- a/apps/mobile/src/features/cloud/CloudAuthProvider.tsx +++ b/apps/mobile/src/features/cloud/CloudAuthProvider.tsx @@ -5,14 +5,18 @@ import { reportAtomCommandResult, settleAsyncResult, settlePromise, + squashAtomCommandFailure, } from "@t3tools/client-runtime/state/runtime"; import * as Effect from "effect/Effect"; import { type ReactNode, useEffect, useRef } from "react"; -import { environmentCatalog } from "../../connection/catalog"; import { runtime } from "../../lib/runtime"; import { appAtomRegistry } from "../../state/atom-registry"; import { useAtomCommand } from "../../state/use-atom-command"; +import { + getComposerCloudAccountId, + restoreCloudComposerDrafts, +} from "../../state/use-composer-drafts"; import { releaseAgentAwarenessRelayTokenProvider, setAgentAwarenessRelayTokenProvider, @@ -20,6 +24,7 @@ import { } from "../agent-awareness/remoteRegistration"; import { clearConnectOnboardingRequest, requestConnectOnboarding } from "./connectOnboarding"; import { resolveCloudPublicConfig, resolveRelayClerkTokenOptions } from "./publicConfig"; +import { removeCloudEnvironments } from "./cloud-drafts"; function resetManagedRelayTokenCache() { return settleAsyncResult(() => @@ -47,7 +52,7 @@ export function activateCloudRelayAccount( function CloudAuthBridge(props: { readonly children: ReactNode }) { const { getToken, isLoaded, isSignedIn, userId } = useAuth({ treatPendingAsSignedOut: false }); - const removeRelayEnvironments = useAtomCommand(environmentCatalog.removeRelayEnvironments, { + const removeRelayEnvironments = useAtomCommand(removeCloudEnvironments, { reportFailure: false, reportDefect: false, }); @@ -81,32 +86,37 @@ function CloudAuthBridge(props: { readonly children: ReactNode }) { clearConnectOnboardingRequest(); } - const queueAccountCleanup = ( + const cleanUpAccount = async ( previous: { readonly userId: string; readonly provider: () => Promise; } | null, + accountId: string | null, ) => { - const previousTransition = accountTransitionRef.current ?? Promise.resolve(); - accountTransitionRef.current = previousTransition.then(async () => { - const cleanup = [ - resetManagedRelayTokenCache(), - removeRelayEnvironments(), - ...(previous - ? [ - settleAsyncResult(() => - runtime.runPromiseExit( - unregisterAgentAwarenessDeviceForCurrentUser(previous.provider), - ), + const removal = await removeRelayEnvironments(accountId); + if (removal._tag !== "Success") throw squashAtomCommandFailure(removal); + const cleanup = [ + resetManagedRelayTokenCache(), + ...(previous + ? [ + settleAsyncResult(() => + runtime.runPromiseExit( + unregisterAgentAwarenessDeviceForCurrentUser(previous.provider), ), - ] - : []), - ]; - const results = await Promise.all(cleanup); - for (const result of results) { - reportAtomCommandResult(result, { label: "cloud account cleanup" }); - } - }); + ), + ] + : []), + ]; + const results = await Promise.all(cleanup); + for (const result of results) { + reportAtomCommandResult(result, { label: "cloud account cleanup" }); + } + }; + const queueAccountCleanup = (previous: typeof previousTokenProviderRef.current) => { + const previousTransition = accountTransitionRef.current ?? Promise.resolve(); + accountTransitionRef.current = previousTransition + .catch(() => {}) + .then(() => cleanUpAccount(previous, previousObservedAccount ?? null)); return accountTransitionRef.current; }; @@ -115,7 +125,9 @@ function CloudAuthBridge(props: { readonly children: ReactNode }) { previousTokenProviderRef.current = null; deactivateCloudRelayAccount(); if (previousObservedAccount !== null) { - void queueAccountCleanup(previous); + void settlePromise(() => queueAccountCleanup(previous)).then((result) => { + reportAtomCommandResult(result, { label: "cloud account cleanup" }); + }); } return; } @@ -133,13 +145,21 @@ function CloudAuthBridge(props: { readonly children: ReactNode }) { } }; const activateAfterTransition = (transition: Promise) => { - void (async () => { - const result = await settlePromise(async () => { - await transition; - activateSession(); - }); - reportAtomCommandResult(result, { label: "cloud account activation" }); + const activation = (async () => { + await transition; + if (cancelled) return; + const storedAccount = await getComposerCloudAccountId(); + if (storedAccount !== null && storedAccount !== userId) { + await cleanUpAccount(null, storedAccount); + } + if (cancelled) return; + await restoreCloudComposerDrafts(userId); + activateSession(); })(); + accountTransitionRef.current = activation; + void settlePromise(() => activation).then((result) => { + reportAtomCommandResult(result, { label: "cloud account activation" }); + }); }; if ( previousObservedAccount !== undefined && @@ -150,7 +170,9 @@ function CloudAuthBridge(props: { readonly children: ReactNode }) { deactivateCloudRelayAccount(); activateAfterTransition(queueAccountCleanup(previous)); } else { - activateAfterTransition(accountTransitionRef.current ?? Promise.resolve()); + // A failed disk write can be retried. The persisted account check above + // still requires cleanup before activating a different account. + activateAfterTransition((accountTransitionRef.current ?? Promise.resolve()).catch(() => {})); } return () => { diff --git a/apps/mobile/src/features/cloud/cloud-drafts.ts b/apps/mobile/src/features/cloud/cloud-drafts.ts new file mode 100644 index 000000000..bc41b2b41 --- /dev/null +++ b/apps/mobile/src/features/cloud/cloud-drafts.ts @@ -0,0 +1,46 @@ +import { EnvironmentRegistry } from "@t3tools/client-runtime/connection"; +import { createRuntimeCommand } from "@t3tools/client-runtime/state/runtime"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as SubscriptionRef from "effect/SubscriptionRef"; + +import { connectionAtomRuntime } from "../../connection/runtime"; +import { archiveCloudComposerDrafts } from "../../state/use-composer-drafts"; + +export class CloudDraftArchiveError extends Schema.TaggedErrorClass()( + "CloudDraftArchiveError", + { + environmentCount: Schema.Number, + hasAccountId: Schema.Boolean, + cause: Schema.Defect(), + }, +) { + override get message(): string { + return `Could not preserve local drafts for ${this.environmentCount} cloud environments before sign-out.`; + } +} + +export const removeCloudEnvironments = createRuntimeCommand(connectionAtomRuntime, { + label: "cloud:preserve-drafts-and-remove-environments", + execute: Effect.fn("removeCloudEnvironments")(function* (accountId: string | null) { + const registry = yield* EnvironmentRegistry; + const entries = yield* SubscriptionRef.get(registry.entries); + const environmentIds = new Set( + [...entries.values()] + .filter((entry) => entry.target._tag === "RelayConnectionTarget") + .map((entry) => entry.target.environmentId), + ); + // Credentials are already revoked. A failed backup must leave the local + // owners intact so a later sign-in can retry without losing their files. + yield* Effect.tryPromise({ + try: () => archiveCloudComposerDrafts(accountId, environmentIds), + catch: (cause) => + new CloudDraftArchiveError({ + environmentCount: environmentIds.size, + hasAccountId: accountId !== null, + cause, + }), + }); + yield* registry.removeRelayEnvironments(); + }), +}); diff --git a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx index 462d07532..5dddac1dd 100644 --- a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx +++ b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx @@ -17,9 +17,11 @@ import { SymbolView } from "../../components/AppSymbol"; import { AppText as Text, AppTextInput as TextInput } from "../../components/AppText"; import { ControlPillMenu } from "../../components/ControlPill"; import { EmptyState } from "../../components/EmptyState"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; import { LoadingScreen } from "../../components/LoadingScreen"; import { resolveFileSelectionNavigationAction } from "../../lib/adaptive-navigation"; import { copyTextWithHaptic } from "../../lib/copyTextWithHaptic"; +import { isPdfFile } from "../../lib/filePreview"; import { tryOpenExternalUrl } from "../../lib/openExternalUrl"; import { useUniwindTheme } from "../../lib/useUniwindTheme"; import { useThreadSelection } from "../../state/use-thread-selection"; @@ -487,6 +489,7 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { readonly mode: FileViewMode; } | null>(null); const [previewRevision, setPreviewRevision] = useState(0); + const [fullScreenPreview, setFullScreenPreview] = useState(null); const isBrowserFile = relativePath !== null && isBrowserPreviewFile(relativePath); const isImageFile = relativePath !== null && isImagePreviewFile(relativePath); const canPreview = @@ -586,6 +589,20 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { inline: false, onPress: () => copyTextWithHaptic(relativePath), } as const, + isPdfFile({ name: relativePath }) && previewUri !== null + ? ({ + id: "open-pdf", + title: "Open PDF", + icon: "arrow.up.left.and.arrow.down.right", + inline: false, + onPress: () => + setFullScreenPreview({ + kind: "pdf", + uri: previewUri, + name: basename(relativePath), + }), + } as const) + : null, isBrowserFile && typeof assetPreviewUri === "string" ? ({ id: "open-browser", @@ -605,7 +622,15 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { } as const) : null, ].filter((action) => action !== null); - }, [assetPreviewUri, canPreview, isBrowserFile, isImageFile, relativePath, resolvedActiveMode]); + }, [ + assetPreviewUri, + previewUri, + canPreview, + isBrowserFile, + isImageFile, + relativePath, + resolvedActiveMode, + ]); const androidFileMenuActions = useMemo( () => @@ -766,6 +791,10 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { truncated={fileData?.truncated ?? false} onRefresh={() => fileQuery.refresh()} /> + setFullScreenPreview(null)} + /> ); diff --git a/apps/mobile/src/features/files/WorkspaceFileImagePreview.tsx b/apps/mobile/src/features/files/WorkspaceFileImagePreview.tsx index 73eca66bf..e725c4d13 100644 --- a/apps/mobile/src/features/files/WorkspaceFileImagePreview.tsx +++ b/apps/mobile/src/features/files/WorkspaceFileImagePreview.tsx @@ -1,24 +1,25 @@ import { useAtomValue } from "@effect/atom-react"; -import { useMemo, useState } from "react"; +import { useId, useMemo, useState } from "react"; import { ActivityIndicator, Image, Pressable, View } from "react-native"; -import ImageViewing from "react-native-image-viewing"; import { AsyncResult } from "effect/unstable/reactivity"; import { AppText as Text } from "../../components/AppText"; import { EmptyState } from "../../components/EmptyState"; import { workspaceFileImageAtom } from "./workspace-file-image-cache"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; +import { PresentationSource } from "../../components/NativePresentation"; function ResolvedWorkspaceFileImagePreview(props: { readonly accessibilityLabel: string; readonly uri: string; }) { const [loadError, setLoadError] = useState(null); - const [fullScreenVisible, setFullScreenVisible] = useState(false); + const [preview, setPreview] = useState(null); + const sourceIdentifier = useId(); const imageSource = useMemo( () => ({ uri: props.uri, cache: "force-cache" as const }), [props.uri], ); - const fullScreenImages = useMemo(() => [imageSource], [imageSource]); return ( @@ -27,18 +28,27 @@ function ResolvedWorkspaceFileImagePreview(props: { accessibilityLabel={`Open full-screen preview of ${props.accessibilityLabel}`} disabled={loadError !== null} className="flex-1 p-4 active:bg-subtle-strong" - onPress={() => setFullScreenVisible(true)} + onPress={() => + setPreview({ + kind: "image", + uri: props.uri, + name: props.accessibilityLabel, + sourceIdentifier, + }) + } > - setLoadError(null)} - onError={(event) => { - setLoadError(event.nativeEvent.error || "The image could not be rendered."); - }} - /> + + setLoadError(null)} + onError={(event) => { + setLoadError(event.nativeEvent.error || "The image could not be rendered."); + }} + /> + {loadError !== null ? ( @@ -47,14 +57,7 @@ function ResolvedWorkspaceFileImagePreview(props: { ) : null} - setFullScreenVisible(false)} - swipeToCloseEnabled - doubleTapToZoomEnabled - /> + setPreview(null)} /> ); } diff --git a/apps/mobile/src/features/review/ReviewCommentComposerSheet.tsx b/apps/mobile/src/features/review/ReviewCommentComposerSheet.tsx index a798e5ebe..74ccc8cf0 100644 --- a/apps/mobile/src/features/review/ReviewCommentComposerSheet.tsx +++ b/apps/mobile/src/features/review/ReviewCommentComposerSheet.tsx @@ -5,7 +5,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { Platform, Pressable, ScrollView, View, useWindowDimensions } from "react-native"; import { KeyboardAvoidingView, KeyboardStickyView } from "react-native-keyboard-controller"; import { useSafeAreaInsets } from "react-native-safe-area-context"; -import ImageViewing from "react-native-image-viewing"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; import { AppText as Text, AppTextInput as TextInput } from "../../components/AppText"; import { SymbolView } from "../../components/AppSymbol"; @@ -53,7 +53,7 @@ export function ReviewCommentComposerSheet(props: ReviewCommentComposerSheetProp Record> >({}); const [attachments, setAttachments] = useState>([]); - const [previewImageUri, setPreviewImageUri] = useState(null); + const [previewFile, setPreviewFile] = useState(null); const selectedLines = useMemo( () => (target ? getSelectedReviewCommentLines(target) : []), @@ -272,7 +272,7 @@ export function ReviewCommentComposerSheet(props: ReviewCommentComposerSheetProp attachments={attachments} imageBorderRadius={16} imageSize={60} - onPressImage={setPreviewImageUri} + onPressPreview={setPreviewFile} removeButtonPlacement="gutter" onRemove={(imageId) => { setAttachments((current) => @@ -332,14 +332,7 @@ export function ReviewCommentComposerSheet(props: ReviewCommentComposerSheetProp ) : null} - setPreviewImageUri(null)} - swipeToCloseEnabled - doubleTapToZoomEnabled - /> + setPreviewFile(null)} /> ); } diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index 09939e4cb..ba610bac4 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -1,3 +1,4 @@ +import { useAtomValue } from "@effect/atom-react"; import { NativeHeaderToolbar, NativeStackScreenOptions } from "../../native/StackHeader"; import { CommonActions, @@ -36,6 +37,12 @@ import { import { AndroidScreenHeader } from "../../components/AndroidScreenHeader"; import { ComposerAttachmentButton } from "../../components/ComposerAttachmentButton"; import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStrip"; +import { + composerAttachmentUploadBlockReason, + composerAttachmentUploadsAtom, +} from "../../state/composer-attachment-uploads"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; +import { VideoPreviewModal, type VideoPreviewSource } from "../../components/VideoPreviewModal"; import { ProviderIcon } from "../../components/ProviderIcon"; import { SymbolView } from "../../components/AppSymbol"; import { AppText as Text } from "../../components/AppText"; @@ -62,6 +69,7 @@ import { convertPastedImagesToAttachments, pickComposerFiles, pickComposerMedia, + type DraftComposerFileAttachment, } from "../../lib/composerImages"; import { useScaledTextRole } from "../settings/appearance/useScaledTextRole"; import { @@ -172,9 +180,47 @@ export function NewTaskDraftScreen(props: { providerAdmissionReason === null ? null : { headline: "Unavailable" as const, detail: providerAdmissionReason }; + const uploadStates = useAtomValue(composerAttachmentUploadsAtom); + const attachmentBlockReason = selectedProject + ? composerAttachmentUploadBlockReason({ + environmentId: selectedProject.environmentId, + attachments: flow.attachments, + connected: environmentConnected, + serverConfig: selectedEnvironmentServerConfig, + states: uploadStates, + }) + : null; const promptInputRef = useRef(null); const loadedBranchesProjectKeyRef = useRef(null); const [isComposerFocused, setIsComposerFocused] = useState(false); + const [previewVideo, setPreviewVideo] = useState(null); + const [previewFile, setPreviewFile] = useState(null); + const wasFocusedBeforePreviewRef = useRef(false); + const openVideoPreview = useCallback( + (attachment: DraftComposerFileAttachment, sourceIdentifier: string) => { + wasFocusedBeforePreviewRef.current = isComposerFocused; + setPreviewFile(null); + setPreviewVideo((current) => current ?? { type: "local", attachment, sourceIdentifier }); + }, + [isComposerFocused], + ); + const openFilePreview = useCallback( + (source: FilePreviewSource) => { + wasFocusedBeforePreviewRef.current = isComposerFocused; + setPreviewVideo(null); + setPreviewFile((current) => current ?? source); + }, + [isComposerFocused], + ); + const closeMediaPreview = useCallback(() => { + setPreviewVideo(null); + setPreviewFile(null); + if (wasFocusedBeforePreviewRef.current) { + setTimeout(() => { + if (navigation.isFocused()) promptInputRef.current?.focus(); + }, 100); + } + }, [navigation]); const settingsSheetPresentation = useThreadSettingsSheetPresentation({ editorRef: promptInputRef, isEditorFocused: isComposerFocused, @@ -849,6 +895,7 @@ export function NewTaskDraftScreen(props: { const initialMessageText = draft.text.trim(); if ( + attachmentBlockReason !== null || !modelSelection || initialMessageText.length === 0 || flow.submitting || @@ -1020,6 +1067,7 @@ export function NewTaskDraftScreen(props: { const isAndroid = Platform.OS === "android"; const canStart = + attachmentBlockReason === null && Boolean(flow.selectedProject?.workspaceRoot?.trim()) && Boolean(flow.selectedModel) && providerUnavailable === null && @@ -1225,6 +1273,7 @@ export function NewTaskDraftScreen(props: { {flow.attachments.length > 0 ? ( undefined : flow.removeAttachment } + onPressPreview={ + isComposerInteractionLocked || voiceInput.isBusy ? undefined : openFilePreview + } + onPressVideo={ + isComposerInteractionLocked || voiceInput.isBusy ? undefined : openVideoPreview + } /> ) : null} @@ -1322,13 +1377,14 @@ export function NewTaskDraftScreen(props: { accessibilityLabel={ providerUnavailable ? `Start unavailable. ${providerUnavailable.detail}` - : flow.submitting - ? "Starting task" - : !canStart - ? "Start unavailable. Add a message and complete the task setup." - : environmentConnected - ? "Start task" - : "Queue task. The environment is disconnected; this task will remain queued." + : (attachmentBlockReason ?? + (flow.submitting + ? "Starting task" + : !canStart + ? "Start unavailable. Add a message and complete the task setup." + : environmentConnected + ? "Start task" + : "Queue task. The environment is disconnected; this task will remain queued.")) } disabled={!canStart} icon={environmentConnected ? "arrow.up" : "tray.and.arrow.up"} @@ -1340,6 +1396,8 @@ export function NewTaskDraftScreen(props: { + + ); diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index 0404ebcfb..192023c92 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -1,3 +1,4 @@ +import { useAtomValue } from "@effect/atom-react"; import type { ContextWindowSnapshot } from "@t3tools/client-runtime/state/context-window"; import { resolveProviderContinuationTransition } from "@t3tools/client-runtime/providerContinuation"; import { @@ -73,7 +74,6 @@ import { import { ActivityIndicator, Alert, - Image, KeyboardAvoidingView, Modal, Platform, @@ -82,7 +82,11 @@ import { View, type ViewStyle, } from "react-native"; -import ImageViewing from "react-native-image-viewing"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; +import { + composerAttachmentUploadBlockReason, + composerAttachmentUploadsAtom, +} from "../../state/composer-attachment-uploads"; import Animated, { FadeIn, FadeInDown, @@ -100,9 +104,12 @@ import { armAgentAwarenessLiveActivityForLocalWork } from "../agent-awareness/re import { scopedThreadKey } from "../../lib/scopedEntities"; import { AppText as Text, AppTextInput as TextInput } from "../../components/AppText"; -import { SymbolView } from "../../components/AppSymbol"; import { ComposerAttachmentButton } from "../../components/ComposerAttachmentButton"; -import { ComposerAttachmentStrip } from "../../components/ComposerAttachmentStrip"; +import { + ComposerAttachmentStrip, + ComposerAttachmentThumbnail, +} from "../../components/ComposerAttachmentStrip"; +import { VideoPreviewModal, type VideoPreviewSource } from "../../components/VideoPreviewModal"; import { GlassSurface } from "../../components/GlassSurface"; import { ComposerEditor, type ComposerEditorHandle } from "../../components/ComposerEditor"; import { @@ -113,7 +120,10 @@ import { } from "../../components/ComposerToolbar"; import { ControlPill, ControlPillMenu } from "../../components/ControlPill"; import { ProviderIcon } from "../../components/ProviderIcon"; -import type { DraftComposerAttachment } from "../../lib/composerImages"; +import type { + DraftComposerAttachment, + DraftComposerFileAttachment, +} from "../../lib/composerImages"; import { buildModelOptions, type ModelOption, @@ -491,7 +501,8 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer const inFlightThreadIdsRef = useRef(new Set()); const { onExpandedChange } = props; - const [previewImageUri, setPreviewImageUri] = useState(null); + const [previewFile, setPreviewFile] = useState(null); + const [previewVideo, setPreviewVideo] = useState(null); const hasContent = props.draftMessage.trim().length > 0 || props.draftAttachments.length > 0; // Opening and presentation count as active so the composer stays expanded // while focus moves between its native editor and the settings picker. @@ -504,20 +515,33 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer onExpandedChange?.(isExpanded); }, [isExpanded, onExpandedChange]); - const onPressImage = useCallback( - (uri: string) => { + const onPressPreview = useCallback( + (source: FilePreviewSource) => { wasExpandedBeforePreviewRef.current = isFocused; - setPreviewImageUri(uri); + setPreviewVideo(null); + setPreviewFile((current) => current ?? source); }, [isFocused], ); const closePreview = useCallback(() => { - setPreviewImageUri(null); + setPreviewFile(null); + setPreviewVideo(null); if (wasExpandedBeforePreviewRef.current) { - setTimeout(() => inputRef.current?.focus(), 100); + setTimeout(() => { + if (navigation.isFocused()) inputRef.current?.focus(); + }, 100); } - }, [inputRef]); + }, [inputRef, navigation]); + + const onPressVideo = useCallback( + (attachment: DraftComposerFileAttachment, sourceIdentifier: string) => { + wasExpandedBeforePreviewRef.current = isFocused; + setPreviewFile(null); + setPreviewVideo((current) => current ?? { type: "local", attachment, sourceIdentifier }); + }, + [isFocused], + ); const onEditorFocusChange = props.onEditorFocusChange; const handleFocus = useCallback(() => { @@ -572,10 +596,19 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer blockingAdmissionReason === null ? null : { headline: "Unavailable" as const, detail: blockingAdmissionReason }; + const uploadStates = useAtomValue(composerAttachmentUploadsAtom); + const attachmentBlockReason = composerAttachmentUploadBlockReason({ + environmentId: props.environmentId, + attachments: props.draftAttachments, + connected: props.connectionState === "connected", + serverConfig: props.serverConfig, + states: uploadStates, + }); const canSend = hasContent && composerAuthority.providerAdmissionAvailable && props.projectCwd !== null && + attachmentBlockReason === null && props.sessionCompactionPendingAction !== "compact" && !isSessionCompactionInProgress(props.sessionCompaction); const activeSessionProviderStatus = useMemo(() => { @@ -1664,9 +1697,11 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer layout={COMPOSER_LAYOUT_TRANSITION} > undefined : props.onRemoveDraftImage} - onPressImage={voiceInput.isBusy ? undefined : onPressImage} + onPressPreview={voiceInput.isBusy ? undefined : onPressPreview} + onPressVideo={voiceInput.isBusy ? undefined : onPressVideo} /> ) : null} @@ -1712,32 +1747,18 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer {!isExpanded && props.draftAttachments.length > 0 ? ( - {props.draftAttachments.slice(0, 3).map((attachment) => - attachment.type === "image" ? ( - onPressImage(attachment.previewUri)} - > - - - ) : ( - - - - ), - )} + {props.draftAttachments.slice(0, 3).map((attachment) => ( + + ))} {props.draftAttachments.length > 3 ? ( @@ -1776,7 +1797,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer /> {canQueueFollowUp ? ( ) : ( )} @@ -2012,7 +2033,7 @@ export const ThreadComposer = memo(function ThreadComposer(props: ThreadComposer the user with no way to send until they dismiss the error. */} {voicePresentation.showsSend ? ( - + + ); }); diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 8160537a9..c3a3abc95 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -24,6 +24,7 @@ import { splitCodexArtifactTemplateMarkdown, } from "@t3tools/client-runtime/codex-markdown-directives"; import { CHAT_LIST_ANCHOR_OFFSET, resolveChatListAnchoredEndSpace } from "@t3tools/shared/chatList"; +import { videoMimeType } from "@t3tools/shared/video"; import { SymbolView, type AppSymbolName } from "../../components/AppSymbol"; import { HeaderHeightContext } from "@react-navigation/elements"; import { useFocusEffect, useNavigation } from "@react-navigation/native"; @@ -36,6 +37,7 @@ import { useMemo, useRef, useState, + useId, type ReactNode, type RefObject, } from "react"; @@ -62,8 +64,9 @@ import { View, type ViewStyle, } from "react-native"; -import { TouchableOpacity } from "react-native-gesture-handler"; -import ImageViewing from "react-native-image-viewing"; +import { FilePreviewModal, type FilePreviewSource } from "../../components/FilePreviewModal"; +import { isPdfFile } from "../../lib/filePreview"; +import { PresentationSource } from "../../components/NativePresentation"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import Animated, { FadeIn, @@ -89,6 +92,8 @@ import { } from "../../native/SelectableMarkdownText"; import { AppText as Text } from "../../components/AppText"; +import { VideoPreviewModal, type VideoPreviewSource } from "../../components/VideoPreviewModal"; +import { VideoAttachmentTile } from "../../components/VideoAttachmentTile"; import { CopyTextButton } from "../../components/CopyTextButton"; import { parseReviewCommentMessageSegments, @@ -210,9 +215,11 @@ export interface ThreadFeedProps { function MessageAttachmentImage(props: { readonly environmentId: EnvironmentId; readonly attachmentId: string; + readonly name: string; readonly className: string; - readonly onPressImage: (uri: string, headers?: Record) => void; + readonly onPressPreview: (source: FilePreviewSource) => void; }) { + const sourceIdentifier = useId(); const uri = useAssetUrl(props.environmentId, { _tag: "attachment", attachmentId: props.attachmentId, @@ -227,9 +234,17 @@ function MessageAttachmentImage(props: { } return ( - props.onPressImage(uri)}> - - + + + props.onPressPreview({ kind: "image", uri, name: props.name, sourceIdentifier }) + } + > + + + ); } @@ -247,12 +262,32 @@ function isFileAttachment(attachment: ChatAttachment): attachment is ChatFileAtt function MessageAttachmentFile(props: { readonly environmentId: EnvironmentId; readonly attachment: ChatFileAttachment; + readonly onPressPreview: (source: FilePreviewSource) => void; + readonly onPressVideo: (attachment: ChatFileAttachment, sourceIdentifier: string) => void; }) { + const sourceIdentifier = useId(); const createAssetUrl = useAtomQueryRunner(assetEnvironment.createUrl, { reportFailure: false, }); const preparedConnection = usePreparedConnection(props.environmentId); const { attachment } = props; + const videoType = videoMimeType(attachment); + const isPdf = isPdfFile(attachment); + const fileTypeLabel = isPdf + ? "PDF" + : (attachment.name.match(/\.([a-z0-9]{1,8})$/i)?.[1]?.toUpperCase() ?? "File"); + const sizeLabel = formatAttachmentSize(attachment.sizeBytes); + const thumbnailUrl = useAssetUrl( + props.environmentId, + videoType === null + ? null + : { + _tag: "attachment", + attachmentId: attachment.id, + fileName: attachment.name, + mimeType: videoType, + }, + ); const httpBaseUrl = Option.isSome(preparedConnection) ? preparedConnection.value.httpBaseUrl : null; @@ -269,73 +304,127 @@ function MessageAttachmentFile(props: { }, [props.environmentId, attachment.id, httpBaseUrl]), ); + const shareFile = (sourceIdentifier?: string) => { + if (httpBaseUrl === null || openingRef.current) return; + const controller = new AbortController(); + openingRef.current = controller; + setOpening(true); + void (async () => { + try { + const result = await createAssetUrl({ + environmentId: props.environmentId, + input: { + resource: { + _tag: "attachment", + attachmentId: attachment.id, + fileName: attachment.name, + mimeType: attachment.mimeType, + }, + }, + }); + if (controller.signal.aborted) return; + if (result._tag === "Failure") { + throw squashAtomCommandFailure(result); + } + const url = resolveAssetUrl(httpBaseUrl, result.value.relativeUrl); + if (url === null) { + throw new Error("The attachment could not be opened."); + } + await downloadAndShareAttachment({ + url, + attachment, + signal: controller.signal, + sourceIdentifier, + }); + } catch (error) { + if (!controller.signal.aborted) { + Alert.alert( + "Could not open attachment", + error instanceof Error ? error.message : "The attachment is unavailable.", + ); + } + } finally { + if (openingRef.current === controller) { + openingRef.current = null; + setOpening(false); + } + } + })(); + }; + + if (videoType !== null) { + return ( + props.onPressVideo(attachment, sourceIdentifier)} + onShare={() => shareFile(`attachment:${props.environmentId}:${attachment.id}`)} + className="my-1 rounded-2xl" + style={{ width: 224, maxWidth: "100%", aspectRatio: 16 / 9 }} + /> + ); + } + return ( - { - if (httpBaseUrl === null || openingRef.current) return; - const controller = new AbortController(); - openingRef.current = controller; - setOpening(true); - void (async () => { - try { - const result = await createAssetUrl({ - environmentId: props.environmentId, - input: { + + + isPdf + ? props.onPressPreview({ + kind: "pdf", + name: attachment.name, + environmentId: props.environmentId, resource: { _tag: "attachment", attachmentId: attachment.id, fileName: attachment.name, - mimeType: attachment.mimeType, + mimeType: "application/pdf", }, - }, - }); - if (controller.signal.aborted) return; - if (result._tag === "Failure") { - throw squashAtomCommandFailure(result); - } - const url = resolveAssetUrl(httpBaseUrl, result.value.relativeUrl); - if (url === null) { - throw new Error("The attachment could not be opened."); - } - await downloadAndShareAttachment({ url, attachment, signal: controller.signal }); - } catch (error) { - if (!controller.signal.aborted) { - Alert.alert( - "Could not open attachment", - error instanceof Error ? error.message : "The attachment is unavailable.", - ); - } - } finally { - if (openingRef.current === controller) { - openingRef.current = null; - setOpening(false); - } - } - })(); - }} - > - {opening ? ( - - ) : ( + sourceIdentifier, + }) + : shareFile(sourceIdentifier) + } + > + + {opening ? ( + + ) : ( + + )} + + + + {attachment.name} + + + {fileTypeLabel} · {sizeLabel} + + - )} - - {attachment.name} - - - {formatAttachmentSize(attachment.sizeBytes)} - - + + ); } @@ -364,8 +453,9 @@ function ThreadMarkdownImageView(props: { readonly sourceKey: string; readonly unavailable: boolean; readonly alt: string | null; - readonly onPressImage: (uri: string) => void; + readonly onPressPreview: (source: FilePreviewSource) => void; }) { + const sourceIdentifier = useId(); const [availableWidth, setAvailableWidth] = useState(0); const [sourceSize, setSourceSize] = useState<{ width: number; height: number } | null>(null); const [failedUri, setFailedUri] = useState(null); @@ -410,27 +500,35 @@ function ThreadMarkdownImageView(props: { )} ) : ( - props.onPressImage(props.uri!)} - style={{ alignSelf: "flex-start" }} - > - + + props.onPressPreview({ + kind: "image", + uri: props.uri!, + name: props.alt ?? "Image", + sourceIdentifier, + }) + } + style={{ alignSelf: "flex-start" }} > - setFailedUri(props.uri)} - /> - - + + setFailedUri(props.uri)} + /> + + + )} {props.alt ? ( @@ -479,7 +577,7 @@ function ThreadMarkdownImage(props: { readonly threadId: ThreadId; readonly path: string; readonly alt: string | null; - readonly onPressImage: (uri: string) => void; + readonly onPressPreview: (source: FilePreviewSource) => void; }) { const assetUrl = useAssetUrlState(props.environmentId, { _tag: "workspace-file", @@ -493,7 +591,7 @@ function ThreadMarkdownImage(props: { sourceKey={props.path} unavailable={assetUrl._tag === "Failure"} alt={props.alt} - onPressImage={props.onPressImage} + onPressPreview={props.onPressPreview} /> ); } @@ -505,7 +603,7 @@ function ThreadMarkdownImageUnavailable(props: { readonly alt: string | null }) sourceKey="unavailable" unavailable alt={props.alt} - onPressImage={() => undefined} + onPressPreview={() => undefined} /> ); } @@ -1237,7 +1335,8 @@ function renderFeedEntry( readonly onToggleWorkGroup: (groupId: string) => void; readonly onToggleWorkRow: (rowId: string) => void; readonly onToggleTurnFold: (turnId: TurnId) => void; - readonly onPressImage: (uri: string, headers?: Record) => void; + readonly onPressPreview: (source: FilePreviewSource) => void; + readonly onPressVideo: (attachment: ChatFileAttachment, sourceIdentifier: string) => void; readonly onMarkdownLinkPress: (href: string) => void; readonly renderMarkdownImage: MarkdownImageRenderer; readonly iconSubtleColor: string | import("react-native").ColorValue; @@ -1347,14 +1446,17 @@ function renderFeedEntry( key={attachment.id} environmentId={props.environmentId} attachmentId={attachment.id} + name={attachment.name} className="aspect-[1.3] w-full rounded-[14px] bg-white/15" - onPressImage={props.onPressImage} + onPressPreview={props.onPressPreview} /> ) : isFileAttachment(attachment) ? ( ) : ( @@ -1407,14 +1509,17 @@ function renderFeedEntry( key={attachment.id} environmentId={props.environmentId} attachmentId={attachment.id} + name={attachment.name} className="mt-1.5 aspect-[1.3] w-full rounded-[18px] bg-adaptive-neutral-200-800" - onPressImage={props.onPressImage} + onPressPreview={props.onPressPreview} /> ) : isFileAttachment(attachment) ? ( ) : ( @@ -1785,10 +1890,12 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { expandedTurnIds: new Set(), }); const { copiedRowId, expandedWorkGroups, expandedWorkRows, expandedTurnIds } = interactionState; - const [expandedImage, setExpandedImage] = useState<{ - uri: string; - headers?: Record; - } | null>(null); + const [expandedFile, setExpandedFile] = useState(null); + const [expandedVideo, setExpandedVideo] = useState(null); + useEffect(() => { + setExpandedVideo(null); + setExpandedFile(null); + }, [props.environmentId, props.threadId, props.contentPresentation.kind]); const horizontalPadding = props.layoutVariant === "split" ? 20 : 16; const contentHorizontalPadding = deriveCenteredContentHorizontalPadding({ viewportWidth, @@ -1833,6 +1940,22 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { ); if (relativePath) { void Haptics.selectionAsync(); + if (isPdfFile({ name: relativePath })) { + setExpandedFile( + (current) => + current ?? { + kind: "pdf", + name: relativePath.split("/").at(-1), + environmentId: props.environmentId, + resource: { + _tag: "workspace-file", + threadId: props.threadId, + path: relativePath, + }, + }, + ); + return; + } navigation.navigate("ThreadFile", { environmentId: String(props.environmentId), threadId: String(props.threadId), @@ -1844,6 +1967,12 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { } if (presentation.href) { + if (/^https?:\/\//i.test(presentation.href) && isPdfFile({ name: presentation.href })) { + setExpandedFile( + (current) => current ?? { kind: "pdf", uri: presentation.href!, name: "Document.pdf" }, + ); + return; + } void tryOpenExternalUrl(presentation.href, "markdown-link"); } }, @@ -1859,7 +1988,7 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { sourceKey={imageSource.uri} unavailable={false} alt={image.alt} - onPressImage={(uri) => setExpandedImage({ uri })} + onPressPreview={(source) => setExpandedFile((current) => current ?? source)} /> ); } @@ -1872,7 +2001,7 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { threadId={props.threadId} path={imageSource.path} alt={image.alt} - onPressImage={(uri) => setExpandedImage({ uri })} + onPressPreview={(source) => setExpandedFile((current) => current ?? source)} /> ); }, @@ -2260,9 +2389,23 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { [suspendEndScrollMaintenanceForDisclosure], ); - const onPressImage = useCallback((uri: string, headers?: Record) => { - setExpandedImage({ uri, headers }); + const onPressPreview = useCallback((source: FilePreviewSource) => { + setExpandedFile((current) => current ?? source); }, []); + const onPressVideo = useCallback( + (attachment: ChatFileAttachment, sourceIdentifier: string) => { + setExpandedVideo( + (current) => + current ?? { + type: "remote", + environmentId: props.environmentId, + attachment, + sourceIdentifier, + }, + ); + }, + [props.environmentId], + ); // Rows whose height is known before they ever render. Without this, every // row above the viewport is assumed to be estimatedItemSize tall, and @@ -2311,7 +2454,8 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { onToggleWorkGroup, onToggleWorkRow, onToggleTurnFold, - onPressImage, + onPressPreview, + onPressVideo, onMarkdownLinkPress, renderMarkdownImage, iconSubtleColor, @@ -2339,7 +2483,8 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { userBubbleMaxWidth, onCopyWorkRow, onMarkdownLinkPress, - onPressImage, + onPressPreview, + onPressVideo, onToggleTurnFold, onToggleWorkGroup, onToggleWorkRow, @@ -2523,23 +2668,8 @@ export const ThreadFeed = memo(function ThreadFeed(props: ThreadFeedProps) { ) : null} - setExpandedImage(null)} - swipeToCloseEnabled - doubleTapToZoomEnabled - /> + setExpandedVideo(null)} /> + setExpandedFile(null)} /> ); }); diff --git a/apps/mobile/src/features/threads/use-project-actions.ts b/apps/mobile/src/features/threads/use-project-actions.ts index f144baae0..e9722e7db 100644 --- a/apps/mobile/src/features/threads/use-project-actions.ts +++ b/apps/mobile/src/features/threads/use-project-actions.ts @@ -85,6 +85,9 @@ export function useCreateProjectThread() { prepared = await prepareTurnAttachments({ environmentId: input.project.environmentId, attachments: input.initialAttachments, + supportsImageUploads: + appAtomRegistry.get(serverEnvironment.configValueAtom(input.project.environmentId)) + ?.environment.capabilities.attachmentUploads === true, persistUploadedReferences: async (draftAttachments) => { await input.onAttachmentsUploaded(draftAttachments); return "persisted"; diff --git a/apps/mobile/src/lib/attachmentDownload.test.ts b/apps/mobile/src/lib/attachmentDownload.test.ts index b0dd934bd..78e182e74 100644 --- a/apps/mobile/src/lib/attachmentDownload.test.ts +++ b/apps/mobile/src/lib/attachmentDownload.test.ts @@ -4,7 +4,9 @@ const mocks = vi.hoisted(() => ({ directories: new Set(), deleted: vi.fn(), download: vi.fn(), + copy: vi.fn(), share: vi.fn(), + shareFromSource: vi.fn(), available: vi.fn(), uuid: vi.fn(), })); @@ -46,8 +48,12 @@ vi.mock("expo-file-system", () => { static downloadFileAsync = mocks.download; readonly uri: string; - constructor(directory: Directory, name: string) { - this.uri = `${directory.uri}/${encodeURIComponent(name)}`; + constructor(source: Directory | string, name?: string) { + this.uri = typeof source === "string" ? source : `${source.uri}/${encodeURIComponent(name!)}`; + } + + async copy(destination: File): Promise { + await mocks.copy(this.uri, destination.uri); } } @@ -60,8 +66,13 @@ vi.mock("expo-sharing", () => ({ })); vi.mock("./uuid", () => ({ uuidv4: mocks.uuid })); +vi.mock("./shareFileFromSource", () => ({ shareFileFromSource: mocks.shareFromSource })); -import { downloadAndShareAttachment } from "./attachmentDownload"; +import { + downloadAndShareAttachment, + downloadAttachmentForPreview, + shareLocalAttachment, +} from "./attachmentDownload"; import { isForegroundHandoffActive } from "./foreground-handoff"; const NOW = 1_787_990_400_000; @@ -76,11 +87,15 @@ beforeEach(() => { mocks.directories.clear(); mocks.deleted.mockReset(); mocks.download.mockReset(); + mocks.copy.mockReset(); mocks.share.mockReset(); + mocks.shareFromSource.mockReset(); mocks.available.mockReset(); mocks.uuid.mockReset(); mocks.download.mockImplementation(async (_url: string, file: { uri: string }) => file); + mocks.copy.mockResolvedValue(undefined); mocks.share.mockResolvedValue(undefined); + mocks.shareFromSource.mockResolvedValue(undefined); mocks.available.mockResolvedValue(true); let sequence = 0; mocks.uuid.mockImplementation( @@ -268,3 +283,118 @@ describe("downloadAndShareAttachment", () => { await first; }); }); + +describe("attachment preview files", () => { + it("does not start a native request after cancellation during setup", async () => { + const controller = new AbortController(); + const loading = downloadAttachmentForPreview({ ...input, signal: controller.signal }); + controller.abort(); + await expect(loading).resolves.toBeNull(); + expect(mocks.download).not.toHaveBeenCalled(); + expect(mocks.share).not.toHaveBeenCalled(); + }); + + it("downloads for playback without requiring a share sheet and removes the file on close", async () => { + mocks.available.mockResolvedValue(false); + const file = await downloadAttachmentForPreview({ + ...input, + signal: new AbortController().signal, + }); + expect(file?.uri.endsWith("/report.pdf")).toBe(true); + expect(mocks.available).not.toHaveBeenCalled(); + expect(mocks.deleted).not.toHaveBeenCalled(); + file?.dispose(); + file?.dispose(); + expect(mocks.deleted).toHaveBeenCalledTimes(1); + }); + + it.each([undefined, "share-button"])( + "keeps a shared preview after its owner closes (source: %s)", + async (sourceIdentifier) => { + const opened = Promise.withResolvers(); + const sharing = Promise.withResolvers(); + const nativeShare = sourceIdentifier ? mocks.shareFromSource : mocks.share; + nativeShare.mockImplementationOnce(() => { + opened.resolve(); + return sharing.promise; + }); + const file = await downloadAttachmentForPreview({ + ...input, + signal: new AbortController().signal, + }); + const share = file!.share(new AbortController().signal, sourceIdentifier); + await opened.promise; + file!.dispose(); + expect(mocks.deleted).not.toHaveBeenCalled(); + expect(isForegroundHandoffActive()).toBe(true); + sharing.resolve(); + await share; + expect(isForegroundHandoffActive()).toBe(false); + expect(mocks.deleted).not.toHaveBeenCalled(); + expect(mocks.download).toHaveBeenCalledTimes(1); + expect(mocks.copy).not.toHaveBeenCalled(); + }, + ); + + it.each([undefined, "share-button"])( + "does not share a disposed preview after availability checking (source: %s)", + async (sourceIdentifier) => { + const checking = Promise.withResolvers(); + const available = Promise.withResolvers(); + mocks.available.mockImplementation(() => { + checking.resolve(); + return available.promise; + }); + const file = await downloadAttachmentForPreview({ + ...input, + signal: new AbortController().signal, + }); + const share = file!.share(new AbortController().signal, sourceIdentifier); + await checking.promise; + file!.dispose(); + available.resolve(true); + await share; + expect(mocks.share).not.toHaveBeenCalled(); + expect(mocks.shareFromSource).not.toHaveBeenCalled(); + expect(mocks.deleted).toHaveBeenCalledTimes(1); + }, + ); + + it("copies a local original before sharing without downloading or deleting the source", async () => { + const uri = "file:///documents/draft/report.pdf"; + await shareLocalAttachment({ + uri, + attachment: input.attachment, + signal: new AbortController().signal, + }); + expect(mocks.copy).toHaveBeenCalledWith( + uri, + expect.stringMatching(/^file:\/\/\/cache\/.+\/report\.pdf$/), + ); + expect(mocks.share).toHaveBeenCalledWith(mocks.copy.mock.calls[0]![1], expect.any(Object)); + expect(mocks.download).not.toHaveBeenCalled(); + expect(mocks.deleted).not.toHaveBeenCalled(); + }); + + it("waits for a local copy to finish before cleaning up a canceled share", async () => { + const copying = Promise.withResolvers(); + const copied = Promise.withResolvers(); + mocks.copy.mockImplementation(() => { + copying.resolve(); + return copied.promise; + }); + const controller = new AbortController(); + const task = shareLocalAttachment({ + uri: "file:///documents/draft/report.pdf", + attachment: input.attachment, + signal: controller.signal, + }); + await copying.promise; + controller.abort(); + expect(mocks.deleted).not.toHaveBeenCalled(); + copied.resolve(); + await task; + expect(mocks.share).not.toHaveBeenCalled(); + expect(mocks.deleted).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/mobile/src/lib/attachmentDownload.ts b/apps/mobile/src/lib/attachmentDownload.ts index 8c9da9bc0..2ae0c729c 100644 --- a/apps/mobile/src/lib/attachmentDownload.ts +++ b/apps/mobile/src/lib/attachmentDownload.ts @@ -1,5 +1,6 @@ import type { ChatFileAttachment } from "@t3tools/contracts"; import type { Directory } from "expo-file-system"; +import type { SharingOptions } from "expo-sharing"; import { beginForegroundHandoff } from "./foreground-handoff"; import { uuidv4 } from "./uuid"; @@ -52,23 +53,27 @@ function removeDownloadDirectory(directory: Directory): void { } } -/** Downloads original bytes for the native save/share sheet, including inline video responses. */ -export async function downloadAndShareAttachment(input: { - readonly url: string; - readonly attachment: Pick; - readonly signal: AbortSignal; -}): Promise { - const [{ Directory, File, Paths }, Sharing] = await Promise.all([ - import("expo-file-system"), - import("expo-sharing"), - ]); - if (input.signal.aborted) return; +type AttachmentFileMetadata = Pick; + +export interface AttachmentPreviewFile { + readonly uri: string; + readonly share: (signal: AbortSignal, sourceIdentifier?: string) => Promise; + readonly dispose: () => void; +} + +async function availableSharing(signal: AbortSignal) { + if (signal.aborted) return null; + const Sharing = await import("expo-sharing"); const canShare = await Sharing.isAvailableAsync(); - if (input.signal.aborted) return; + if (signal.aborted) return null; if (!canShare) { throw new Error("Saving and sharing files is unavailable on this device."); } + return Sharing; +} +async function createCachedAttachmentFile(attachment: AttachmentFileMetadata) { + const { Directory, File, Paths } = await import("expo-file-system"); const cache = new Directory(Paths.cache, ATTACHMENT_DOWNLOAD_DIRECTORY); cache.create({ idempotent: true, intermediates: true }); const now = Date.now(); @@ -89,40 +94,135 @@ export async function downloadAndShareAttachment(input: { } const directory = new Directory(cache, `${now}-${uuidv4()}`); + directory.create(); + let file: InstanceType; + try { + file = new File(directory, downloadFileName(attachment.name)); + } catch (error) { + removeDownloadDirectory(directory); + throw error; + } activeDirectories.add(directory.uri); + let disposed = false; let shared = false; - let openingShareSheet = false; - try { - directory.create(); - const destination = new File(directory, downloadFileName(input.attachment.name)); - const file = await File.downloadFileAsync(input.url, destination, { signal: input.signal }); - if (input.signal.aborted) return; + let sharing = false; + const release = () => { + if (!disposed || sharing) return; + activeDirectories.delete(directory.uri); + // A receiver can still be reading after Android's chooser returns. + if (!shared) removeDownloadDirectory(directory); + }; + const preview: AttachmentPreviewFile = { + uri: file.uri, + dispose: () => { + disposed = true; + release(); + }, + share: async (signal, sourceIdentifier) => { + if (disposed || sharing || signal.aborted) return; + sharing = true; + try { + const Sharing = await availableSharing(signal); + if (Sharing === null || disposed) return; + const endHandoff = beginForegroundHandoff(); + try { + const options: SharingOptions = { + mimeType: attachment.mimeType.split(";", 1)[0]?.trim() || "application/octet-stream", + dialogTitle: attachment.name, + }; + if (sourceIdentifier) { + const { shareFileFromSource } = await import("./shareFileFromSource"); + if (signal.aborted || disposed) return; + await shareFileFromSource(file.uri, options, sourceIdentifier); + } else { + await Sharing.shareAsync(file.uri, options); + } + shared = true; + } catch (cause) { + if (!signal.aborted) { + throw new Error("Could not open the share sheet. Try again.", { cause }); + } + } finally { + endHandoff(); + } + } finally { + sharing = false; + release(); + } + }, + }; + return { file, preview }; +} - openingShareSheet = true; - const endHandoff = beginForegroundHandoff(); - try { - await Sharing.shareAsync(file.uri, { - mimeType: input.attachment.mimeType.split(";", 1)[0]?.trim() || "application/octet-stream", - dialogTitle: input.attachment.name, - }); - shared = true; - } finally { - endHandoff(); +/** The caller owns this cached file until disposal, unless it has been shared with another app. */ +export async function downloadAttachmentForPreview(input: { + readonly url: string; + readonly attachment: AttachmentFileMetadata; + readonly signal: AbortSignal; +}): Promise { + if (input.signal.aborted) return null; + const { File } = await import("expo-file-system"); + const cached = await createCachedAttachmentFile(input.attachment); + try { + if (input.signal.aborted) { + cached.preview.dispose(); + return null; + } + await File.downloadFileAsync(input.url, cached.file, { signal: input.signal }); + if (input.signal.aborted) { + cached.preview.dispose(); + return null; } + return cached.preview; } catch (cause) { - if (input.signal.aborted) return; - throw new Error( - openingShareSheet - ? "Could not open the share sheet. Try again." - : "Could not download the attachment. Check the connection and try again.", - { cause }, - ); + // Android may leave a partial file after a failed or interrupted request. + cached.preview.dispose(); + if (input.signal.aborted) return null; + throw new Error("Could not download the attachment. Check the connection and try again.", { + cause, + }); + } +} + +/** Downloads original bytes for the native save/share sheet, including inline video responses. */ +export async function downloadAndShareAttachment(input: { + readonly url: string; + readonly attachment: AttachmentFileMetadata; + readonly signal: AbortSignal; + readonly sourceIdentifier?: string; +}): Promise { + if ((await availableSharing(input.signal)) === null) return; + const file = await downloadAttachmentForPreview(input); + if (file === null) return; + try { + await file.share(input.signal, input.sourceIdentifier); } finally { - activeDirectories.delete(directory.uri); - // A receiver can still be reading after Android's chooser returns. - // Successful exports expire on a later open; partial downloads do not. - if (!shared) { - removeDownloadDirectory(directory); + file.dispose(); + } +} + +/** Shares a cache copy so another app never relies on the lifetime of a composer draft. */ +export async function shareLocalAttachment(input: { + readonly uri: string; + readonly attachment: AttachmentFileMetadata; + readonly signal: AbortSignal; + readonly sourceIdentifier?: string; +}): Promise { + if ((await availableSharing(input.signal)) === null) return; + const { File } = await import("expo-file-system"); + const cached = await createCachedAttachmentFile(input.attachment); + try { + if (input.signal.aborted) return; + try { + await new File(input.uri).copy(cached.file); + } catch (cause) { + if (input.signal.aborted) return; + throw new Error("Could not prepare the attachment for sharing.", { cause }); } + if (!input.signal.aborted) { + await cached.preview.share(input.signal, input.sourceIdentifier); + } + } finally { + cached.preview.dispose(); } } diff --git a/apps/mobile/src/lib/attachmentUpload.test.ts b/apps/mobile/src/lib/attachmentUpload.test.ts index 488c8375a..5e8a34dd1 100644 --- a/apps/mobile/src/lib/attachmentUpload.test.ts +++ b/apps/mobile/src/lib/attachmentUpload.test.ts @@ -12,6 +12,8 @@ const mocks = vi.hoisted(() => ({ runAtomCommand: vi.fn(), readAtom: vi.fn(), upload: vi.fn(), + writeFile: vi.fn(), + deleteFile: vi.fn(), })); vi.mock("@t3tools/client-runtime/state/runtime", () => ({ @@ -53,13 +55,25 @@ vi.mock("./uuid", () => ({ vi.mock("expo-file-system", () => ({ File: class { - constructor(readonly uri: string) {} + readonly uri: string; + exists = true; + constructor(uri: string, name?: string) { + this.uri = name ? `${uri}/${name}` : uri; + } + create() {} + write(bytes: string, options: unknown) { + mocks.writeFile(this.uri, bytes, options); + } + delete() { + mocks.deleteFile(this.uri); + } upload(url: string, options: unknown) { return mocks.upload(this.uri, url, options); } }, Paths: { + cache: "file:///cache", get document() { return { uri: mocks.documentUri }; }, @@ -158,6 +172,8 @@ describe("prepareTurnAttachments", () => { mocks.runAtomCommand.mockReset(); mocks.readAtom.mockReset(); mocks.upload.mockReset(); + mocks.writeFile.mockReset(); + mocks.deleteFile.mockReset(); mocks.readAtom.mockReturnValue(Option.some({ httpBaseUrl: "https://environment.example/" })); mocks.runAtomCommand.mockImplementation(async (_registry: unknown, command: unknown) => command === mocks.createUploadUrl @@ -198,11 +214,11 @@ describe("prepareTurnAttachments", () => { expect(mocks.upload).toHaveBeenCalledWith( "file:///documents/report.pdf", "https://environment.example/api/attachments/upload/signed", - { + expect.objectContaining({ httpMethod: "POST", uploadType: 0, headers: { "Content-Type": "application/pdf" }, - }, + }), ); expect(prepared.status).toBe("ready"); if (prepared.status !== "ready") return; @@ -354,6 +370,123 @@ describe("prepareTurnAttachments", () => { expect(prepared.pendingAttachmentIds).toEqual([MINTED_ID]); }); + it("uploads image bytes over HTTP while retaining the durable offline image", async () => { + const persisted = vi.fn(async () => "persisted" as const); + const prepared = await prepareTurnAttachments({ + environmentId, + attachments: [image], + supportsImageUploads: true, + persistUploadedReferences: persisted, + }); + expect(mocks.writeFile).toHaveBeenCalledWith("file:///cache/t3-upload-uuid", "YWJj", { + encoding: "base64", + }); + expect(mocks.upload).toHaveBeenCalledWith( + "file:///cache/t3-upload-uuid", + "https://environment.example/api/attachments/upload/signed", + expect.objectContaining({ headers: { "Content-Type": "image/png" } }), + ); + expect(mocks.deleteFile).toHaveBeenCalledExactlyOnceWith("file:///cache/t3-upload-uuid"); + expect(prepared.status).toBe("ready"); + if (prepared.status !== "ready") return; + expect(prepared.attachments).toEqual([ + { + type: "image", + id: MINTED_ID, + name: image.name, + mimeType: image.mimeType, + sizeBytes: image.sizeBytes, + }, + ]); + expect(prepared.draftAttachments).toEqual([ + { ...image, uploadedAttachmentId: MINTED_ID, uploadEnvironmentId: environmentId }, + ]); + expect(persisted).toHaveBeenCalledWith(prepared.draftAttachments); + }); + + it("reuses an uploaded image and reuploads its local bytes after server expiry", async () => { + const saved = { + ...image, + uploadedAttachmentId: "saved-image", + uploadEnvironmentId: environmentId, + }; + const reused = await prepareTurnAttachments({ + environmentId, + attachments: [saved], + supportsImageUploads: true, + }); + expect(reused.status === "ready" && reused.attachments[0]).toEqual({ + type: "image", + id: "saved-image", + name: image.name, + mimeType: image.mimeType, + sizeBytes: image.sizeBytes, + }); + expect(mocks.upload).not.toHaveBeenCalled(); + mocks.executeAtomQuery.mockResolvedValueOnce({ + _tag: "Failure", + error: { _tag: "AssetAttachmentNotFoundError" }, + }); + const restored = await prepareTurnAttachments({ + environmentId, + attachments: [saved], + supportsImageUploads: true, + }); + expect(restored.status === "ready" && restored.draftAttachments[0]).toEqual({ + ...saved, + uploadedAttachmentId: MINTED_ID, + }); + expect(mocks.writeFile).toHaveBeenCalledWith("file:///cache/t3-upload-uuid", "YWJj", { + encoding: "base64", + }); + }); + + it("does not reuse an image upload from another environment", async () => { + const prepared = await prepareTurnAttachments({ + environmentId, + attachments: [ + { + ...image, + uploadedAttachmentId: "other-image", + uploadEnvironmentId: EnvironmentId.make("other"), + }, + ], + supportsImageUploads: true, + }); + expect(mocks.executeAtomQuery).not.toHaveBeenCalled(); + expect(mocks.upload).toHaveBeenCalledOnce(); + expect(prepared.status === "ready" && prepared.draftAttachments[0]?.uploadEnvironmentId).toBe( + environmentId, + ); + }); + + it("aborts an active transfer without dropping local bytes or stamping a partial upload", async () => { + const started = Promise.withResolvers(); + const controller = new AbortController(); + const persist = vi.fn(async () => "persisted" as const); + mocks.upload.mockImplementation( + (_uri: string, _url: string, options: { signal: AbortSignal }) => + new Promise((_, reject) => { + options.signal.addEventListener("abort", () => reject(new Error("cancelled")), { + once: true, + }); + started.resolve(); + }), + ); + const preparing = prepareTurnAttachments({ + environmentId, + attachments: [file], + signal: controller.signal, + persistUploadedReferences: persist, + }); + await started.promise; + controller.abort(); + expect(await preparing).toEqual({ status: "abandoned" }); + expect(persist).not.toHaveBeenCalled(); + expect(mocks.deleteFile).not.toHaveBeenCalled(); + expect(removeCallsFor(MINTED_ID)).toBe(1); + }); + it("removes pending uploads when the native HTTP request fails", async () => { mocks.upload.mockResolvedValue({ status: 500, body: "failed", headers: {} }); diff --git a/apps/mobile/src/lib/attachmentUpload.ts b/apps/mobile/src/lib/attachmentUpload.ts index afe669d6f..f39329373 100644 --- a/apps/mobile/src/lib/attachmentUpload.ts +++ b/apps/mobile/src/lib/attachmentUpload.ts @@ -9,9 +9,11 @@ import { import { runAtomCommand, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import type { ChatFileAttachment, + ChatImageAttachment, EnvironmentId, UploadChatImageAttachment, } from "@t3tools/contracts"; +import { PROVIDER_SEND_TURN_SUPPORTED_IMAGE_MIME_TYPES } from "@t3tools/contracts"; import * as Option from "effect/Option"; import { appAtomRegistry } from "../state/atom-registry"; @@ -20,6 +22,7 @@ import { attachmentEnvironment } from "../state/attachments"; import { environmentSession } from "../state/session"; import { resolveOwnedComposerAttachmentFileUri } from "./composerAttachmentFiles"; import { toUploadChatImageAttachments, type DraftComposerAttachment } from "./composerImages"; +import { uuidv4 } from "./uuid"; /** * This module owns the server side of a composer attachment's lifecycle. @@ -31,7 +34,10 @@ import { toUploadChatImageAttachments, type DraftComposerAttachment } from "./co * owned by `removeThreadOutboxMessage` / the composer draft mutators, which * release files through `releaseUnusedComposerAttachmentFiles`. */ -export type UploadedMobileAttachment = UploadChatImageAttachment | ChatFileAttachment; +export type UploadedMobileAttachment = + | UploadChatImageAttachment + | ChatImageAttachment + | ChatFileAttachment; export function validateDraftFileAttachments(input: { readonly attachments: ReadonlyArray; @@ -56,7 +62,7 @@ export function validateDraftFileAttachments(input: { return oversized ? fileAttachmentTooLargeMessage(oversized.name, maxBytes) : null; } -/** Keep uploaded file ids on durable drafts so a later send can reuse their bytes. */ +/** Keep uploaded ids alongside the local bytes so a later send can reuse them. */ export function withUploadedMobileAttachmentReferences(input: { readonly environmentId: EnvironmentId; readonly attachments: ReadonlyArray; @@ -65,8 +71,9 @@ export function withUploadedMobileAttachmentReferences(input: { return input.attachments.map((attachment, index) => { const uploaded = input.uploadedAttachments[index]; if ( - attachment.type !== "file" || - uploaded?.type !== "file" || + !uploaded || + !("id" in uploaded) || + attachment.type !== uploaded.type || (attachment.uploadedAttachmentId === uploaded.id && attachment.uploadEnvironmentId === input.environmentId) ) { @@ -145,21 +152,73 @@ export type PrepareTurnAttachmentsResult = | PreparedTurnAttachments | { readonly status: "abandoned" }; +function uploadedReference( + attachment: DraftComposerAttachment, + id: string, +): ChatImageAttachment | ChatFileAttachment { + const fields = { + id, + name: attachment.name, + mimeType: attachment.mimeType, + sizeBytes: attachment.sizeBytes, + }; + return attachment.type === "image" ? { type: "image", ...fields } : { type: "file", ...fields }; +} + +function attachmentUploadInput(attachment: DraftComposerAttachment) { + const fields = { + name: attachment.name, + mimeType: attachment.mimeType, + sizeBytes: attachment.sizeBytes, + }; + if (attachment.type === "file") return { type: "file" as const, ...fields }; + const mimeType = PROVIDER_SEND_TURN_SUPPORTED_IMAGE_MIME_TYPES.find( + (type) => type === attachment.mimeType.toLowerCase(), + ); + if (!mimeType) throw new Error(`Unsupported image type for '${attachment.name}'.`); + return { ...fields, mimeType }; +} + async function uploadFileBytes( - attachment: Extract, + attachment: DraftComposerAttachment, url: string, + signal: AbortSignal, + onProgress?: (progress: number) => void, ): Promise { const { File, Paths, UploadType } = await import("expo-file-system"); - const fileUri = - resolveOwnedComposerAttachmentFileUri(attachment.fileUri, Paths.document.uri) ?? - attachment.fileUri; - const result = await new File(fileUri).upload(url, { - httpMethod: "POST", - uploadType: UploadType.BINARY_CONTENT, - headers: { "Content-Type": attachment.mimeType }, - }); - if (result.status < 200 || result.status >= 300) { - throw new Error(`Upload failed for '${attachment.name}' (${result.status}).`); + if (signal.aborted) throw new Error("Upload cancelled."); + const file = + attachment.type === "image" + ? new File(Paths.cache, `t3-upload-${uuidv4()}`) + : new File( + resolveOwnedComposerAttachmentFileUri(attachment.fileUri, Paths.document.uri) ?? + attachment.fileUri, + ); + try { + if (attachment.type === "image") { + file.create(); + file.write(attachment.dataUrl.slice(attachment.dataUrl.indexOf(",") + 1), { + encoding: "base64", + }); + } + const result = await file.upload(url, { + httpMethod: "POST", + uploadType: UploadType.BINARY_CONTENT, + headers: { "Content-Type": attachment.mimeType }, + signal, + ...(onProgress + ? { + onProgress: ({ bytesSent, totalBytes }) => { + if (totalBytes > 0) onProgress(bytesSent / totalBytes); + }, + } + : {}), + }); + if (result.status < 200 || result.status >= 300) { + throw new Error(`Upload failed for '${attachment.name}' (${result.status}).`); + } + } finally { + if (attachment.type === "image" && file.exists) file.delete(); } } @@ -176,11 +235,16 @@ async function uploadFileBytes( export async function prepareTurnAttachments(input: { readonly environmentId: EnvironmentId; readonly attachments: ReadonlyArray; + /** Older environments continue to receive inline images. */ + readonly supportsImageUploads?: boolean; + readonly signal?: AbortSignal; + readonly onUploadProgress?: (attachmentId: string, progress: number) => void; readonly persistUploadedReferences?: ( draftAttachments: ReadonlyArray, ) => Promise<"persisted" | "abandon">; }): Promise { const { environmentId } = input; + if (input.signal?.aborted) return { status: "abandoned" }; const files = input.attachments.filter((attachment) => attachment.type === "file"); const ready = ( attachments: ReadonlyArray, @@ -194,7 +258,7 @@ export async function prepareTurnAttachments(input: { releaseUploads: () => releasePendingAttachmentUploads(environmentId, pendingAttachmentIds), }); - if (files.length === 0) { + if (input.attachments.length === 0 || (files.length === 0 && !input.supportsImageUploads)) { return ready( toUploadChatImageAttachments( input.attachments.filter((attachment) => attachment.type === "image"), @@ -214,9 +278,13 @@ export async function prepareTurnAttachments(input: { const uploadedAttachments: UploadedMobileAttachment[] = []; const pendingAttachmentIds: string[] = []; const createdAttachmentIds: string[] = []; + const controller = new AbortController(); + const abort = () => controller.abort(); + input.signal?.addEventListener("abort", abort, { once: true }); try { for (const attachment of input.attachments) { - if (attachment.type === "image") { + if (controller.signal.aborted) throw new Error("Upload cancelled."); + if (attachment.type === "image" && !input.supportsImageUploads) { uploadedAttachments.push(...toUploadChatImageAttachments([attachment])); continue; } @@ -238,13 +306,7 @@ export async function prepareTurnAttachments(input: { } if (verification.status === "verified") { pendingAttachmentIds.push(attachment.uploadedAttachmentId); - uploadedAttachments.push({ - type: "file", - id: attachment.uploadedAttachmentId, - name: attachment.name, - mimeType: attachment.mimeType, - sizeBytes: attachment.sizeBytes, - }); + uploadedAttachments.push(uploadedReference(attachment, attachment.uploadedAttachmentId)); continue; } // "missing": the pending upload expired, upload the bytes again. @@ -255,12 +317,7 @@ export async function prepareTurnAttachments(input: { createUploadUrl: attachmentEnvironment.createUploadUrl, remove: attachmentEnvironment.remove, environmentId, - upload: { - type: "file", - name: attachment.name, - mimeType: attachment.mimeType, - sizeBytes: attachment.sizeBytes, - }, + upload: attachmentUploadInput(attachment), // Read the connection at transfer time: the environment may have // reconnected on a new base URL since this cycle started. resolveUploadUrl: (relativeUrl) => { @@ -272,11 +329,18 @@ export async function prepareTurnAttachments(input: { : resolveAssetUrl(currentConnection.value.httpBaseUrl, relativeUrl); }, transport: (url) => ({ - done: uploadFileBytes(attachment, url), - // expo-file-system uploads cannot abort mid-flight. - abort: () => {}, + done: uploadFileBytes( + attachment, + url, + controller.signal, + input.onUploadProgress + ? (progress) => input.onUploadProgress?.(attachment.id, progress) + : undefined, + ), + abort, }), onMinted: (attachmentId) => { + if (controller.signal.aborted) return "cancel"; pendingAttachmentIds.push(attachmentId); createdAttachmentIds.push(attachmentId); return "continue"; @@ -287,15 +351,11 @@ export async function prepareTurnAttachments(input: { ? result.error : new Error(`Upload failed for '${attachment.name}'.`); } - uploadedAttachments.push({ - type: "file", - id: result.attachmentId, - name: attachment.name, - mimeType: attachment.mimeType, - sizeBytes: attachment.sizeBytes, - }); + uploadedAttachments.push(uploadedReference(attachment, result.attachmentId)); } + if (controller.signal.aborted) throw new Error("Upload cancelled."); + const draftAttachments = withUploadedMobileAttachmentReferences({ environmentId, attachments: input.attachments, @@ -313,6 +373,9 @@ export async function prepareTurnAttachments(input: { return ready(uploadedAttachments, pendingAttachmentIds, draftAttachments); } catch (error) { await releaseCreatedUploadsQuietly(environmentId, createdAttachmentIds); + if (controller.signal.aborted) return { status: "abandoned" }; throw error; + } finally { + input.signal?.removeEventListener("abort", abort); } } diff --git a/apps/mobile/src/lib/composer-image-schema.ts b/apps/mobile/src/lib/composer-image-schema.ts index 401a5fd51..3303dad36 100644 --- a/apps/mobile/src/lib/composer-image-schema.ts +++ b/apps/mobile/src/lib/composer-image-schema.ts @@ -9,6 +9,8 @@ export const DraftComposerImageAttachmentSchema = Schema.Struct({ mimeType: Schema.String, sizeBytes: Schema.Number, dataUrl: Schema.String, + uploadedAttachmentId: Schema.optional(Schema.String), + uploadEnvironmentId: Schema.optional(EnvironmentId), }); export const DraftComposerFileAttachmentSchema = Schema.Struct({ diff --git a/apps/mobile/src/lib/composerAttachmentFiles.ts b/apps/mobile/src/lib/composerAttachmentFiles.ts index a50daa30b..963566b6a 100644 --- a/apps/mobile/src/lib/composerAttachmentFiles.ts +++ b/apps/mobile/src/lib/composerAttachmentFiles.ts @@ -6,6 +6,7 @@ const IOS_DOCUMENTS_PATH = new RegExp( `^(.*/Containers/Data/Application/)${UUID_PATTERN}/Documents$`, "i", ); +const retainedFiles = new Map(); function fileUriPath(uri: string): string | null { try { @@ -52,6 +53,30 @@ export function composerAttachmentFileReferenceKey(uri: string): string { return `file://${documentPath}/${COMPOSER_ATTACHMENT_DIRECTORY}/${encodeURIComponent(location.name)}`; } +/** Holds a local copy until its last player or share-copy operation releases it. */ +export function retainComposerAttachmentFile(uri: string, onLastRelease: () => void): () => void { + const key = composerAttachmentFileReferenceKey(uri); + retainedFiles.set(key, (retainedFiles.get(key) ?? 0) + 1); + let released = false; + return () => { + if (released) { + return; + } + released = true; + const remaining = (retainedFiles.get(key) ?? 1) - 1; + if (remaining > 0) { + retainedFiles.set(key, remaining); + return; + } + retainedFiles.delete(key); + onLastRelease(); + }; +} + +export function isComposerAttachmentFileRetained(uri: string): boolean { + return retainedFiles.has(composerAttachmentFileReferenceKey(uri)); +} + /** * Resolves only our saved attachment copies. iOS preserves Documents on updates * but can change its container UUID. Picker and open-in-place source URIs must diff --git a/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts b/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts new file mode 100644 index 000000000..6b040b698 --- /dev/null +++ b/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts @@ -0,0 +1,258 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + composerAttachmentUploadBlockReason, + composerAttachmentUploadKey, + composerDraftEnvironmentId, + createComposerAttachmentUploadQueue, + type ComposerAttachmentUploadRequest, + type ComposerAttachmentUploadState, +} from "./composerAttachmentUploadQueue"; + +const environmentId = EnvironmentId.make("environment-1"); +function request(id: string, environment = environmentId): ComposerAttachmentUploadRequest { + return { + environmentId: environment, + attachment: { + id, + type: "file", + name: `${id}.pdf`, + mimeType: "application/pdf", + sizeBytes: 42, + fileUri: `file:///documents/${id}.pdf`, + }, + }; +} + +describe("composer attachment upload queue", () => { + it("bounds concurrency, deduplicates updates, and drains all attachments", async () => { + const gates = new Map>>(); + const fourthStarted = Promise.withResolvers(); + const firstThreeStarted = Promise.withResolvers(); + let active = 0; + let maximum = 0; + const upload = vi.fn(async (input: ComposerAttachmentUploadRequest) => { + active += 1; + maximum = Math.max(maximum, active); + const gate = Promise.withResolvers(); + gates.set(input.attachment.id, gate); + if (gates.size === 3) firstThreeStarted.resolve(); + if (gates.size === 4) fourthStarted.resolve(); + try { + return await gate.promise; + } finally { + active -= 1; + } + }); + const queue = createComposerAttachmentUploadQueue({ upload, onChange: () => {} }); + const requests = [request("one"), request("two"), request("three"), request("four")]; + queue.sync(requests); + queue.sync(requests); + await firstThreeStarted.promise; + expect(upload).toHaveBeenCalledTimes(3); + gates.get("one")!.resolve(true); + await fourthStarted.promise; + for (const gate of gates.values()) gate.resolve(true); + await queue.settled(); + queue.sync(requests); + await queue.settled(); + expect(maximum).toBe(3); + expect(upload).toHaveBeenCalledTimes(4); + queue.dispose(); + }); + + it("cancels on disconnect and resumes from the same local draft on reconnect", async () => { + const started = Promise.withResolvers(); + let states: Readonly> = {}; + let signal: AbortSignal | undefined; + const upload = vi.fn( + async (_request: ComposerAttachmentUploadRequest, currentSignal: AbortSignal) => { + signal = currentSignal; + started.resolve(); + return new Promise((resolve) => + currentSignal.addEventListener("abort", () => resolve(false), { once: true }), + ); + }, + ); + const queue = createComposerAttachmentUploadQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + const local = request("offline-draft"); + queue.sync([local]); + await started.promise; + queue.sync([]); + await queue.settled(); + expect(signal?.aborted).toBe(true); + expect(states).toEqual({}); + upload.mockResolvedValueOnce(true); + queue.sync([local]); + await queue.settled(); + expect(upload.mock.calls[1]?.[0]).toBe(local); + expect(states[composerAttachmentUploadKey(environmentId, local.attachment.id)]).toEqual({ + status: "ready", + }); + expect(local.attachment).toMatchObject({ fileUri: "file:///documents/offline-draft.pdf" }); + queue.dispose(); + }); + + it("ignores a late completion after removal or environment switch", async () => { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + let states: Readonly> = {}; + const upload = vi.fn(async () => { + started.resolve(); + return gate.promise; + }); + const queue = createComposerAttachmentUploadQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + queue.sync([request("photo")]); + await started.promise; + upload.mockResolvedValueOnce(true); + const other = EnvironmentId.make("environment-2"); + queue.sync([request("photo", other)]); + gate.resolve(true); + await queue.settled(); + expect(states).toEqual({ [composerAttachmentUploadKey(other, "photo")]: { status: "ready" } }); + queue.sync([]); + expect(states).toEqual({}); + queue.dispose(); + }); + + it("restarts a re-added attachment after its aborted transfer finishes settling", async () => { + const firstStarted = Promise.withResolvers(); + const firstSettled = Promise.withResolvers(); + const secondStarted = Promise.withResolvers(); + const secondSettled = Promise.withResolvers(); + let states: Readonly> = {}; + let firstSignal: AbortSignal | undefined; + const upload = vi.fn(async (_request: ComposerAttachmentUploadRequest, signal: AbortSignal) => { + if (!firstSignal) { + firstSignal = signal; + firstStarted.resolve(); + return firstSettled.promise; + } + secondStarted.resolve(); + return secondSettled.promise; + }); + const queue = createComposerAttachmentUploadQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + const local = request("re-added"); + queue.sync([local]); + await firstStarted.promise; + queue.sync([]); + queue.sync([local]); + expect(firstSignal?.aborted).toBe(true); + expect(upload).toHaveBeenCalledOnce(); + firstSettled.resolve(false); + await secondStarted.promise; + expect(upload).toHaveBeenCalledTimes(2); + secondSettled.resolve(true); + await queue.settled(); + expect(states[composerAttachmentUploadKey(environmentId, local.attachment.id)]).toEqual({ + status: "ready", + }); + queue.dispose(); + }); + + it("keeps failures stable until retry and reports bounded progress", async () => { + let states: Readonly> = {}; + const progress: number[] = []; + const upload = vi.fn( + async ( + _request: ComposerAttachmentUploadRequest, + _signal: AbortSignal, + report: (value: number) => void, + ): Promise => { + report(0.12); + report(0.13); + report(1.1); + throw new Error("Server unavailable"); + }, + ); + const queue = createComposerAttachmentUploadQueue({ + upload, + onChange: (next) => { + states = next; + const state = next[composerAttachmentUploadKey(environmentId, "file")]; + if (state?.status === "uploading") progress.push(state.progress); + }, + }); + queue.sync([request("file")]); + await queue.settled(); + queue.sync([request("file")]); + expect(upload).toHaveBeenCalledOnce(); + expect(states[composerAttachmentUploadKey(environmentId, "file")]).toEqual({ + status: "failed", + reason: "Server unavailable", + }); + expect(progress).toEqual([0, 0.1, 1]); + upload.mockImplementationOnce(async () => true); + queue.retry(environmentId, "file"); + await queue.settled(); + expect(states[composerAttachmentUploadKey(environmentId, "file")]).toEqual({ status: "ready" }); + queue.dispose(); + }); + + it("does not spin when an upload's draft was abandoned before persistence", async () => { + const upload = vi.fn(async () => false); + const queue = createComposerAttachmentUploadQueue({ upload, onChange: () => {} }); + queue.sync([request("discarded")]); + await queue.settled(); + expect(upload).toHaveBeenCalledOnce(); + queue.dispose(); + }); +}); + +describe("draft upload scope and offline submission", () => { + it("resolves thread, new-task, and queued-task drafts without crossing environments", () => { + expect(composerDraftEnvironmentId("environment-1:thread", [])).toBe(environmentId); + expect(composerDraftEnvironmentId("new-task:environment-1:project", [])).toBe(environmentId); + expect( + composerDraftEnvironmentId("pending-task:message", [{ messageId: "message", environmentId }]), + ).toBe(environmentId); + expect(composerDraftEnvironmentId("pending-task:missing", [])).toBeNull(); + const colonEnvironment = EnvironmentId.make("a:vcs-status:b"); + expect(composerDraftEnvironmentId(`${colonEnvironment}:thread`, [])).toBe(colonEnvironment); + expect(composerDraftEnvironmentId(`new-task:${colonEnvironment}:project`, [])).toBe( + colonEnvironment, + ); + }); + + it("allows offline queuing while a connected composer waits for upload or retry", () => { + const key = composerAttachmentUploadKey(environmentId, "file"); + const input = { + environmentId, + attachments: [request("file").attachment], + connected: true, + serverConfig: { + environment: { + capabilities: { attachmentUploads: true, fileAttachments: { maxUploadBytes: 1024 } }, + }, + }, + states: {}, + }; + expect(composerAttachmentUploadBlockReason(input)).toBe("Attachment still uploading"); + expect(composerAttachmentUploadBlockReason({ ...input, connected: false })).toBeNull(); + expect( + composerAttachmentUploadBlockReason({ + ...input, + states: { [key]: { status: "failed", reason: "Offline" } }, + }), + ).toBe("Retry or remove the failed attachment"); + expect( + composerAttachmentUploadBlockReason({ ...input, states: { [key]: { status: "ready" } } }), + ).toBeNull(); + }); +}); diff --git a/apps/mobile/src/lib/composerAttachmentUploadQueue.ts b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts new file mode 100644 index 000000000..071afefa4 --- /dev/null +++ b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts @@ -0,0 +1,193 @@ +import { EnvironmentId, type ServerConfig } from "@t3tools/contracts"; +import { clampFileAttachmentUploadBytes } from "@t3tools/client-runtime/state/attachments"; + +import type { DraftComposerAttachment } from "./composerImages"; + +export interface ComposerAttachmentUploadRequest { + readonly environmentId: EnvironmentId; + readonly attachment: DraftComposerAttachment; +} + +export type ComposerAttachmentUploadState = + | { readonly status: "uploading"; readonly progress: number } + | { readonly status: "ready" } + | { readonly status: "failed"; readonly reason: string }; + +export function composerAttachmentUploadKey( + environmentId: EnvironmentId, + attachmentId: string, +): string { + return `${environmentId}:${attachmentId}`; +} + +export function composerDraftEnvironmentId( + draftKey: string, + queuedMessages: ReadonlyArray<{ + readonly messageId: string; + readonly environmentId: EnvironmentId; + }>, +): EnvironmentId | null { + if (draftKey.startsWith("pending-task:")) { + return ( + queuedMessages.find((message) => `pending-task:${message.messageId}` === draftKey) + ?.environmentId ?? null + ); + } + const scope = draftKey.startsWith("new-task:") ? draftKey.slice("new-task:".length) : draftKey; + const separator = scope.lastIndexOf(":"); + return separator > 0 ? EnvironmentId.make(scope.slice(0, separator)) : null; +} + +type UploadServerConfig = { + readonly environment: { + readonly capabilities: Pick< + ServerConfig["environment"]["capabilities"], + "attachmentUploads" | "fileAttachments" + >; + }; +}; + +export function canUploadComposerAttachment( + attachment: DraftComposerAttachment, + config: UploadServerConfig | null | undefined, +): boolean { + const capabilities = config?.environment.capabilities; + return ( + capabilities?.attachmentUploads === true && + (attachment.type === "image" || + (capabilities.fileAttachments !== undefined && + attachment.sizeBytes <= + clampFileAttachmentUploadBytes(capabilities.fileAttachments.maxUploadBytes))) + ); +} + +export function composerAttachmentUploadBlockReason(input: { + readonly environmentId: EnvironmentId; + readonly attachments: ReadonlyArray; + readonly connected: boolean; + readonly serverConfig: UploadServerConfig | null; + readonly states: Readonly>; +}): string | null { + if (!input.connected) return null; + for (const attachment of input.attachments) { + if (!canUploadComposerAttachment(attachment, input.serverConfig)) continue; + const state = input.states[composerAttachmentUploadKey(input.environmentId, attachment.id)]; + if (state?.status === "failed") return "Retry or remove the failed attachment"; + if (state?.status !== "ready") return "Attachment still uploading"; + } + return null; +} + +/** Bounds transfers across environments; disconnected or discarded drafts keep their local bytes. */ +export function createComposerAttachmentUploadQueue(options: { + readonly upload: ( + request: ComposerAttachmentUploadRequest, + signal: AbortSignal, + onProgress: (progress: number) => void, + ) => Promise; + readonly onChange: (states: Readonly>) => void; +}) { + const jobs = new Map< + string, + { readonly controller: AbortController; readonly done: Promise } + >(); + let desired = new Map(); + let states: Readonly> = {}; + let disposed = false; + + function setState(key: string, state: ComposerAttachmentUploadState | undefined) { + const previous = states[key]; + if ( + previous === state || + (previous?.status === "uploading" && + state?.status === "uploading" && + previous.progress === state.progress) + ) + return; + const next = { ...states }; + if (state) next[key] = state; + else delete next[key]; + states = next; + options.onChange(states); + } + + function pump() { + if (disposed) return; + for (const [key, request] of desired) { + if (jobs.size >= 3) break; + if (jobs.has(key) || states[key]?.status === "ready" || states[key]?.status === "failed") + continue; + const controller = new AbortController(); + setState(key, { status: "uploading", progress: 0 }); + // Publish the job before starting async work, including synchronous test transports. + const done = Promise.resolve() + .then(() => + options.upload(request, controller.signal, (progress) => { + if (controller.signal.aborted) return; + setState(key, { + status: "uploading", + progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, + }); + }), + ) + .then((persisted) => { + if (!controller.signal.aborted && desired.has(key)) { + if (!persisted) desired.delete(key); + setState(key, persisted ? { status: "ready" } : undefined); + } + }) + .catch((error: unknown) => { + if (!controller.signal.aborted && desired.has(key)) { + setState(key, { + status: "failed", + reason: error instanceof Error ? error.message : "Upload failed. Tap to retry.", + }); + } + }) + .finally(() => { + jobs.delete(key); + pump(); + }); + jobs.set(key, { controller, done }); + } + } + + return { + sync(requests: ReadonlyArray) { + if (disposed) return; + desired = new Map( + requests.map((request) => [ + composerAttachmentUploadKey(request.environmentId, request.attachment.id), + request, + ]), + ); + for (const [key, job] of jobs) { + if (!desired.has(key)) job.controller.abort(); + } + for (const key of Object.keys(states)) { + if (!desired.has(key)) setState(key, undefined); + } + for (const key of desired.keys()) { + if (!states[key]) setState(key, { status: "uploading", progress: 0 }); + } + pump(); + }, + retry(environmentId: EnvironmentId, attachmentId: string) { + const key = composerAttachmentUploadKey(environmentId, attachmentId); + if (states[key]?.status !== "failed") return; + setState(key, undefined); + pump(); + }, + /** Waits for the current transfers, useful for shutdown and focused verification. */ + async settled() { + while (jobs.size > 0) await Promise.all([...jobs.values()].map((job) => job.done)); + }, + dispose() { + disposed = true; + desired.clear(); + for (const job of jobs.values()) job.controller.abort(); + states = {}; + options.onChange(states); + }, + }; +} diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index f52bd9276..b38c0813c 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { PROVIDER_SEND_TURN_MAX_IMAGE_BYTES } from "@t3tools/contracts"; import type { ImagePickerAsset } from "expo-image-picker"; const mocks = vi.hoisted(() => ({ @@ -9,6 +10,7 @@ const mocks = vi.hoisted(() => ({ delete: vi.fn(), open: vi.fn(), size: vi.fn(), + readBase64: vi.fn(), })); vi.mock("expo-file-system", () => { @@ -23,8 +25,6 @@ vi.mock("expo-file-system", () => { } class File { - static pickFileAsync = mocks.pickFile; - readonly uri: string; constructor(source: string | Directory, name?: string) { @@ -57,6 +57,10 @@ vi.mock("expo-file-system", () => { mocks.copy(this.uri, destination.uri); } + async base64(): Promise { + return mocks.readBase64(this.uri); + } + delete(): void { mocks.delete(this.uri); } @@ -75,6 +79,7 @@ vi.mock("expo-file-system", () => { }); vi.mock("expo-image-picker", () => ({ launchImageLibraryAsync: mocks.pickMedia })); +vi.mock("expo-document-picker", () => ({ getDocumentAsync: mocks.pickFile })); vi.mock("./uuid", () => ({ uuidv4: () => "attachment-id" })); import { @@ -85,6 +90,7 @@ import { removePersistedComposerAttachmentFile, } from "./composerImages"; import { isForegroundHandoffActive } from "./foreground-handoff"; +import { retainComposerAttachmentFile } from "./composerAttachmentFiles"; describe("composer file attachments", () => { beforeEach(() => { @@ -95,9 +101,115 @@ describe("composer file attachments", () => { mocks.delete.mockReset(); mocks.open.mockReset(); mocks.size.mockReset(); + mocks.readBase64.mockReset(); mocks.size.mockImplementation((uri: string) => (uri.startsWith("content:") ? null : 42)); }); + describe("photo library image conversion", () => { + const jpeg = "/9j/2Q=="; + const photo: ImagePickerAsset = { + uri: "file:///picker/photo.heic", + type: "image", + fileName: "photo.HEIC", + mimeType: "image/heic", + fileSize: 20 * 1024 * 1024, + base64: jpeg, + width: 1, + height: 1, + }; + + it.each(["image/heic", "image/heif", undefined])( + "attaches the native JPEG conversion with matching metadata when the source MIME is %s", + async (mimeType) => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, mimeType }], + }); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(result).toEqual({ + images: [ + { + id: "attachment-id", + type: "image", + name: "photo.jpg", + mimeType: "image/jpeg", + sizeBytes: 4, + dataUrl: `data:image/jpeg;base64,${jpeg}`, + previewUri: `data:image/jpeg;base64,${jpeg}`, + }, + ], + error: null, + }); + }, + ); + + it.each([ + { extension: "png", mimeType: "image/png", base64: "iVBORw0KGgo=" }, + { extension: "gif", mimeType: "image/gif", base64: "R0lGODlh" }, + { extension: "webp", mimeType: "image/webp", base64: "UklGRgQAAABXRUJQ" }, + ])("preserves original $extension bytes instead of the picker's JPEG", async (original) => { + const name = `photo.${original.extension}`; + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, fileName: name, mimeType: original.mimeType }], + }); + mocks.readBase64.mockResolvedValue(original.base64); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(result.error).toBeNull(); + expect(result.images).toEqual([ + expect.objectContaining({ + name, + mimeType: original.mimeType, + dataUrl: `data:${original.mimeType};base64,${original.base64}`, + sizeBytes: Buffer.from(original.base64, "base64").byteLength, + }), + ]); + }); + + it("checks the converted JPEG size even when the HEIC source was smaller", async () => { + const oversized = + jpeg.slice(0, 4) + "A".repeat(Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4); + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, fileSize: 42, base64: oversized }], + }); + + await expect(pickComposerImages({ existingCount: 0 })).resolves.toEqual({ + images: [], + error: "'photo.HEIC' exceeds the 10 MB attachment limit.", + }); + }); + + it("does not relabel unconverted HEIC bytes as JPEG", async () => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, base64: "AAAAGGZ0eXBoZWlj" }], + }); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(result.images).toEqual([]); + expect(result.error).toContain("not a supported image type"); + }); + + it("retains a converted photo when another original cannot be read", async () => { + mocks.pickMedia.mockResolvedValue({ + canceled: false, + assets: [{ ...photo, fileName: "missing.gif", mimeType: "image/gif" }, photo], + }); + mocks.readBase64.mockRejectedValue(new Error("missing file")); + + const result = await pickComposerImages({ existingCount: 0 }); + + expect(result.images).toEqual([expect.objectContaining({ name: "photo.jpg" })]); + expect(result.error).toBe("Failed to read 'missing.gif'."); + }); + }); + describe("photo library videos", () => { const image: ImagePickerAsset = { uri: "file:///picker/photo.png", @@ -274,11 +386,11 @@ describe("composer file attachments", () => { it("copies picked files into app-owned storage without loading their contents", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/report.pdf", name: "report.pdf", - type: "application/pdf", + mimeType: "application/pdf", size: 42, }, ], @@ -303,14 +415,120 @@ describe("composer file attachments", () => { ); }); + it("preserves Android picker metadata instead of using the content URI document id", async () => { + const uri = "content://com.android.providers.media.documents/document/video%3A18"; + mocks.pickFile.mockResolvedValue({ + canceled: false, + assets: [ + { + uri, + name: "preview-h264.mp4", + mimeType: "video/mp4", + size: 620_992, + lastModified: 0, + }, + ], + }); + mocks.size.mockReturnValue(620_992); + + await expect(pickComposerFiles({ existingCount: 0 })).resolves.toEqual({ + files: [ + { + id: "attachment-id", + type: "file", + name: "preview-h264.mp4", + mimeType: "video/mp4", + sizeBytes: 620_992, + fileUri: "file:///documents/t3-composer-attachments/attachment-id-preview-h264.mp4", + }, + ], + error: null, + }); + expect(mocks.pickFile).toHaveBeenCalledWith({ multiple: true, copyToCacheDirectory: true }); + expect(mocks.copy).toHaveBeenCalledWith( + uri, + "file:///documents/t3-composer-attachments/attachment-id-preview-h264.mp4", + ); + expect(mocks.delete).not.toHaveBeenCalled(); + }); + + it("persists provider selections that require a readable cache copy", async () => { + const providerUri = "content://cloud-provider/documents/clip"; + const cachedUri = "file:///cache/DocumentPicker/clip.mp4"; + mocks.pickFile.mockImplementation(async (options) => ({ + canceled: false, + assets: [ + { + uri: options.copyToCacheDirectory ? cachedUri : providerUri, + name: "Cloud recording.mp4", + mimeType: "video/mp4", + size: 42, + lastModified: 0, + }, + ], + })); + mocks.copy.mockImplementation((uri: string) => { + if (uri === providerUri) throw new Error("The provider URI is not directly readable."); + }); + + const result = await pickComposerFiles({ existingCount: 0 }); + + expect(result.error).toBeNull(); + expect(result.files).toEqual([ + expect.objectContaining({ + name: "Cloud recording.mp4", + fileUri: "file:///documents/t3-composer-attachments/attachment-id-Cloud recording.mp4", + }), + ]); + expect(mocks.copy).toHaveBeenCalledWith(cachedUri, result.files[0]!.fileUri); + }); + + it("ends the foreground handoff when the picker is canceled without copying files", async () => { + mocks.pickFile.mockImplementation(async () => { + expect(isForegroundHandoffActive()).toBe(true); + return { canceled: true, assets: null }; + }); + + await expect(pickComposerFiles({ existingCount: 0 })).resolves.toEqual({ + files: [], + error: null, + }); + + expect(isForegroundHandoffActive()).toBe(false); + expect(mocks.copy).not.toHaveBeenCalled(); + expect(mocks.open).not.toHaveBeenCalled(); + }); + + it("reports picker failures and releases the foreground handoff", async () => { + mocks.pickFile.mockRejectedValue(new Error("The document provider is unavailable.")); + + await expect(pickComposerFiles({ existingCount: 0 })).resolves.toEqual({ + files: [], + error: "The document provider is unavailable.", + }); + + expect(isForegroundHandoffActive()).toBe(false); + expect(mocks.copy).not.toHaveBeenCalled(); + }); + + it("does not open the picker when the draft has no remaining attachment slots", async () => { + await expect(pickComposerFiles({ existingCount: 8 })).resolves.toEqual({ + files: [], + error: "You can attach up to 8 files per message.", + }); + + expect(mocks.pickFile).not.toHaveBeenCalled(); + expect(isForegroundHandoffActive()).toBe(false); + }); + it("falls back to a usable name when the picker reports a blank one", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/unnamed", name: " ", - type: "application/pdf", + mimeType: "application/pdf", size: 42, }, ], @@ -325,11 +543,11 @@ describe("composer file attachments", () => { it("rejects files that exceed the environment's advertised upload limit", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/archive.zip", name: "archive.zip", - type: "application/zip", + mimeType: "application/zip", size: 2 * 1024 * 1024, }, ], @@ -345,11 +563,11 @@ describe("composer file attachments", () => { it("never accepts files above the 50 MB contract limit", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/archive.zip", name: "archive.zip", - type: "application/zip", + mimeType: "application/zip", size: 51 * 1024 * 1024, }, ], @@ -366,11 +584,11 @@ describe("composer file attachments", () => { it("rejects a file that grew after the picker reported its size", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/archive.zip", name: "archive.zip", - type: "application/zip", + mimeType: "application/zip", size: 42, }, ], @@ -434,11 +652,11 @@ describe("composer file attachments", () => { mocks.size.mockReturnValue(0); mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/empty.txt", name: "empty.txt", - type: "text/plain", + mimeType: "text/plain", size: 0, }, ], @@ -450,7 +668,7 @@ describe("composer file attachments", () => { }); }); - it("copies an Android SAF file when the picker reports an unknown zero size", async () => { + it.each([0, undefined])("copies an Android SAF file when the picker size is %s", async (size) => { const reader = { readBytes: vi .fn() @@ -463,12 +681,12 @@ describe("composer file attachments", () => { mocks.open.mockImplementation((uri: string) => (uri.startsWith("content:") ? reader : writer)); mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "content://shared/report", name: "report.pdf", - type: "application/pdf", - size: 0, + mimeType: "application/pdf", + size, }, ], }); @@ -491,17 +709,17 @@ describe("composer file attachments", () => { it("uses the remaining slot for the first valid file after an oversized selection", async () => { mocks.pickFile.mockResolvedValue({ canceled: false, - result: [ + assets: [ { uri: "file:///downloads/huge.zip", name: "huge.zip", - type: "application/zip", + mimeType: "application/zip", size: 2 * 1024 * 1024, }, { uri: "file:///downloads/report.pdf", name: "report.pdf", - type: "application/pdf", + mimeType: "application/pdf", size: 42, }, ], @@ -557,6 +775,26 @@ describe("composer file attachments", () => { ]); }); + it("rechecks preview ownership after loading the native filesystem", async () => { + const fileName = "33333333-3333-4333-8333-333333333333-recording.mp4"; + const oldUri = `file:///private/var/mobile/Containers/Data/Application/11111111-1111-4111-8111-111111111111/Documents/t3-composer-attachments/${fileName}`; + mocks.documentUri = + "file:///var/mobile/Containers/Data/Application/22222222-2222-4222-8222-222222222222/Documents"; + const currentUri = `${mocks.documentUri}/t3-composer-attachments/${fileName}`; + + const deleting = removePersistedComposerAttachmentFile(oldUri); + const release = retainComposerAttachmentFile(currentUri, () => {}); + try { + await deleting; + expect(mocks.delete).not.toHaveBeenCalled(); + } finally { + release(); + } + + await removePersistedComposerAttachmentFile(oldUri); + expect(mocks.delete.mock.calls).toEqual([[currentUri]]); + }); + it("copies an open-in-place source from its actual container without rebasing it", async () => { const sourceUri = "file:///var/mobile/Containers/Data/Application/11111111-1111-4111-8111-111111111111/Documents/t3-composer-attachments/33333333-3333-4333-8333-333333333333-report.pdf"; diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index c19193150..77c2ec225 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -10,10 +10,11 @@ import { type EnvironmentId, type UploadChatImageAttachment, } from "@t3tools/contracts"; -import type { PickMultipleFilesResult } from "expo-file-system"; +import type { DocumentPickerResult } from "expo-document-picker"; import { estimateBase64ByteSize } from "./base64"; import { COMPOSER_ATTACHMENT_DIRECTORY, + isComposerAttachmentFileRetained, resolveOwnedComposerAttachmentFileUri, } from "./composerAttachmentFiles"; import { beginForegroundHandoff } from "./foreground-handoff"; @@ -22,6 +23,8 @@ import { uuidv4 } from "./uuid"; export interface DraftComposerImageAttachment extends UploadChatImageAttachment { readonly id: string; readonly previewUri: string; + readonly uploadedAttachmentId?: string; + readonly uploadEnvironmentId?: EnvironmentId; } export interface DraftComposerFileAttachment { @@ -145,7 +148,7 @@ export async function removePersistedComposerAttachmentFile(uri: string): Promis try { const { File, Paths } = await import("expo-file-system"); const ownedUri = resolveOwnedComposerAttachmentFileUri(uri, Paths.document.uri); - if (ownedUri === null) { + if (ownedUri === null || isComposerAttachmentFileRetained(ownedUri)) { return; } const file = new File(ownedUri); @@ -206,11 +209,18 @@ export async function pickComposerFiles(input: { }; } - const { File } = await import("expo-file-system"); + const { getDocumentAsync } = await import("expo-document-picker"); const endHandoff = beginForegroundHandoff(); - let result: PickMultipleFilesResult; + let result: DocumentPickerResult; try { - result = await File.pickFileAsync({ multipleFiles: true }); + // File providers may expose a URI that FileSystem cannot read directly. + // Import a readable cache copy before persisting the draft's owned file. + result = await getDocumentAsync({ multiple: true, copyToCacheDirectory: true }); + } catch (cause) { + return { + files: [], + error: cause instanceof Error ? cause.message : "Could not open the file picker.", + }; } finally { endHandoff(); } @@ -224,7 +234,7 @@ export async function pickComposerFiles(input: { const attachments: DraftComposerFileAttachment[] = []; let error: string | null = null; let exceededAttachmentLimit = false; - for (const file of result.result) { + for (const file of result.assets) { if (attachments.length >= remainingSlots) { exceededAttachmentLimit = true; break; @@ -238,7 +248,7 @@ export async function pickComposerFiles(input: { await createComposerFileAttachment({ uri: file.uri, name, - mimeType: file.type || "application/octet-stream", + mimeType: file.mimeType || "application/octet-stream", sizeBytes: file.size ?? null, maxBytes, }), @@ -343,7 +353,7 @@ export async function pickComposerMedia(input: { error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} attachments per message.`; break; } - const mimeType = asset.mimeType?.toLowerCase(); + let mimeType = asset.mimeType?.toLowerCase(); if (asset.type === "video" || mimeType?.startsWith("video/")) { if (input.maxVideoBytes === undefined) { error = "Video attachments are unavailable here."; @@ -367,35 +377,61 @@ export async function pickComposerMedia(input: { } continue; } - if (!mimeType?.startsWith("image/")) { + if (asset.type !== "image" && !mimeType?.startsWith("image/")) { error = `Unsupported file type for '${asset.fileName ?? "image"}'.`; continue; } - if (!isProviderSendTurnSupportedImageMimeType(mimeType)) { - error = `'${asset.fileName ?? "image"}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; - continue; - } - const base64 = asset.base64; + let base64 = asset.base64; if (!base64) { error = `Failed to read '${asset.fileName ?? "image"}'.`; continue; } - const sizeBytes = asset.fileSize ?? estimateBase64ByteSize(base64); + let name = asset.fileName?.trim() || "image"; + // The iOS picker returns JPEG base64 even when its metadata describes HEIC, + // PNG, or GIF. Keep supported originals so transparency and animation survive; + // use the native JPEG conversion for formats providers cannot accept. + if (base64.startsWith("/9j/")) { + if ( + mimeType && + mimeType !== "image/jpeg" && + isProviderSendTurnSupportedImageMimeType(mimeType) + ) { + try { + const { File } = await import("expo-file-system"); + base64 = await new File(asset.uri).base64(); + } catch { + error = `Failed to read '${name}'.`; + continue; + } + } else { + mimeType = "image/jpeg"; + if (!/\.jpe?g$/i.test(name)) { + name = `${name.replace(/\.[^.]+$/, "")}.jpg`; + } + } + } + if (!mimeType || !isProviderSendTurnSupportedImageMimeType(mimeType)) { + error = `'${name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`; + continue; + } + + const sizeBytes = estimateBase64ByteSize(base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { error = `'${asset.fileName ?? "image"}' exceeds the 10 MB attachment limit.`; continue; } + const dataUrl = `data:${mimeType};base64,${base64}`; attachments.push({ id: uuidv4(), type: "image", - name: asset.fileName ?? "image", + name, mimeType, sizeBytes, - dataUrl: `data:${mimeType};base64,${base64}`, - previewUri: asset.uri, + dataUrl, + previewUri: mimeType === asset.mimeType?.toLowerCase() ? asset.uri : dataUrl, }); } diff --git a/apps/mobile/src/lib/filePreview.test.ts b/apps/mobile/src/lib/filePreview.test.ts new file mode 100644 index 000000000..50be5369c --- /dev/null +++ b/apps/mobile/src/lib/filePreview.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { isPdfFile } from "./filePreview"; + +describe("PDF preview detection", () => { + it.each([ + [{ name: "download", mimeType: "application/pdf" }, true], + [{ name: "download", mimeType: "APPLICATION/PDF; charset=binary" }, true], + [{ name: "Report.PDF", mimeType: "application/octet-stream" }, true], + [{ name: "https://example.com/report.pdf?signature=abc#page=2" }, true], + [{ name: "report.pdf", mimeType: "text/plain" }, false], + [{ name: "report.pdf.exe" }, false], + [{ name: "https://example.com/page?download=report.pdf" }, false], + ])("classifies %j as %s", (file, expected) => { + expect(isPdfFile(file)).toBe(expected); + }); +}); diff --git a/apps/mobile/src/lib/filePreview.ts b/apps/mobile/src/lib/filePreview.ts new file mode 100644 index 000000000..7ee96476d --- /dev/null +++ b/apps/mobile/src/lib/filePreview.ts @@ -0,0 +1,6 @@ +/** MIME metadata wins; use the extension for files reported without a specific type. */ +export function isPdfFile(file: { readonly name: string; readonly mimeType?: string }): boolean { + const mimeType = file.mimeType?.split(";", 1)[0]?.trim().toLowerCase(); + if (mimeType && mimeType !== "application/octet-stream") return mimeType === "application/pdf"; + return /\.pdf$/i.test(file.name.split(/[?#]/, 1)[0] ?? ""); +} diff --git a/apps/mobile/src/lib/localAttachmentPreview.test.ts b/apps/mobile/src/lib/localAttachmentPreview.test.ts new file mode 100644 index 000000000..2ed83b26f --- /dev/null +++ b/apps/mobile/src/lib/localAttachmentPreview.test.ts @@ -0,0 +1,127 @@ +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ + retain: vi.fn(), + share: vi.fn(), + exists: vi.fn(), +})); + +vi.mock("../state/use-composer-drafts", () => ({ + retainComposerAttachmentFileForPreview: mocks.retain, +})); +vi.mock("./attachmentDownload", () => ({ shareLocalAttachment: mocks.share })); +vi.mock("expo-file-system", () => ({ + File: class { + constructor(readonly uri: string) {} + get exists(): boolean { + return mocks.exists(this.uri); + } + }, + Paths: { + document: { + uri: "file:///var/mobile/Containers/Data/Application/22222222-2222-4222-8222-222222222222/Documents/", + }, + }, +})); + +import { loadLocalAttachmentPreview } from "./localAttachmentPreview"; + +const attachment = { + type: "file" as const, + id: "draft-video", + name: "clip.mov", + mimeType: "video/quicktime", + sizeBytes: 12, + fileUri: + "file:///var/mobile/Containers/Data/Application/11111111-1111-4111-8111-111111111111/Documents/t3-composer-attachments/33333333-3333-4333-8333-333333333333-clip.mov", +}; + +beforeEach(() => { + mocks.retain.mockReset(); + mocks.share.mockReset(); + mocks.exists.mockReset(); + mocks.retain.mockImplementation(() => vi.fn()); + mocks.exists.mockReturnValue(true); + mocks.share.mockResolvedValue(undefined); +}); + +describe("loadLocalAttachmentPreview", () => { + it("retains and shares a PDF with its original filename and type", async () => { + const pdf = { ...attachment, name: "report.pdf", mimeType: "application/pdf" }; + const preview = await loadLocalAttachmentPreview(pdf, new AbortController().signal); + await preview!.share(new AbortController().signal); + expect(mocks.share).toHaveBeenCalledWith( + expect.objectContaining({ + attachment: { name: "report.pdf", mimeType: "application/pdf" }, + }), + ); + expect(mocks.retain.mock.results[0]!.value).not.toHaveBeenCalled(); + preview!.dispose(); + expect(mocks.retain.mock.results[0]!.value).toHaveBeenCalledTimes(1); + }); + it("resolves the current iOS container and releases its playback lease once", async () => { + const preview = await loadLocalAttachmentPreview(attachment, new AbortController().signal); + expect(preview?.uri).toContain("/22222222-2222-4222-8222-222222222222/Documents/"); + expect(mocks.retain).toHaveBeenCalledWith(attachment); + const release = mocks.retain.mock.results[0]!.value; + expect(release).not.toHaveBeenCalled(); + preview?.dispose(); + preview?.dispose(); + expect(release).toHaveBeenCalledTimes(1); + }); + + it.each([undefined, "share-button"])( + "keeps a separate share lease after playback closes (source: %s)", + async (sourceIdentifier) => { + const shared = Promise.withResolvers(); + mocks.share.mockReturnValue(shared.promise); + const preview = await loadLocalAttachmentPreview(attachment, new AbortController().signal); + const share = preview!.share(new AbortController().signal, sourceIdentifier); + expect(mocks.retain).toHaveBeenCalledTimes(2); + const releasePlayback = mocks.retain.mock.results[0]!.value; + const releaseShare = mocks.retain.mock.results[1]!.value; + preview!.dispose(); + expect(releasePlayback).toHaveBeenCalledTimes(1); + expect(releaseShare).not.toHaveBeenCalled(); + shared.resolve(); + await share; + expect(releaseShare).toHaveBeenCalledTimes(1); + }, + ); + + it("releases a failed share while keeping playback retained", async () => { + mocks.share.mockRejectedValue(new Error("Sharing unavailable")); + const preview = await loadLocalAttachmentPreview(attachment, new AbortController().signal); + await expect(preview!.share(new AbortController().signal)).rejects.toThrow( + "Sharing unavailable", + ); + expect(mocks.retain.mock.results[1]!.value).toHaveBeenCalledTimes(1); + expect(mocks.retain.mock.results[0]!.value).not.toHaveBeenCalled(); + preview!.dispose(); + }); + + it("releases a load canceled during native module loading", async () => { + const controller = new AbortController(); + const loading = loadLocalAttachmentPreview(attachment, controller.signal); + controller.abort(); + await expect(loading).resolves.toBeNull(); + expect(mocks.retain.mock.results[0]!.value).toHaveBeenCalledTimes(1); + expect(mocks.exists).not.toHaveBeenCalled(); + }); + + it("reports missing files and releases their lease", async () => { + mocks.exists.mockReturnValue(false); + await expect( + loadLocalAttachmentPreview(attachment, new AbortController().signal), + ).rejects.toThrow("This attachment is no longer available. Attach the file again."); + expect(mocks.retain.mock.results[0]!.value).toHaveBeenCalledTimes(1); + }); + + it("does not start sharing a disposed preview", async () => { + const preview = await loadLocalAttachmentPreview(attachment, new AbortController().signal); + preview!.dispose(); + await preview!.share(new AbortController().signal); + expect(mocks.share).not.toHaveBeenCalled(); + expect(mocks.retain).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/mobile/src/lib/localAttachmentPreview.ts b/apps/mobile/src/lib/localAttachmentPreview.ts new file mode 100644 index 000000000..bdd20e2e6 --- /dev/null +++ b/apps/mobile/src/lib/localAttachmentPreview.ts @@ -0,0 +1,59 @@ +import { videoMimeType } from "@t3tools/shared/video"; + +import type { DraftComposerFileAttachment } from "./composerImages"; +import { resolveOwnedComposerAttachmentFileUri } from "./composerAttachmentFiles"; +import { shareLocalAttachment, type AttachmentPreviewFile } from "./attachmentDownload"; +import { retainComposerAttachmentFileForPreview } from "../state/use-composer-drafts"; + +/** Retains the draft original for preview and gives each outgoing share its own lease. */ +export async function loadLocalAttachmentPreview( + attachment: DraftComposerFileAttachment, + signal: AbortSignal, +): Promise { + if (signal.aborted) return null; + const release = retainComposerAttachmentFileForPreview(attachment); + try { + const { File, Paths } = await import("expo-file-system"); + if (signal.aborted) { + release(); + return null; + } + const uri = + resolveOwnedComposerAttachmentFileUri(attachment.fileUri, Paths.document.uri) ?? + attachment.fileUri; + const file = new File(uri); + if (!file.exists) { + throw new Error("The local attachment file is missing."); + } + let disposed = false; + return { + uri: file.uri, + dispose: () => { + if (disposed) return; + disposed = true; + release(); + }, + share: async (shareSignal, sourceIdentifier) => { + if (disposed || shareSignal.aborted) return; + const releaseShare = retainComposerAttachmentFileForPreview(attachment); + try { + await shareLocalAttachment({ + uri: file.uri, + attachment: { + name: attachment.name, + mimeType: videoMimeType(attachment) ?? attachment.mimeType, + }, + signal: shareSignal, + sourceIdentifier, + }); + } finally { + releaseShare(); + } + }, + }; + } catch (cause) { + release(); + if (signal.aborted) return null; + throw new Error("This attachment is no longer available. Attach the file again.", { cause }); + } +} diff --git a/apps/mobile/src/lib/projectThreadStartTurn.ts b/apps/mobile/src/lib/projectThreadStartTurn.ts index 75a84a906..aac1abc4b 100644 --- a/apps/mobile/src/lib/projectThreadStartTurn.ts +++ b/apps/mobile/src/lib/projectThreadStartTurn.ts @@ -2,15 +2,14 @@ import { CommandId, MessageId, ThreadId, - type ChatFileAttachment, type ModelSelection, type ProjectId, type ProviderInteractionMode, type RuntimeMode, - type UploadChatImageAttachment, } from "@t3tools/contracts"; import { toUploadChatImageAttachments, type DraftComposerAttachment } from "./composerImages"; +import type { UploadedMobileAttachment } from "./attachmentUpload"; export function deriveThreadTitleFromPrompt(value: string): string { const trimmed = value.trim(); @@ -31,7 +30,7 @@ export interface ProjectThreadStartTurnSpec { readonly createdAt: string; readonly text: string; readonly attachments: ReadonlyArray; - readonly uploadedAttachments?: ReadonlyArray; + readonly uploadedAttachments?: ReadonlyArray; readonly modelSelection: ModelSelection; readonly runtimeMode: RuntimeMode; readonly interactionMode: ProviderInteractionMode; diff --git a/apps/mobile/src/lib/shareFileFromSource.ios.ts b/apps/mobile/src/lib/shareFileFromSource.ios.ts new file mode 100644 index 000000000..5de0e0cbd --- /dev/null +++ b/apps/mobile/src/lib/shareFileFromSource.ios.ts @@ -0,0 +1,14 @@ +import { requireNativeModule } from "expo"; +import type { SharingOptions } from "expo-sharing"; + +const NativeControls = requireNativeModule<{ + shareFileFromSource(uri: string, title: string, sourceIdentifier: string): Promise; +}>("T3NativeControls"); + +export function shareFileFromSource( + uri: string, + options: SharingOptions, + sourceIdentifier: string, +) { + return NativeControls.shareFileFromSource(uri, options.dialogTitle ?? "", sourceIdentifier); +} diff --git a/apps/mobile/src/lib/shareFileFromSource.ts b/apps/mobile/src/lib/shareFileFromSource.ts new file mode 100644 index 000000000..5e806612a --- /dev/null +++ b/apps/mobile/src/lib/shareFileFromSource.ts @@ -0,0 +1,9 @@ +import { shareAsync, type SharingOptions } from "expo-sharing"; + +export function shareFileFromSource( + uri: string, + options: SharingOptions, + _sourceIdentifier: string, +) { + return shareAsync(uri, options); +} diff --git a/apps/mobile/src/lib/videoThumbnails.test.ts b/apps/mobile/src/lib/videoThumbnails.test.ts new file mode 100644 index 000000000..e577e448e --- /dev/null +++ b/apps/mobile/src/lib/videoThumbnails.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ createPlayer: vi.fn() })); +vi.mock("expo-video", () => ({ createVideoPlayer: mocks.createPlayer })); + +let thumbnails: typeof import("./videoThumbnails"); +const frame = { width: 480, height: 270 }; +const player = () => ({ + replaceAsync: vi.fn(async (): Promise => {}), + generateThumbnailsAsync: vi.fn(async () => [frame]), + release: vi.fn(), +}); +const source = () => ({ uri: "file:///clip.mp4", dispose: vi.fn() }); + +beforeEach(async () => { + vi.resetModules(); + mocks.createPlayer.mockReset().mockImplementation(player); + thumbnails = await import("./videoThumbnails"); +}); + +afterEach(() => vi.useRealTimers()); + +describe("video thumbnails", () => { + it("reuses a frame for duplicate requests and refreshed signed URLs", async () => { + const file = source(); + const resolveSource = vi.fn(async () => file); + const signal = new AbortController().signal; + const results = await Promise.all([ + thumbnails.loadVideoThumbnail("env:clip", resolveSource, signal), + thumbnails.loadVideoThumbnail("env:clip", resolveSource, signal), + ]); + expect(results).toEqual([frame, frame]); + expect(resolveSource).toHaveBeenCalledTimes(1); + expect(mocks.createPlayer).toHaveBeenCalledTimes(1); + expect(file.dispose).toHaveBeenCalledTimes(1); + const refreshed = vi.fn(async () => ({ ...source(), uri: "https://host/new-token/clip.mp4" })); + expect(await thumbnails.loadVideoThumbnail("env:clip", refreshed, signal)).toBe(frame); + expect(refreshed).not.toHaveBeenCalled(); + }); + + it("serializes decoding and skips queued requests that scroll out of view", async () => { + const started = Promise.withResolvers(); + const generated = Promise.withResolvers<(typeof frame)[]>(); + const first = player(); + first.generateThumbnailsAsync.mockImplementation(() => { + started.resolve(); + return generated.promise; + }); + mocks.createPlayer.mockReturnValueOnce(first); + const firstRequest = thumbnails.loadVideoThumbnail( + "first", + async () => source(), + new AbortController().signal, + ); + await started.promise; + const removed = new AbortController(); + const skipped = vi.fn(async () => source()); + const queued = thumbnails.loadVideoThumbnail("removed", skipped, removed.signal); + const next = vi.fn(async () => source()); + const nextRequest = thumbnails.loadVideoThumbnail("next", next, new AbortController().signal); + expect(next).not.toHaveBeenCalled(); + removed.abort(); + generated.resolve([frame]); + expect(await firstRequest).toBe(frame); + expect(await queued).toBeNull(); + expect(await nextRequest).toBe(frame); + expect(skipped).not.toHaveBeenCalled(); + expect(first.release).toHaveBeenCalledTimes(1); + }); + + it("releases an active canceled player and ignores late source loading", async () => { + const started = Promise.withResolvers(); + const replaced = Promise.withResolvers(); + const first = player(); + first.replaceAsync.mockImplementation(() => { + started.resolve(); + return replaced.promise; + }); + mocks.createPlayer.mockReturnValueOnce(first); + const file = source(); + const controller = new AbortController(); + const request = thumbnails.loadVideoThumbnail("canceled", async () => file, controller.signal); + await started.promise; + controller.abort(); + expect(await request).toBeNull(); + expect(first.release).toHaveBeenCalledTimes(1); + expect(file.dispose).toHaveBeenCalledTimes(1); + replaced.resolve(); + expect( + await thumbnails.loadVideoThumbnail( + "next", + async () => source(), + new AbortController().signal, + ), + ).toBe(frame); + expect(first.generateThumbnailsAsync).not.toHaveBeenCalled(); + expect(thumbnails.cachedVideoThumbnail("canceled")).toBeNull(); + }); + + it("releases failed extractions and permits a later retry", async () => { + const broken = player(); + broken.generateThumbnailsAsync.mockRejectedValue(new Error("Invalid video")); + mocks.createPlayer.mockReturnValueOnce(broken); + const file = source(); + expect( + await thumbnails.loadVideoThumbnail("retry", async () => file, new AbortController().signal), + ).toBeNull(); + expect(broken.release).toHaveBeenCalledTimes(1); + expect(file.dispose).toHaveBeenCalledTimes(1); + expect( + await thumbnails.loadVideoThumbnail( + "retry", + async () => source(), + new AbortController().signal, + ), + ).toBe(frame); + }); + + it("does not let an unreachable source block the queue indefinitely", async () => { + vi.useFakeTimers(); + const started = Promise.withResolvers(); + const first = player(); + first.replaceAsync.mockImplementation(() => { + started.resolve(); + return new Promise(() => {}); + }); + mocks.createPlayer.mockReturnValueOnce(first); + const file = source(); + const request = thumbnails.loadVideoThumbnail( + "unreachable", + async () => file, + new AbortController().signal, + ); + await started.promise; + await vi.advanceTimersByTimeAsync(15_000); + expect(await request).toBeNull(); + expect(first.release).toHaveBeenCalledTimes(1); + expect(file.dispose).toHaveBeenCalledTimes(1); + expect( + await thumbnails.loadVideoThumbnail( + "reachable", + async () => source(), + new AbortController().signal, + ), + ).toBe(frame); + }); + + it("bounds the retained native images without invalidating frames still displayed", async () => { + for (let i = 0; i < 33; i++) { + await thumbnails.loadVideoThumbnail( + `clip:${i}`, + async () => source(), + new AbortController().signal, + ); + } + expect(thumbnails.cachedVideoThumbnail("clip:0")).toBeNull(); + expect(thumbnails.cachedVideoThumbnail("clip:32")).toBe(frame); + expect(mocks.createPlayer).toHaveBeenCalledTimes(33); + expect( + await thumbnails.loadVideoThumbnail( + "clip:0", + async () => source(), + new AbortController().signal, + ), + ).toBe(frame); + expect(mocks.createPlayer).toHaveBeenCalledTimes(34); + }); +}); diff --git a/apps/mobile/src/lib/videoThumbnails.ts b/apps/mobile/src/lib/videoThumbnails.ts new file mode 100644 index 000000000..927e1174a --- /dev/null +++ b/apps/mobile/src/lib/videoThumbnails.ts @@ -0,0 +1,81 @@ +import type { VideoThumbnail } from "expo-video"; + +import type { AttachmentPreviewFile } from "./attachmentDownload"; + +const thumbnails = new Map(); +const MAX_CACHED_THUMBNAILS = 32; +let pending: Promise = Promise.resolve(); + +export function cachedVideoThumbnail(key: string): VideoThumbnail | null { + return thumbnails.get(key) ?? null; +} + +async function extractFrame(uri: string, signal: AbortSignal) { + const { createVideoPlayer } = await import("expo-video"); + if (signal.aborted) return null; + const player = createVideoPlayer(null); + let disposed = false; + let cancel = () => {}; + let timeout: ReturnType | undefined; + try { + // Never play or change audio settings: thumbnails must leave the shared audio session alone. + player.bufferOptions = { preferredForwardBufferDuration: 1 }; + const canceled = new Promise((resolve) => { + cancel = () => resolve(null); + }); + signal.addEventListener("abort", cancel, { once: true }); + // An unreachable environment must not hold up thumbnails for other environments. + timeout = setTimeout(cancel, 15_000); + const frame = (async () => { + await player.replaceAsync({ uri, contentType: "progressive" }); + if (disposed || signal.aborted) return null; + const [thumbnail] = await player.generateThumbnailsAsync([0], { + maxWidth: 480, + maxHeight: 480, + }); + return thumbnail ?? null; + })(); + return await Promise.race([frame, canceled]); + } finally { + disposed = true; + clearTimeout(timeout); + signal.removeEventListener("abort", cancel); + player.release(); + } +} + +/** Serializes frame extraction and releases each temporary player and local-file lease. */ +export function loadVideoThumbnail( + key: string, + resolveSource: ( + signal: AbortSignal, + ) => Promise | null>, + signal: AbortSignal, +): Promise { + if (signal.aborted) return Promise.resolve(null); + const cached = cachedVideoThumbnail(key); + if (cached) return Promise.resolve(cached); + const load = pending + .then(async () => { + if (signal.aborted) return null; + const cached = cachedVideoThumbnail(key); + if (cached) return cached; + + const source = await resolveSource(signal); + if (!source) return null; + try { + const thumbnail = await extractFrame(source.uri, signal); + if (!thumbnail || signal.aborted) return null; + thumbnails.set(key, thumbnail); + if (thumbnails.size > MAX_CACHED_THUMBNAILS) { + thumbnails.delete(thumbnails.keys().next().value!); + } + return thumbnail; + } finally { + source.dispose(); + } + }) + .catch(() => null); + pending = load; + return load; +} diff --git a/apps/mobile/src/state/composer-attachment-uploads.ts b/apps/mobile/src/state/composer-attachment-uploads.ts new file mode 100644 index 000000000..efc3fe1c3 --- /dev/null +++ b/apps/mobile/src/state/composer-attachment-uploads.ts @@ -0,0 +1,126 @@ +import { useAtomValue } from "@effect/atom-react"; +import type { EnvironmentId } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; +import { useEffect, useRef } from "react"; + +import { prepareTurnAttachments } from "../lib/attachmentUpload"; +import { + composerAttachmentUploadKey, + composerDraftEnvironmentId, + canUploadComposerAttachment, + createComposerAttachmentUploadQueue, + type ComposerAttachmentUploadState, +} from "../lib/composerAttachmentUploadQueue"; +import { appAtomRegistry } from "./atom-registry"; +import { useServerConfigs } from "./entities"; +import { flattenQueuedThreadMessages, threadOutboxManager } from "./thread-outbox"; +import { useThreadOutboxMessages } from "./use-thread-outbox"; +import { + composerDraftsAtom, + ensureComposerDraftsLoaded, + flushComposerDrafts, + retainComposerAttachmentFileForPreview, + setComposerDraftAttachmentUpload, +} from "./use-composer-drafts"; +import { useRemoteConnectionStatus } from "./use-remote-environment-registry"; + +export { composerAttachmentUploadBlockReason } from "../lib/composerAttachmentUploadQueue"; + +export const composerAttachmentUploadsAtom = Atom.make< + Readonly> +>({}).pipe(Atom.keepAlive); +const uploadStateAtom = Atom.family((key: string) => + Atom.map(composerAttachmentUploadsAtom, (states) => states[key]), +); +let uploadQueue: ReturnType | null = null; + +export function useComposerAttachmentUploadState( + environmentId: EnvironmentId | undefined, + attachmentId: string, +) { + return useAtomValue( + uploadStateAtom(environmentId ? composerAttachmentUploadKey(environmentId, attachmentId) : ""), + ); +} + +export function retryComposerAttachmentUpload(environmentId: EnvironmentId, attachmentId: string) { + uploadQueue?.retry(environmentId, attachmentId); +} + +/** Runs outside mounted composers so a transfer can finish after navigation. */ +export function useComposerAttachmentUploadWorker() { + const drafts = useAtomValue(composerDraftsAtom); + const queuedMessages = useThreadOutboxMessages(); + const serverConfigs = useServerConfigs(); + const { connectedEnvironments } = useRemoteConnectionStatus(); + const queueRef = useRef | null>(null); + + useEffect(() => { + ensureComposerDraftsLoaded(); + const queue = createComposerAttachmentUploadQueue({ + onChange: (states) => appAtomRegistry.set(composerAttachmentUploadsAtom, states), + upload: async ({ environmentId, attachment }, signal, onProgress) => { + const release = + attachment.type === "file" + ? retainComposerAttachmentFileForPreview(attachment) + : undefined; + try { + const result = await prepareTurnAttachments({ + environmentId, + attachments: [attachment], + supportsImageUploads: true, + signal, + onUploadProgress: (_, progress) => onProgress(progress), + persistUploadedReferences: async ([uploaded]) => { + if (signal.aborted || !uploaded) return "abandon"; + const queued = flattenQueuedThreadMessages( + appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom), + ); + let retained = false; + for (const [key, draft] of Object.entries(appAtomRegistry.get(composerDraftsAtom))) { + if ( + composerDraftEnvironmentId(key, queued) === environmentId && + draft.attachments.some((candidate) => candidate.id === attachment.id) + ) { + retained = setComposerDraftAttachmentUpload(key, uploaded) || retained; + } + } + if (!retained) return "abandon"; + await flushComposerDrafts(); + return "persisted"; + }, + }); + return result.status === "ready"; + } finally { + release?.(); + } + }, + }); + queueRef.current = queue; + uploadQueue = queue; + return () => { + queue.dispose(); + if (uploadQueue === queue) uploadQueue = null; + queueRef.current = null; + }; + }, []); + + useEffect(() => { + const queued = flattenQueuedThreadMessages(queuedMessages); + const connected = new Set( + connectedEnvironments + .filter((environment) => environment.connectionState === "connected") + .map((environment) => environment.environmentId), + ); + const requests = Object.entries(drafts).flatMap(([key, draft]) => { + const environmentId = composerDraftEnvironmentId(key, queued); + if (environmentId === null || !connected.has(environmentId)) return []; + return draft.attachments + .filter((attachment) => + canUploadComposerAttachment(attachment, serverConfigs.get(environmentId)), + ) + .map((attachment) => ({ environmentId, attachment })); + }); + queueRef.current?.sync(requests); + }, [connectedEnvironments, drafts, queuedMessages, serverConfigs]); +} diff --git a/apps/mobile/src/state/use-composer-drafts.test.ts b/apps/mobile/src/state/use-composer-drafts.test.ts index 558f4ef8f..0bb34c96e 100644 --- a/apps/mobile/src/state/use-composer-drafts.test.ts +++ b/apps/mobile/src/state/use-composer-drafts.test.ts @@ -118,10 +118,12 @@ import { appAtomRegistry } from "./atom-registry"; import { threadOutboxManager } from "./thread-outbox"; import { appendComposerDraftAttachments, + archiveCloudComposerDrafts, clearComposerDraftContentState, clearComposerDraftsEnvironment, ComposerDraftPersistenceError, composerDraftsAtom, + composerCloudDraftsAtom, copyComposerDraftContentIfEmpty, copyComposerDraftContentState, decodePersistedComposerState, @@ -134,9 +136,13 @@ import { releaseUnusedComposerAttachmentFiles, removeComposerDraftsForEnvironment, resetComposerDraftsLoadState, + retainComposerAttachmentFileForPreview, restoreComposerDraftSnapshotState, + restoreCloudComposerDrafts, restorePendingSendComposerDraftState, setComposerDraftText, + setComposerDraftAttachmentUpload, + waitForComposerDraftsLoaded, setStickyComposerModelSelection, stickyComposerModelSelectionAtom, undoComposerDraftMerge, @@ -157,6 +163,7 @@ afterEach(() => { composerDraftFileMocks.setOnWrite(null); composerDraftFileMocks.resetWrites(); appAtomRegistry.set(composerDraftsAtom, {}); + appAtomRegistry.set(composerCloudDraftsAtom, { accountId: null, signedOut: {} }); appAtomRegistry.set(stickyComposerModelSelectionAtom, null); appAtomRegistry.set(threadOutboxManager.queuedMessagesByThreadKeyAtom, {}); composerAttachmentCleanupMocks.remove.mockClear(); @@ -320,6 +327,226 @@ describe("mobile composer drafts", () => { expect(composerAttachmentCleanupMocks.releaseUploads).not.toHaveBeenCalled(); }); + it("retains offline image bytes and newer edits when an early upload finishes", async () => { + const key = "environment-1:thread-1"; + const image = { + id: "photo", + type: "image" as const, + name: "photo.png", + mimeType: "image/png", + sizeBytes: 3, + dataUrl: "data:image/png;base64,YWJj", + previewUri: "file:///photo.png", + }; + const second = { ...image, id: "second", name: "second.png" }; + const uploaded = { + ...image, + uploadedAttachmentId: "pending-photo", + uploadEnvironmentId: EnvironmentId.make("environment-1"), + }; + composerDraftFileMocks.setDocument({ schemaVersion: 1, drafts: {} }); + appendComposerDraftAttachments(key, [image]); + setComposerDraftText(key, "Edited while uploading"); + appendComposerDraftAttachments(key, [second]); + expect(setComposerDraftAttachmentUpload(key, uploaded)).toBe(true); + await flushComposerDrafts(); + + appAtomRegistry.set(composerDraftsAtom, {}); + resetComposerDraftsLoadState(); + await waitForComposerDraftsLoaded(); + expect(getComposerDraftSnapshot(key)).toMatchObject({ + text: "Edited while uploading", + attachments: [uploaded, second], + }); + expect(setComposerDraftAttachmentUpload(key, { ...uploaded, id: "removed-photo" })).toBe(false); + expect(getComposerDraftSnapshot(key).attachments).toHaveLength(2); + }); + + it("cleans up an unreferenced image upload even when there is no local file URI", async () => { + const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => outboxLoad.mockRestore()); + const environmentId = EnvironmentId.make("environment-1"); + await releaseUnusedComposerAttachmentFiles([ + { + id: "photo", + type: "image", + name: "photo.png", + mimeType: "image/png", + sizeBytes: 3, + dataUrl: "data:image/png;base64,YWJj", + previewUri: "file:///photo.png", + uploadedAttachmentId: "pending-photo", + uploadEnvironmentId: environmentId, + }, + ]); + expect(composerAttachmentCleanupMocks.releaseUploads).toHaveBeenCalledWith(environmentId, [ + "pending-photo", + ]); + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + }); + + it("keeps signed-out files through cleanup and restart, and restores only the owning account", async () => { + const load = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => load.mockRestore()); + await waitForComposerDraftsLoaded(); + const environmentId = EnvironmentId.make("cloud-environment"); + const key = `${environmentId}:thread-1`; + const file = { + id: "local-pdf", + type: "file" as const, + name: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 42, + fileUri: "file:///documents/t3-composer-attachments/notes.pdf", + uploadEnvironmentId: environmentId, + uploadedAttachmentId: "pending-pdf", + }; + const queued = { + environmentId, + threadId: ThreadId.make("thread-2"), + messageId: MessageId.make("queued-1"), + commandId: CommandId.make("command-1"), + text: "Send later", + attachments: [file], + createdAt: "2026-08-31T12:00:00.000Z", + }; + appAtomRegistry.set(composerDraftsAtom, { + [key]: { text: "Unsent notes", attachments: [file] }, + "direct-environment:thread-1": DRAFT, + "pending-task:queued-1": { text: "Edited queued task", attachments: [file] }, + }); + appAtomRegistry.set(threadOutboxManager.queuedMessagesByThreadKeyAtom, { queued: [queued] }); + await archiveCloudComposerDrafts("account-a", new Set([environmentId])); + expect(appAtomRegistry.get(composerDraftsAtom)).toEqual({ + "direct-environment:thread-1": DRAFT, + }); + // The registry can remove the active outbox and drafts after the backup lands. + appAtomRegistry.set(threadOutboxManager.queuedMessagesByThreadKeyAtom, {}); + await clearComposerDraftsEnvironment(environmentId); + await releaseUnusedComposerAttachmentFiles([file]); + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + expect(composerAttachmentCleanupMocks.releaseUploads).not.toHaveBeenCalled(); + + appAtomRegistry.set(composerDraftsAtom, {}); + appAtomRegistry.set(composerCloudDraftsAtom, { accountId: null, signedOut: {} }); + resetComposerDraftsLoadState(); + await waitForComposerDraftsLoaded(); + await restoreCloudComposerDrafts("account-b"); + expect(getComposerDraftSnapshot(key).attachments).toEqual([]); + expect(appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom)).toEqual({}); + const enqueue = vi.spyOn(threadOutboxManager, "enqueue").mockResolvedValue(); + onTestFinished(() => enqueue.mockRestore()); + await restoreCloudComposerDrafts("account-a"); + expect(getComposerDraftSnapshot(key)).toEqual({ text: "Unsent notes", attachments: [file] }); + expect(getComposerDraftSnapshot("pending-task:queued-1").text).toBe("Edited queued task"); + expect(enqueue).toHaveBeenCalledExactlyOnceWith(queued); + expect(appAtomRegistry.get(composerCloudDraftsAtom).signedOut).toEqual({}); + const persisted = decodePersistedComposerState( + JSON.parse(composerDraftFileMocks.getDocument()), + ); + expect(persisted.drafts[key]?.attachments).toEqual([file]); + expect(persisted.cloudDrafts.accountId).toBe("account-a"); + }); + + it("fails sign-out preservation before cleanup if a durable backup cannot be written", async () => { + const load = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => load.mockRestore()); + await waitForComposerDraftsLoaded(); + appAtomRegistry.set(composerDraftsAtom, { "environment-1:thread-1": DRAFT }); + composerDraftFileMocks.setWriteError(new Error("Storage is full")); + await expect( + archiveCloudComposerDrafts("account-a", new Set([EnvironmentId.make("environment-1")])), + ).rejects.toThrow(); + expect( + appAtomRegistry.get(composerCloudDraftsAtom).signedOut["account-a"]?.drafts[ + "environment-1:thread-1" + ], + ).toEqual(DRAFT); + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + composerDraftFileMocks.setWriteError(null); + await archiveCloudComposerDrafts(null, new Set([EnvironmentId.make("environment-1")])); + expect( + decodePersistedComposerState(JSON.parse(composerDraftFileMocks.getDocument())).cloudDrafts + .signedOut["account-a"]?.drafts["environment-1:thread-1"], + ).toEqual(DRAFT); + }); + + it("keeps a removed file until both playback and a share copy finish", async () => { + const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => outboxLoad.mockRestore()); + const fileName = "33333333-3333-4333-8333-333333333333-recording.mp4"; + const file = { + id: "file-preview", + type: "file" as const, + name: "recording.mp4", + mimeType: "video/mp4", + sizeBytes: 42, + fileUri: `file:///private/var/mobile/Containers/Data/Application/11111111-1111-4111-8111-111111111111/Documents/t3-composer-attachments/${fileName}`, + }; + const currentFile = { + ...file, + fileUri: `file:///var/mobile/Containers/Data/Application/22222222-2222-4222-8222-222222222222/Documents/t3-composer-attachments/${fileName}`, + }; + const releasePlayback = retainComposerAttachmentFileForPreview(file); + const releaseShareCopy = retainComposerAttachmentFileForPreview(currentFile); + onTestFinished(releasePlayback); + onTestFinished(releaseShareCopy); + + await releaseUnusedComposerAttachmentFiles([currentFile]); + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + + releasePlayback(); + releasePlayback(); + await releaseUnusedComposerAttachmentFiles([file]); + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + + const deleted = Promise.withResolvers(); + composerAttachmentCleanupMocks.remove.mockImplementationOnce(async () => { + deleted.resolve(); + return undefined; + }); + releaseShareCopy(); + await deleted.promise; + + expect(composerAttachmentCleanupMocks.remove.mock.calls).toEqual([[currentFile.fileUri]]); + }); + + it("preserves a preview opened while cleanup is checking the incoming inbox", async () => { + const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => outboxLoad.mockRestore()); + const file = { + id: "file-opening-preview", + type: "file" as const, + name: "recording.mp4", + mimeType: "video/mp4", + sizeBytes: 42, + fileUri: "file:///documents/t3-composer-attachments/recording.mp4", + }; + const ownershipReadStarted = Promise.withResolvers(); + const ownershipRead = Promise.withResolvers<[]>(); + incomingShareStorageMocks.load.mockImplementationOnce(() => { + ownershipReadStarted.resolve(); + return ownershipRead.promise; + }); + + const cleanup = releaseUnusedComposerAttachmentFiles([file]); + await ownershipReadStarted.promise; + const release = retainComposerAttachmentFileForPreview(file); + onTestFinished(release); + ownershipRead.resolve([]); + await cleanup; + expect(composerAttachmentCleanupMocks.remove).not.toHaveBeenCalled(); + + const deleted = Promise.withResolvers(); + composerAttachmentCleanupMocks.remove.mockImplementationOnce(async () => { + deleted.resolve(); + return undefined; + }); + release(); + await deleted.promise; + expect(composerAttachmentCleanupMocks.remove.mock.calls).toEqual([[file.fileUri]]); + }); + it("removes an unreferenced local file and its pending upload", async () => { const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); onTestFinished(() => outboxLoad.mockRestore()); diff --git a/apps/mobile/src/state/use-composer-drafts.ts b/apps/mobile/src/state/use-composer-drafts.ts index 64dc26e14..10f82b851 100644 --- a/apps/mobile/src/state/use-composer-drafts.ts +++ b/apps/mobile/src/state/use-composer-drafts.ts @@ -15,11 +15,22 @@ import { Atom } from "effect/unstable/reactivity"; import { writeFileAtomically } from "../lib/atomic-file"; import { DraftComposerAttachmentSchema } from "../lib/composer-image-schema"; -import { composerAttachmentFileReferenceKey } from "../lib/composerAttachmentFiles"; -import type { DraftComposerAttachment } from "../lib/composerImages"; +import { + composerAttachmentFileReferenceKey, + isComposerAttachmentFileRetained, + retainComposerAttachmentFile, +} from "../lib/composerAttachmentFiles"; +import type { DraftComposerAttachment, DraftComposerFileAttachment } from "../lib/composerImages"; import { SerializedAsyncQueue } from "../lib/serialized-async-queue"; import { appAtomRegistry } from "./atom-registry"; +import { + decodeQueuedThreadMessage, + encodeQueuedThreadMessage, + QueuedThreadMessageSchema, + type QueuedThreadMessage, +} from "./thread-outbox-model"; import { flushThreadOutbox, threadOutboxManager } from "./thread-outbox"; +import { composerDraftEnvironmentId } from "../lib/composerAttachmentUploadQueue"; const COMPOSER_DRAFTS_SCHEMA_VERSION = 1; const COMPOSER_DRAFTS_DIRECTORY = "composer-drafts"; @@ -105,6 +116,16 @@ const PersistedComposerDraftsSchema = Schema.Struct({ schemaVersion: Schema.Literal(COMPOSER_DRAFTS_SCHEMA_VERSION), drafts: Schema.Record(Schema.String, ComposerDraftSchema), stickyModelSelection: Schema.optional(ModelSelectionSchema), + cloudAccountId: Schema.optional(Schema.String), + signedOutDrafts: Schema.optional( + Schema.Record( + Schema.String, + Schema.Struct({ + drafts: Schema.Record(Schema.String, ComposerDraftSchema), + queuedMessages: Schema.Array(QueuedThreadMessageSchema), + }), + ), + ), }); const decodePersistedComposerDraftsDocument = Schema.decodeUnknownSync( @@ -126,6 +147,21 @@ export const stickyComposerModelSelectionAtom = Atom.make Atom.withLabel("mobile:sticky-composer-model-selection"), ); +interface SignedOutDrafts { + readonly drafts: Record; + readonly queuedMessages: ReadonlyArray; +} + +interface ComposerCloudDraftState { + readonly accountId: string | null; + readonly signedOut: Record; +} + +export const composerCloudDraftsAtom = Atom.make({ + accountId: null, + signedOut: {}, +}).pipe(Atom.keepAlive); + let loadPromise: Promise | null = null; let persistTimer: ReturnType | null = null; const persistenceQueue = new SerializedAsyncQueue(); @@ -169,6 +205,7 @@ function isEmptyDraft(draft: ComposerDraft): boolean { export function decodePersistedComposerState(value: unknown): { readonly drafts: Record; readonly stickyModelSelection: ModelSelection | null; + readonly cloudDrafts: ComposerCloudDraftState; } { const parsed = decodePersistedComposerDraftsDocument(value); return { @@ -202,6 +239,18 @@ export function decodePersistedComposerState(value: unknown): { .filter(([, draft]) => !isEmptyDraft(draft) || (draft.importedShareIds?.length ?? 0) > 0), ), stickyModelSelection: parsed.stickyModelSelection ?? null, + cloudDrafts: { + accountId: parsed.cloudAccountId ?? null, + signedOut: Object.fromEntries( + Object.entries(parsed.signedOutDrafts ?? {}).map(([id, saved]) => [ + id, + { + drafts: saved.drafts, + queuedMessages: saved.queuedMessages.map(decodeQueuedThreadMessage), + }, + ]), + ), + }, }; } @@ -216,15 +265,18 @@ async function getComposerDraftsFile() { return new File(directory, COMPOSER_DRAFTS_FILE); } -async function loadPersistedComposerState(): Promise<{ - readonly drafts: Record; - readonly stickyModelSelection: ModelSelection | null; -}> { +async function loadPersistedComposerState(): Promise< + ReturnType +> { let operation: ComposerDraftPersistenceError["operation"] = "open"; try { const file = await getComposerDraftsFile(); if (!file.exists) { - return { drafts: {}, stickyModelSelection: null }; + return { + drafts: {}, + stickyModelSelection: null, + cloudDrafts: { accountId: null, signedOut: {} }, + }; } operation = "read"; const raw = await file.text(); @@ -240,13 +292,18 @@ async function loadPersistedComposerState(): Promise<{ cause, }), ); - return { drafts: {}, stickyModelSelection: null }; + return { + drafts: {}, + stickyModelSelection: null, + cloudDrafts: { accountId: null, signedOut: {} }, + }; } } async function writePersistedComposerState( drafts: Record, stickyModelSelection: ModelSelection | null, + cloudDrafts = appAtomRegistry.get(composerCloudDraftsAtom), ): Promise { let operation: ComposerDraftPersistenceError["operation"] = "open"; try { @@ -259,6 +316,20 @@ async function writePersistedComposerState( schemaVersion: COMPOSER_DRAFTS_SCHEMA_VERSION, drafts: nonEmptyDrafts, ...(stickyModelSelection ? { stickyModelSelection } : {}), + ...(cloudDrafts.accountId ? { cloudAccountId: cloudDrafts.accountId } : {}), + ...(Object.keys(cloudDrafts.signedOut).length > 0 + ? { + signedOutDrafts: Object.fromEntries( + Object.entries(cloudDrafts.signedOut).map(([id, saved]) => [ + id, + { + drafts: saved.drafts, + queuedMessages: saved.queuedMessages.map(encodeQueuedThreadMessage), + }, + ]), + ), + } + : {}), } as const; const encoded = JSON.stringify(document); operation = "write"; @@ -304,13 +375,23 @@ export async function flushComposerDrafts(): Promise { } while (persistTimer !== null); } +function signedOutAttachmentOwners() { + return Object.values(appAtomRegistry.get(composerCloudDraftsAtom).signedOut).flatMap((saved) => [ + ...Object.values(saved.drafts), + ...saved.queuedMessages, + ]); +} + function isComposerAttachmentFileReferenced(fileUri: string): boolean { + if (isComposerAttachmentFileRetained(fileUri)) { + return true; + } const referenceKey = composerAttachmentFileReferenceKey(fileUri); const drafts = Object.values(appAtomRegistry.get(composerDraftsAtom)); const queuedMessages = Object.values( appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom), ).flat(); - return [...drafts, ...queuedMessages].some((owner) => + return [...drafts, ...queuedMessages, ...signedOutAttachmentOwners()].some((owner) => owner.attachments.some( (attachment) => attachment.type === "file" && @@ -327,10 +408,9 @@ function isComposerAttachmentUploadReferenced( const queuedMessages = Object.values( appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom), ).flat(); - return [...drafts, ...queuedMessages].some((owner) => + return [...drafts, ...queuedMessages, ...signedOutAttachmentOwners()].some((owner) => owner.attachments.some( (attachment) => - attachment.type === "file" && attachment.uploadEnvironmentId === environmentId && attachment.uploadedAttachmentId === attachmentId, ), @@ -348,7 +428,6 @@ export async function releaseUnusedComposerAttachmentFiles( const uploadCandidates = new Map>(); for (const attachment of attachments) { if ( - attachment.type !== "file" || attachment.uploadEnvironmentId === undefined || attachment.uploadedAttachmentId === undefined ) { @@ -358,7 +437,7 @@ export async function releaseUnusedComposerAttachmentFiles( ids.add(attachment.uploadedAttachmentId); uploadCandidates.set(attachment.uploadEnvironmentId, ids); } - if (candidates.size === 0) { + if (candidates.size === 0 && uploadCandidates.size === 0) { return; } @@ -446,7 +525,11 @@ export async function releaseUnusedComposerAttachmentFiles( export function scheduleUnusedComposerAttachmentCleanup( attachments: ReadonlyArray, ): void { - if (!attachments.some((attachment) => attachment.type === "file")) { + if ( + !attachments.some( + (attachment) => attachment.type === "file" || attachment.uploadedAttachmentId !== undefined, + ) + ) { return; } void releaseUnusedComposerAttachmentFiles(attachments).catch((error) => { @@ -454,6 +537,15 @@ export function scheduleUnusedComposerAttachmentCleanup( }); } +/** Keeps a native preview or upload readable until it finishes, then retries ownership cleanup. */ +export function retainComposerAttachmentFileForPreview( + attachment: DraftComposerFileAttachment, +): () => void { + return retainComposerAttachmentFile(attachment.fileUri, () => { + scheduleUnusedComposerAttachmentCleanup([attachment]); + }); +} + function schedulePersistComposerState(): void { if (persistTimer !== null) { clearTimeout(persistTimer); @@ -486,6 +578,7 @@ export function ensureComposerDraftsLoaded(): void { } loadPromise = loadPersistedComposerState() .then((persisted) => { + appAtomRegistry.set(composerCloudDraftsAtom, persisted.cloudDrafts); if (Object.keys(persisted.drafts).length > 0) { const current = appAtomRegistry.get(composerDraftsAtom); appAtomRegistry.set(composerDraftsAtom, { @@ -522,6 +615,192 @@ export async function waitForComposerDraftsLoaded(): Promise { } } +export async function getComposerCloudAccountId(): Promise { + await waitForComposerDraftsLoaded(); + return appAtomRegistry.get(composerCloudDraftsAtom).accountId; +} + +/** Save an account's local work before its relay environments are removed. */ +export async function archiveCloudComposerDrafts( + accountId: string | null, + environmentIds: ReadonlySet, +): Promise { + await waitForComposerDraftsLoaded(); + if (!(await threadOutboxManager.load())) throw new Error("Could not preserve queued messages."); + await flushThreadOutbox(); + const cloud = appAtomRegistry.get(composerCloudDraftsAtom); + const owner = accountId ?? cloud.accountId; + if (owner === null) return; + const queued = Object.values( + appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom), + ).flat(); + const current = appAtomRegistry.get(composerDraftsAtom); + const remaining = { ...current }; + const savedDrafts = { ...cloud.signedOut[owner]?.drafts }; + for (const [key, draft] of Object.entries(current)) { + const environmentId = composerDraftEnvironmentId(key, queued); + if (environmentId !== null && environmentIds.has(environmentId)) { + savedDrafts[key] = draft; + delete remaining[key]; + } + } + const savedMessages = new Map( + (cloud.signedOut[owner]?.queuedMessages ?? []).map((message) => [message.messageId, message]), + ); + for (const message of queued) { + if (environmentIds.has(message.environmentId)) savedMessages.set(message.messageId, message); + } + appAtomRegistry.set(composerDraftsAtom, remaining); + appAtomRegistry.set(composerCloudDraftsAtom, { + // Keep the owner through removal. A crash or failed cleanup can retry it + // on cold start before a different account activates. + accountId: owner, + signedOut: { + ...cloud.signedOut, + [owner]: { drafts: savedDrafts, queuedMessages: [...savedMessages.values()] }, + }, + }); + schedulePersistComposerState(); + await flushComposerDrafts(); +} + +function sameDraftAttachmentIds( + left: ReadonlyArray, + right: ReadonlyArray, +): boolean { + return ( + left.length === right.length && + left.every((attachment, index) => attachment.id === right[index]?.id) + ); +} + +/** An in-flight delivery can finish after sign-out took its snapshot. */ +export async function removeDeliveredCloudQueuedMessage( + message: QueuedThreadMessage, +): Promise { + await waitForComposerDraftsLoaded(); + const cloud = appAtomRegistry.get(composerCloudDraftsAtom); + const signedOut = { ...cloud.signedOut }; + let changed = false; + for (const [accountId, saved] of Object.entries(signedOut)) { + const archived = saved.queuedMessages.find( + (candidate) => + candidate.environmentId === message.environmentId && + candidate.messageId === message.messageId, + ); + if ( + !archived || + archived.commandId !== message.commandId || + archived.threadId !== message.threadId || + archived.text !== message.text || + !sameDraftAttachmentIds(archived.attachments, message.attachments) + ) + continue; + // Upload ids may change during preparation; user edits must remain recoverable. + if ( + JSON.stringify([ + archived.modelSelection, + archived.runtimeMode, + archived.interactionMode, + archived.creation, + ]) !== + JSON.stringify([ + message.modelSelection, + message.runtimeMode, + message.interactionMode, + message.creation, + ]) + ) + continue; + const editorKey = `pending-task:${message.messageId}`; + const editor = saved.drafts[editorKey]; + if ( + editor && + (editor.text !== message.text || + !sameDraftAttachmentIds(editor.attachments, message.attachments) || + (editor.modelSelection !== undefined && + JSON.stringify(editor.modelSelection) !== JSON.stringify(message.modelSelection)) || + (editor.runtimeMode !== undefined && editor.runtimeMode !== message.runtimeMode) || + (editor.interactionMode !== undefined && + editor.interactionMode !== message.interactionMode) || + (editor.workspaceSelection !== undefined && + (editor.workspaceSelection.mode !== message.creation?.workspaceMode || + editor.workspaceSelection.branch !== message.creation?.branch || + editor.workspaceSelection.worktreePath !== message.creation?.worktreePath || + (editor.workspaceSelection.startFromOrigin ?? false) !== + (message.creation?.startFromOrigin ?? false)))) + ) + continue; + const drafts = { ...saved.drafts }; + delete drafts[editorKey]; + signedOut[accountId] = { + drafts, + queuedMessages: saved.queuedMessages.filter((candidate) => candidate !== archived), + }; + changed = true; + } + if (!changed) return; + appAtomRegistry.set(composerCloudDraftsAtom, { ...cloud, signedOut }); + schedulePersistComposerState(); + try { + await flushComposerDrafts(); + } catch (error) { + // The live outbox can still remove this acknowledged message. Keep the + // archive update pending so a later successful flush lands it too. + schedulePersistComposerState(); + throw error; + } +} + +/** Restores only this account, before its connections can deliver queued turns. */ +export async function restoreCloudComposerDrafts(accountId: string): Promise { + await waitForComposerDraftsLoaded(); + const cloud = appAtomRegistry.get(composerCloudDraftsAtom); + const saved = cloud.signedOut[accountId]; + if (saved) { + if (!(await threadOutboxManager.load())) throw new Error("Could not restore queued messages."); + for (const message of saved.queuedMessages) { + const alreadyQueued = Object.values( + appAtomRegistry.get(threadOutboxManager.queuedMessagesByThreadKeyAtom), + ) + .flat() + .some((current) => current.messageId === message.messageId); + if (!alreadyQueued) await threadOutboxManager.enqueue(message); + } + updateComposerDrafts((current) => { + const restored = { ...current }; + for (const [key, draft] of Object.entries(saved.drafts)) { + const existing = current[key]; + const attachmentIds = new Set(existing?.attachments.map((attachment) => attachment.id)); + restored[key] = existing + ? { + ...draft, + ...existing, + text: mergeComposerDraftText(existing.text, draft.text), + // A concurrent import must not lose files, even above the send limit. + attachments: [ + ...existing.attachments, + ...draft.attachments.filter((attachment) => !attachmentIds.has(attachment.id)), + ], + importedShareIds: [ + ...new Set([ + ...(existing.importedShareIds ?? []), + ...(draft.importedShareIds ?? []), + ]), + ], + } + : draft; + } + return restored; + }); + } + const signedOut = { ...cloud.signedOut }; + delete signedOut[accountId]; + appAtomRegistry.set(composerCloudDraftsAtom, { accountId, signedOut }); + schedulePersistComposerState(); + await flushComposerDrafts(); +} + function updateComposerDrafts( update: (current: Record) => Record, ): void { @@ -657,6 +936,41 @@ export function removeComposerDraftAttachment(draftKey: string, imageId: string) ); } +/** Stamps a finished upload without overwriting text, removals, or newer attachments. */ +export function setComposerDraftAttachmentUpload( + draftKey: string, + attachment: DraftComposerAttachment, +): boolean { + let previous: DraftComposerAttachment | undefined; + updateComposerDrafts((current) => { + const draft = current[draftKey]; + previous = draft?.attachments.find((candidate) => candidate.id === attachment.id); + if (!draft || !previous) return current; + if ( + previous.uploadedAttachmentId === attachment.uploadedAttachmentId && + previous.uploadEnvironmentId === attachment.uploadEnvironmentId + ) + return current; + return { + ...current, + [draftKey]: { + ...draft, + attachments: draft.attachments.map((candidate) => + candidate.id === attachment.id + ? { + ...candidate, + uploadedAttachmentId: attachment.uploadedAttachmentId, + uploadEnvironmentId: attachment.uploadEnvironmentId, + } + : candidate, + ), + }, + }; + }); + if (previous) scheduleUnusedComposerAttachmentCleanup([previous]); + return previous !== undefined; +} + export function updateComposerDraftSettings( draftKey: string, settings: Partial, diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 2eade205e..318390d4f 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -104,6 +104,10 @@ import { removeThreadOutboxMessageIfCurrent } from "./thread-outbox-removal"; import { recoverPendingSendToComposer } from "./thread-outbox-recovery"; import { useThreadOutboxMessages } from "./use-thread-outbox"; import { useAtomCommand } from "./use-atom-command"; +import { + composerAttachmentUploadBlockReason, + composerAttachmentUploadsAtom, +} from "./composer-attachment-uploads"; import { threadEnvironment } from "./threads"; import { resolveExistingThreadComposerSettings } from "./use-thread-composer-state.logic"; @@ -147,7 +151,7 @@ export function useThreadDraftForThread(input: { export function useThreadComposerState() { const navigation = useNavigation(); - const { selectedThread: selectedThreadShell } = useThreadSelection(); + const { selectedThread: selectedThreadShell, selectedEnvironmentRuntime } = useThreadSelection(); const selectedThreadDetail = useSelectedThreadDetail(); const selectedThreadContextWindow = useMemo( () => deriveLatestContextWindowSnapshot(selectedThreadDetail?.activities ?? []), @@ -565,6 +569,16 @@ export function useThreadComposerState() { const thread = selectedThreadDetail ?? selectedThreadShell; const text = draft.text.trim(); const attachments = draft.attachments; + if ( + composerAttachmentUploadBlockReason({ + environmentId: selectedThreadShell.environmentId, + attachments, + connected: selectedEnvironmentRuntime?.connectionState === "connected", + serverConfig: selectedEnvironmentRuntime?.serverConfig ?? null, + states: appAtomRegistry.get(composerAttachmentUploadsAtom), + }) !== null + ) + return null; if (text.length === 0 && attachments.length === 0) { return null; } @@ -723,6 +737,8 @@ export function useThreadComposerState() { ); return messageId; }, [ + selectedEnvironmentRuntime?.connectionState, + selectedEnvironmentRuntime?.serverConfig, selectedSessionProviderInstanceId, selectedThreadDetail, sessionCompactionBlocksSubmission, @@ -760,6 +776,21 @@ export function useThreadComposerState() { const attachments = draft.attachments; if (text.length === 0 && attachments.length === 0) return null; + // Same gate onSendMessage applies: queueing while an upload is still in + // flight would start a second transfer of the same bytes alongside the + // background worker's. + if ( + composerAttachmentUploadBlockReason({ + environmentId: selectedThreadShell.environmentId, + attachments, + connected: selectedEnvironmentRuntime?.connectionState === "connected", + serverConfig: selectedEnvironmentRuntime?.serverConfig ?? null, + states: appAtomRegistry.get(composerAttachmentUploadsAtom), + }) !== null + ) { + return null; + } + if (attachments.length > PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { Alert.alert( "Too many attachments", diff --git a/apps/mobile/src/state/use-thread-outbox-drain.test.ts b/apps/mobile/src/state/use-thread-outbox-drain.test.ts index 2e1d5a8d5..917b01cb2 100644 --- a/apps/mobile/src/state/use-thread-outbox-drain.test.ts +++ b/apps/mobile/src/state/use-thread-outbox-drain.test.ts @@ -229,6 +229,7 @@ beforeEach(() => { afterEach(() => { appAtomRegistry.set(harness.manager.queuedMessagesByThreadKeyAtom, {}); appAtomRegistry.set(composerDrafts.composerDraftsAtom, {}); + appAtomRegistry.set(composerDrafts.composerCloudDraftsAtom, { accountId: null, signedOut: {} }); appAtomRegistry.set(editingQueuedMessageIdsAtom, {}); harness.draftFile.setWriteError(null); harness.removePersistedFile.mockClear(); @@ -357,6 +358,72 @@ describe("thread outbox attachment preparation", () => { }); describe("thread outbox drain delivery cleanup", () => { + it("removes an acknowledged outbox item even when the sign-out archive write fails", async () => { + const message = queuedMessage({ messageId: "archive-write-failure", text: "Delivered" }); + await harness.manager.enqueue(message); + await composerDrafts.archiveCloudComposerDrafts("account-a", new Set([message.environmentId])); + harness.draftFile.setWriteError(new Error("Draft storage unavailable")); + + await expect( + completeQueuedMessageDelivery(message, harness.manager.revisionOf(message.messageId)), + ).resolves.toBe("removed"); + expect(remainingMessages()).toEqual([]); + + harness.draftFile.setWriteError(null); + await composerDrafts.flushComposerDrafts(); + appAtomRegistry.set(composerDrafts.composerCloudDraftsAtom, { accountId: null, signedOut: {} }); + composerDrafts.resetComposerDraftsLoadState(); + await composerDrafts.restoreCloudComposerDrafts("account-a"); + expect(remainingMessages()).toEqual([]); + }); + + it.each([false, true])( + "does not restore a message delivered after the sign-out snapshot (outbox already cleared: %s)", + async (cleared) => { + const message = queuedMessage({ + messageId: "delivered-during-sign-out", + text: "Already delivered", + }); + await harness.manager.enqueue(message); + const deliveryRevision = harness.manager.revisionOf(message.messageId); + await composerDrafts.archiveCloudComposerDrafts( + "account-a", + new Set([message.environmentId]), + ); + expect( + appAtomRegistry.get(composerDrafts.composerCloudDraftsAtom).signedOut["account-a"] + ?.queuedMessages, + ).toEqual([message]); + + if (cleared) await harness.manager.clearEnvironment(message.environmentId); + await expect(completeQueuedMessageDelivery(message, deliveryRevision)).resolves.toBe( + cleared ? "edited" : "removed", + ); + + // Restart before signing back in: the archived copy must be removed on disk too. + appAtomRegistry.set(composerDrafts.composerCloudDraftsAtom, { + accountId: null, + signedOut: {}, + }); + composerDrafts.resetComposerDraftsLoadState(); + await composerDrafts.restoreCloudComposerDrafts("account-a"); + expect(remainingMessages()).toEqual([]); + }, + ); + + it("preserves an archived edit when an older payload finishes delivery", async () => { + const message = queuedMessage({ messageId: "edited-during-sign-out", text: "Original" }); + await harness.manager.enqueue(message); + const deliveryRevision = harness.manager.revisionOf(message.messageId); + const edited = { ...message, text: "Keep this edit" }; + await harness.manager.update(edited); + await composerDrafts.archiveCloudComposerDrafts("account-a", new Set([message.environmentId])); + await harness.manager.clearEnvironment(message.environmentId); + await expect(completeQueuedMessageDelivery(message, deliveryRevision)).resolves.toBe("edited"); + await composerDrafts.restoreCloudComposerDrafts("account-a"); + expect(remainingMessages()).toEqual([edited]); + }); + it("retries only cleanup after an acknowledged send removal fails", async () => { const message = queuedMessage({ messageId: "message-acknowledged", text: "delivered" }); const acknowledged = new Set([message.messageId]); diff --git a/apps/mobile/src/state/use-thread-outbox-drain.ts b/apps/mobile/src/state/use-thread-outbox-drain.ts index 0b4b4500a..a6503b021 100644 --- a/apps/mobile/src/state/use-thread-outbox-drain.ts +++ b/apps/mobile/src/state/use-thread-outbox-drain.ts @@ -56,6 +56,7 @@ import { getComposerDraftSnapshot, mergeComposerDraftContent, replaceComposerDraftAttachments, + removeDeliveredCloudQueuedMessage, undoComposerDraftMerge, updateComposerDraftSettings, waitForComposerDraftsLoaded, @@ -114,7 +115,10 @@ function findCreationProject( * `deliveryRevision` is the revision of the payload this attempt will send, * used for the delivery removal's compare-and-set. */ -export async function prepareQueuedMessageAttachments(queuedMessage: QueuedThreadMessage): Promise< +export async function prepareQueuedMessageAttachments( + queuedMessage: QueuedThreadMessage, + supportsImageUploads = false, +): Promise< | { readonly status: "ready"; readonly prepared: PreparedTurnAttachments; @@ -135,6 +139,7 @@ export async function prepareQueuedMessageAttachments(queuedMessage: QueuedThrea const result = await prepareTurnAttachments({ environmentId: queuedMessage.environmentId, attachments: queuedMessage.attachments, + supportsImageUploads, persistUploadedReferences: async (draftAttachments) => { if (appAtomRegistry.get(editingQueuedMessageIdsAtom)[queuedMessage.messageId]) { return "abandon"; @@ -179,13 +184,19 @@ export async function completeQueuedMessageDelivery( queuedMessage: QueuedThreadMessage, deliveryRevision: number, ): Promise<"removed" | "edited" | "failed"> { - // The editor may have taken the entry while startTurn was in flight; its - // unsaved edits have not bumped the revision yet, so the CAS alone would - // let removal win and the editor would lose them once it saves. - if (appAtomRegistry.get(editingQueuedMessageIdsAtom)[queuedMessage.messageId]) { - return "edited"; - } try { + await removeDeliveredCloudQueuedMessage(queuedMessage).catch((error) => { + console.warn("[thread-outbox] could not update sign-out snapshot after delivery", { + messageId: queuedMessage.messageId, + error, + }); + }); + // The editor may have taken the entry while startTurn was in flight; its + // unsaved edits have not bumped the revision yet, so the CAS alone would + // let removal win and the editor would lose them once it saves. + if (appAtomRegistry.get(editingQueuedMessageIdsAtom)[queuedMessage.messageId]) { + return "edited"; + } // Removal also releases the message's local attachment files. const removed = await removeThreadOutboxMessage( queuedMessage, @@ -221,6 +232,12 @@ export async function removeAcknowledgedExistingThreadMessage( acknowledgedMessageIds: Set, ): Promise { try { + await removeDeliveredCloudQueuedMessage(queuedMessage).catch((error) => { + console.warn("[thread-outbox] could not update sign-out snapshot after delivery", { + messageId: queuedMessage.messageId, + error, + }); + }); const removed = await removeThreadOutboxMessage(queuedMessage); if (removed) { acknowledgedMessageIds.delete(queuedMessage.messageId); @@ -453,15 +470,10 @@ async function preserveUploadedAttachmentsForEditor( const draftKey = `pending-task:${originalMessage.messageId}`; const draft = getComposerDraftSnapshot(draftKey); const uploadedById = new Map( - uploadedMessage.attachments - .filter((attachment) => attachment.type === "file") - .map((attachment) => [attachment.id, attachment] as const), + uploadedMessage.attachments.map((attachment) => [attachment.id, attachment] as const), ); let changed = false; const nextAttachments = draft.attachments.map((attachment) => { - if (attachment.type !== "file") { - return attachment; - } const uploaded = uploadedById.get(attachment.id); if ( !uploaded?.uploadedAttachmentId || @@ -653,7 +665,11 @@ export function useThreadOutboxDrain(): void { let persistedMessage: QueuedThreadMessage; let deliveryRevision: number; try { - const preparedResult = await prepareQueuedMessageAttachments(queuedMessage); + const preparedResult = await prepareQueuedMessageAttachments( + queuedMessage, + serverConfigs.get(queuedMessage.environmentId)?.environment.capabilities + .attachmentUploads === true, + ); if (preparedResult.status === "abandoned") { return "complete"; } @@ -717,7 +733,7 @@ export function useThreadOutboxDrain(): void { } return outcome === "edited" ? "complete" : "retry"; }, - [makeDeliveryHelpers, restoreQueuedMessage, startTurn], + [makeDeliveryHelpers, restoreQueuedMessage, serverConfigs, startTurn], ); const sendQueuedCreation = useCallback( @@ -735,7 +751,11 @@ export function useThreadOutboxDrain(): void { let persistedMessage: QueuedThreadMessage; let deliveryRevision: number; try { - const preparedResult = await prepareQueuedMessageAttachments(queuedMessage); + const preparedResult = await prepareQueuedMessageAttachments( + queuedMessage, + serverConfigs.get(queuedMessage.environmentId)?.environment.capabilities + .attachmentUploads === true, + ); if (preparedResult.status === "abandoned") { return "complete"; } @@ -807,7 +827,7 @@ export function useThreadOutboxDrain(): void { } return "retry"; }, - [makeDeliveryHelpers, restoreQueuedMessage, startTurn], + [makeDeliveryHelpers, restoreQueuedMessage, serverConfigs, startTurn], ); useEffect(() => { diff --git a/apps/server/src/http.test.ts b/apps/server/src/http.test.ts index 9ee36ffeb..7ae036bdc 100644 --- a/apps/server/src/http.test.ts +++ b/apps/server/src/http.test.ts @@ -1,13 +1,102 @@ import { expect, it } from "@effect/vitest"; import { describe } from "vite-plus/test"; +import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import { HttpServerResponse } from "effect/unstable/http"; import { assetResponseHeaders, + assetFileResponse, downloadContentDisposition, isLoopbackHostname, resolveDevRedirectUrl, } from "./http.ts"; +const fileResponseLayer = Layer.mergeAll(NodeHttpPlatform.layer, NodeServices.layer); + +describe("video asset byte ranges", () => { + it.effect("streams exactly the requested bytes and leaves full downloads intact", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-video-range-" }); + const file = path.join(directory, "clip.mp4"); + yield* fs.writeFileString(file, "0123456789"); + const asset = { path: file, mimeType: "video/mp4" }; + for (const [header, expected, contentRange] of [ + ["bytes=0-1", "01", "bytes 0-1/10"], + ["bytes=4-", "456789", "bytes 4-9/10"], + ["bytes=-3", "789", "bytes 7-9/10"], + ["bytes=-999999999999999999999999", "0123456789", "bytes 0-9/10"], + ["bytes=8-999999999999999999999999", "89", "bytes 8-9/10"], + ] as const) { + const response = HttpServerResponse.toWeb(yield* assetFileResponse(asset, header)); + expect(response.status).toBe(206); + expect(response.headers.get("accept-ranges")).toBe("bytes"); + expect(response.headers.get("content-range")).toBe(contentRange); + expect(response.headers.get("content-length")).toBe(String(expected.length)); + expect(yield* Effect.promise(() => response.text())).toBe(expected); + } + for (const header of [ + undefined, + "items=0-1", + "bytes=0-1,4-5", + "bytes=8-2", + "bytes=-", + "bytes=bad", + ]) { + const response = HttpServerResponse.toWeb(yield* assetFileResponse(asset, header)); + expect(response.status).toBe(200); + expect(yield* Effect.promise(() => response.text())).toBe("0123456789"); + } + const conditional = HttpServerResponse.toWeb( + yield* assetFileResponse(asset, "bytes=0-1", '"old-etag"'), + ); + expect(conditional.status).toBe(200); + expect(yield* Effect.promise(() => conditional.text())).toBe("0123456789"); + const uppercase = HttpServerResponse.toWeb( + yield* assetFileResponse({ ...asset, mimeType: "Video/MP4" }, "bytes=0-1"), + ); + expect(uppercase.status).toBe(206); + expect(yield* Effect.promise(() => uppercase.text())).toBe("01"); + const image = HttpServerResponse.toWeb( + yield* assetFileResponse({ path: file, mimeType: "image/png" }, "bytes=0-1"), + ); + expect(image.status).toBe(200); + expect(image.headers.has("accept-ranges")).toBe(false); + expect(yield* Effect.promise(() => image.text())).toBe("0123456789"); + }).pipe(Effect.provide(fileResponseLayer)), + ); + + it.effect("rejects ranges outside the file, including empty files", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-video-range-" }); + const file = path.join(directory, "clip.mp4"); + yield* fs.writeFileString(file, "0123456789"); + for (const header of ["bytes=10-", "bytes=-0", "bytes=999999999999999999999999-"]) { + const response = HttpServerResponse.toWeb( + yield* assetFileResponse({ path: file, mimeType: "video/mp4" }, header), + ); + expect(response.status).toBe(416); + expect(response.headers.get("content-range")).toBe("bytes */10"); + expect(yield* Effect.promise(() => response.text())).toBe(""); + } + yield* fs.writeFileString(file, ""); + const empty = HttpServerResponse.toWeb( + yield* assetFileResponse({ path: file, mimeType: "video/mp4" }, "bytes=0-1"), + ); + expect(empty.status).toBe(416); + expect(empty.headers.get("content-range")).toBe("bytes */0"); + }).pipe(Effect.provide(fileResponseLayer)), + ); +}); + describe("http dev routing", () => { it("treats localhost and loopback addresses as local", () => { expect(isLoopbackHostname("127.0.0.1")).toBe(true); diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index 63e499961..d42224ed1 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -117,6 +117,63 @@ export function assetResponseHeaders( }; } +/** A single byte range for native video readers; unsupported range syntax uses the full file. */ +function assetByteRange(header: string, size: bigint) { + const match = /^bytes=(\d*)-(\d*)$/i.exec(header.trim()); + if (!match || (!match[1] && !match[2])) return null; + const first = match[1] ? BigInt(match[1]) : null; + const last = match[2] ? BigInt(match[2]) : null; + if (first !== null && last !== null && last < first) return null; + if (size === 0n || (first !== null && first >= size) || (first === null && last === 0n)) { + return { _tag: "Unsatisfiable" as const }; + } + const start = first ?? (last! >= size ? 0n : size - last!); + const end = first === null || last === null || last >= size ? size - 1n : last; + return { + _tag: "Range" as const, + offset: start, + bytesToRead: end - start + 1n, + contentRange: `bytes ${start}-${end}/${size}`, + }; +} + +export const assetFileResponse = Effect.fn("assetFileResponse")(function* ( + asset: { + readonly path: string; + readonly download?: boolean; + readonly fileName?: string; + readonly mimeType?: string; + }, + rangeHeader?: string, + ifRangeHeader?: string, +) { + const headers = assetResponseHeaders(asset.path, asset); + if (headers["Content-Type"]?.toLowerCase().startsWith("video/")) { + headers["Accept-Ranges"] = "bytes"; + // If-Range requires a matching validator. A full response is safe when we cannot validate it. + if (rangeHeader && !ifRangeHeader) { + const fs = yield* FileSystem.FileSystem; + const info = yield* fs.stat(asset.path); + const range = assetByteRange(rangeHeader, info.size); + if (range?._tag === "Unsatisfiable") { + return HttpServerResponse.empty({ + status: 416, + headers: { ...headers, "Content-Range": `bytes */${info.size}` }, + }); + } + if (range?._tag === "Range") { + return yield* HttpServerResponse.file(asset.path, { + status: 206, + offset: range.offset, + bytesToRead: range.bytesToRead, + headers: { ...headers, "Content-Range": range.contentRange }, + }); + } + } + } + return yield* HttpServerResponse.file(asset.path, { status: 200, headers }); +}); + export const httpCompressionLayer = HttpRouter.middleware(HttpMiddleware.compression(), { global: true, }); @@ -282,19 +339,11 @@ export const assetRouteLayer = HttpRouter.add( if (!asset) { return HttpServerResponse.text("Not Found", { status: 404 }); } - return yield* HttpServerResponse.file(asset.path, { - status: 200, - headers: assetResponseHeaders( - asset.path, - asset.download || asset.mimeType !== undefined - ? { - ...(asset.download ? { download: true } : {}), - ...(asset.fileName !== undefined ? { fileName: asset.fileName } : {}), - ...(asset.mimeType !== undefined ? { mimeType: asset.mimeType } : {}), - } - : undefined, - ), - }).pipe( + return yield* assetFileResponse( + asset, + request.method === "GET" ? request.headers.range : undefined, + request.headers["if-range"], + ).pipe( Effect.orElseSucceed(() => HttpServerResponse.text("Internal Server Error", { status: 500 })), ); }), diff --git a/apps/web/src/types.ts b/apps/web/src/types.ts index e2702d792..8638cade7 100644 --- a/apps/web/src/types.ts +++ b/apps/web/src/types.ts @@ -17,6 +17,9 @@ import type { EnvironmentThread, EnvironmentThreadShell, } from "@t3tools/client-runtime/state/shell"; +import { videoMimeType } from "@t3tools/shared/video"; + +export { videoMimeType } from "@t3tools/shared/video"; export type SessionPhase = "disconnected" | "connecting" | "ready" | "running"; export const DEFAULT_RUNTIME_MODE: RuntimeMode = "full-access"; @@ -59,41 +62,6 @@ export function isFileAttachment(attachment: ChatAttachment): attachment is Chat return attachment.type === "file"; } -const VIDEO_MIME_TYPE_BY_EXTENSION: Readonly> = { - avi: "video/x-msvideo", - m4v: "video/mp4", - mkv: "video/x-matroska", - mov: "video/quicktime", - mp4: "video/mp4", - ogv: "video/ogg", - webm: "video/webm", -}; - -const PLAYABLE_VIDEO_MIME_TYPES: ReadonlySet = new Set( - Object.values(VIDEO_MIME_TYPE_BY_EXTENSION), -); - -/** - * The container this attachment should be presented as, or null when it is not - * a video Pylon offers to play. - * - * The extension decides first. Trusting a bare `video/*` prefix misreads files - * the host maps to a transport stream — a TypeScript `.ts` source is reported as - * `video/mp2t` — which would turn source files into blank play tiles. - */ -export function videoMimeType( - attachment: Pick, -): string | null { - const dotIndex = attachment.name.lastIndexOf("."); - const byExtension = - dotIndex < 0 - ? null - : (VIDEO_MIME_TYPE_BY_EXTENSION[attachment.name.slice(dotIndex + 1).toLowerCase()] ?? null); - if (byExtension !== null) return byExtension; - const mimeType = attachment.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; - return PLAYABLE_VIDEO_MIME_TYPES.has(mimeType) ? mimeType : null; -} - export function isVideoAttachment(attachment: ChatFileAttachment): boolean { return videoMimeType(attachment) !== null; } diff --git a/docs/internals/connection-runtime.md b/docs/internals/connection-runtime.md index d19ead596..76294ed95 100644 --- a/docs/internals/connection-runtime.md +++ b/docs/internals/connection-runtime.md @@ -138,6 +138,19 @@ connection policy. `EnvironmentOwnedDataCleanup` is part of this contract: on removal the registry clears its cache and calls the platform implementation, so web clears composer drafts and mobile clears drafts plus the thread outbox. +Mobile cloud sign-out first saves relay drafts and queued messages in the local +composer store under the owning account. These saved copies retain attachment +files during cleanup and remain outside the active composer and upload queue. +Signing back into that account restores them before relay credentials activate. +Directly paired environments keep their drafts and outbox when cloud sign-out runs. + +Mobile composer attachments upload over HTTP while their environment is connected, +with at most three concurrent transfers. Drafts retain local image data or an owned +file URI alongside the pending upload ID. Sending verifies and reuses that ID, or +uploads the local bytes again if it expired. Disconnecting cancels active transfers +without discarding drafts; reconnecting resumes preparation. Older servers without +attachment-upload support continue to receive inline images. + ## Source Boundaries Applications must import explicit package subpaths; the package intentionally diff --git a/docs/internals/mobile-navigation.md b/docs/internals/mobile-navigation.md index 86c61921f..61b97ab6a 100644 --- a/docs/internals/mobile-navigation.md +++ b/docs/internals/mobile-navigation.md @@ -1,4 +1,4 @@ -# Mobile navigation headers +# Mobile navigation The iOS Home and thread routes share the root native stack in [`Stack.tsx`](../../apps/mobile/src/Stack.tsx). Keeping them in one navigation @@ -48,3 +48,76 @@ iOS `MenuView` action, including nested actions. The menu library's Fabric bridg rendering `MenuView` directly. Explicit colors are preserved, and destructive actions default to the theme's danger foreground color. Native stack header menus use a separate implementation and do not need this workaround. + +## Native media presentations + +`PresentationSource` in `NativePresentation` registers a thumbnail for AVKit, +image zoom transitions, and UIKit's share sheet. Wrap the thumbnail as its single child +and pass the stable identifier to the presentation. The registry keeps weak +references to source views; recycled or compact composer thumbnails can register +the same identifier. Identifiers must distinguish simultaneously visible attachments. +The source registration does not own the preview. Android uses a regular view. + +On iOS, video previews mount `AVPlayerViewController` temporarily inside the +registered source and enter full screen through AVKit. AVKit +owns that zoom, its playback controls, Close button, and interactive dismissal. +Do not replace AVKit's transition with `preferredTransition`: in the iOS 27 +simulator, that leaves native Close unable to exit full screen. When the source is unavailable, +the player uses a standard modal presentation. Programmatic entry uses the same +guarded `enterFullScreenAnimated:completionHandler:` selector as Expo Video; +if that selector is unavailable, the player also falls back to a standard modal. + +`FilePreviewModal` resolves image and PDF sources from a URI, a signed environment asset, +or a retained composer file. On iOS, Quick Look owns image and document layout, controls, +zooming, sharing, and interactive dismissal. Its delegate supplies the registered thumbnail +and its bounds for Quick Look's source-view zoom. Do not layer `preferredTransition` or +another image scroll view over that presentation: Quick Look coordinates its image gestures +with the return to the thumbnail. Missing sources use the standard transition, and Reduce +Motion disables animation. A pending programmatic Close waits until the current presentation +or cancelled dismissal has settled before starting another transition. + +The shared native presenter copies original bytes into its own temporary directory and +removes that copy after dismissal. Network downloads write to disk, and sharing never edits +the source attachment. Draft images use their stored upload data rather than a potentially +expired picker URI. No React Navigation route or custom transition animator is needed. + +The same viewer handles message images, markdown images, PDF attachments and links, +composer thumbnails, and workspace image previews. The workspace PDF web preview has an +Open PDF action for the native viewer. Android retains its image viewer and uses the +system chooser for PDFs. Saving images on iOS uses the add-only photo-library permission. + +Received videos open directly from their signed asset URL. AVKit handles buffering; +the client does not download the entire file or show a separate opening overlay before +presentation. The URL is captured once per preview so credential refresh does not +restart playback. Saving or sharing still downloads the original file. + +The native presentation promise completes after dismissal. Local draft previews +hold their file lease until that promise settles. The iOS preview component requests +native dismissal when its source screen unmounts. Playback pauses in the background. +AVPlayer activates audio as playback starts. The presenter pauses and releases +its own player on close, then restores the previous audio-session configuration +if no other component changed it during playback. It does not deactivate the +shared session, which may still serve another player or recorder. Android retains +its React Native modal and Expo Video player. + +`shareFileFromSource` uses the same source registration to anchor UIKit's activity +controller. Its promise completes when the native share flow finishes, keeping +the existing attachment lease and foreground handoff active for that duration. +Android uses Expo Sharing. On iOS, received and draft video attachments expose +Save or share through `VideoAttachmentMenu`. The attachment supplies the source +identifier, and the native share presentation inherits its appearance. AVKit's +iOS playback controls do not expose a public custom-share-action API. + +Video attachment thumbnails use Expo Video's native frame extraction and Expo Image. +Received attachments use their signed asset URL; drafts retain and resolve their local file +until extraction ends. Extraction is serial; temporary players never play or change audio settings. Leaving the screen cancels +pending work; a 15-second limit prevents an unreachable source from holding up the queue. +The client keeps at most 32 native images, each bounded to 480 pixels per side, keyed by +environment and attachment identity rather than expiring URLs. Images still displayed keep +their own references when evicted from that cache. + +The asset HTTP route supports single byte ranges for videos so iOS can read metadata and +frames without first downloading the whole file. Normal downloads keep their full response; +unsupported ranges and conditional `If-Range` requests also fall back to the full file. +An older environment without range support may still show the play-card fallback. Thumbnail +failure never disables playback or sharing. diff --git a/docs/user/composer.md b/docs/user/composer.md index 61bd092ed..5b9f15b84 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -13,23 +13,40 @@ videos, text files, PDFs, ZIP archives, and other files. Each file can be up to by the server, capped at 50 MB. Each message can carry up to eight attachments in total. Files upload directly to the environment, where your agent can read, copy, or edit them by their file path. -In the web and desktop apps, attachments upload as soon as you add them. The send button becomes -available after every upload finishes. Failed uploads can be retried or removed. In the mobile app, -tap **+** to open the photo library from either the compact or expanded composer. When the connected -server supports file uploads, **+** opens a menu beside the button with **Photo Library** and -**Choose Files**. Videos use the server's file upload limit. You can also share photos, videos, and -files into Pylon from other apps through the system share sheet. Mobile uploads happen when the -message sends, so queued messages keep their files until they deliver. Select a received file on -mobile to save it or open it in another app through the system share sheet. - -On web and desktop, select a video attachment before or after sending to play it with the browser's -built-in controls. Playback depends on the video formats and codecs that the browser supports. +Attachments upload as soon as you add them while connected to a server that supports uploads. +The send button becomes available after every upload finishes. Failed uploads can be retried or +removed. In the mobile app, tap **+** to open the photo library from either the compact or expanded +composer. When the connected server supports file uploads, **+** opens a menu beside the button with +**Photo Library** and **Choose Files**. Videos use the server's file upload limit. You can also +share photos, videos, and files into Pylon from other apps through the system share sheet. Mobile +keeps a local copy of each draft attachment, so you can still preview it and queue messages while +offline. Uploads resume when you reconnect. Drafts and queued messages survive app restarts; +signing out of Pylon Connect keeps them on your device until you sign back into the same account. +Select a received file on mobile to preview it, save it, or open it in another app through the +system share sheet. + +Tap an image or PDF before or after sending to open it. On iOS, images zoom from their thumbnail +into the native viewer. Pinch or double-tap to zoom, and swipe down or tap Close to return. +Use Share to save a copy or send it to another app. PDFs support page navigation and search. +PDF links in assistant responses open the same preview. On Android, images open in the image +viewer and PDFs open the system chooser. + +Select a video attachment before or after sending to play it. Web and desktop use the browser's +built-in controls. On mobile, videos open in a full-screen player with native playback controls. +Supported videos show a thumbnail in the conversation and composer. +On iOS, received videos stream from their environment as they play. Supported formats and codecs +depend on the browser or device; you can save an unsupported video to open it in another app. + +On iOS, the system player zooms from the attachment. Swipe down or tap Close to return to the +conversation or draft. Touch and hold the attachment, then choose **Save or share video** to open +the system share options. On Android, use **Save or share video** inside the preview. On web and desktop, if you reload before a file finishes uploading, the draft keeps the file's name and shows **Attach again** next to it. Attach the file again or remove it, then send. On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into -the composer or paste them into a message. +the composer or paste them into a message. On iOS, selecting them from **Photo Library** also +converts them to JPEG. The 10 MB image limit applies to the converted photo. On mobile, the model picker shows each OpenCode model's upstream provider, such as Anthropic, GitHub Copilot, or OpenCode Zen, beneath its name. Search by that provider name to narrow the list diff --git a/packages/shared/package.json b/packages/shared/package.json index 3b7d762e3..7e2d7b20a 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -203,6 +203,10 @@ "types": "./src/filePreview.ts", "import": "./src/filePreview.ts" }, + "./video": { + "types": "./src/video.ts", + "import": "./src/video.ts" + }, "./chatList": { "types": "./src/chatList.ts", "import": "./src/chatList.ts" diff --git a/packages/shared/src/video.test.ts b/packages/shared/src/video.test.ts new file mode 100644 index 000000000..bbe15f40f --- /dev/null +++ b/packages/shared/src/video.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { videoMimeType } from "./video.ts"; + +describe("videoMimeType", () => { + it("recognizes a saved video with a generic picker MIME type", () => { + expect(videoMimeType({ name: "Recording.MOV", mimeType: "application/octet-stream" })).toBe( + "video/quicktime", + ); + }); + + it("lets a known extension outrank the reported MIME type", () => { + expect(videoMimeType({ name: "recording.mp4", mimeType: " VIDEO/WebM; codecs=vp9 " })).toBe( + "video/mp4", + ); + }); + + it("trusts a playable MIME type and removes parameters when the extension is unknown", () => { + expect(videoMimeType({ name: "recording", mimeType: " VIDEO/WebM; codecs=vp9 " })).toBe( + "video/webm", + ); + }); + + // Hosts map a TypeScript source to video/mp2t. Trusting a bare `video/*` + // prefix turns source files into blank play tiles, so it is not playable. + it.each(["session-logic.ts", "clip.ts", "recording"])( + "does not treat %s reported as video/mp2t as a video", + (name) => { + expect(videoMimeType({ name, mimeType: "video/mp2t" })).toBeNull(); + }, + ); + + it.each(["README", "report.pdf", "file.constructor", "file.__proto__"])( + "does not mistake %s for a video", + (name) => { + expect(videoMimeType({ name, mimeType: "application/octet-stream" })).toBeNull(); + }, + ); +}); diff --git a/packages/shared/src/video.ts b/packages/shared/src/video.ts new file mode 100644 index 000000000..82343c75b --- /dev/null +++ b/packages/shared/src/video.ts @@ -0,0 +1,35 @@ +const VIDEO_MIME_TYPE_BY_EXTENSION = new Map([ + ["avi", "video/x-msvideo"], + ["m4v", "video/mp4"], + ["mkv", "video/x-matroska"], + ["mov", "video/quicktime"], + ["mp4", "video/mp4"], + ["ogv", "video/ogg"], + ["webm", "video/webm"], +]); + +const PLAYABLE_VIDEO_MIME_TYPES = new Set(VIDEO_MIME_TYPE_BY_EXTENSION.values()); + +/** + * The container this attachment should be presented as, or null when it is not + * a video Pylon offers to play. Recognizes videos even when the file picker + * omitted their MIME type. + * + * The extension decides first. Trusting a bare `video/*` prefix misreads files + * the host maps to a transport stream — a TypeScript `.ts` source is reported as + * `video/mp2t` — which would turn source files into blank play tiles. + */ +export function videoMimeType(attachment: { + readonly name: string; + readonly mimeType: string; +}): string | null { + const dotIndex = attachment.name.lastIndexOf("."); + const byExtension = + dotIndex < 0 + ? null + : (VIDEO_MIME_TYPE_BY_EXTENSION.get(attachment.name.slice(dotIndex + 1).toLowerCase()) ?? + null); + if (byExtension !== null) return byExtension; + const mimeType = attachment.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + return PLAYABLE_VIDEO_MIME_TYPES.has(mimeType) ? mimeType : null; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a184574ff..1cc980b6a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -332,6 +332,9 @@ importers: expo-device: specifier: ~57.0.1 version: 57.0.1(expo@57.0.18) + expo-document-picker: + specifier: ~57.0.1 + version: 57.0.1(expo@57.0.18) expo-file-system: specifier: ~57.0.6 version: 57.0.6(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)) @@ -383,6 +386,9 @@ importers: expo-updates: specifier: ~57.0.19 version: 57.0.19(expo-dev-client@57.0.16(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)))(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) + expo-video: + specifier: ~57.0.3 + version: 57.0.3(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) expo-web-browser: specifier: ~57.0.2 version: 57.0.2(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)) @@ -6732,6 +6738,11 @@ packages: peerDependencies: expo: '*' + expo-document-picker@57.0.1: + resolution: {integrity: sha512-qBwM5oxDZ3I9kwFD3pUE1oK/WNv9artoEKO6UpqhQgNRr0XA1ALRVWYjkF4+ge9lUNDRehjTm/jenINkzqg84g==} + peerDependencies: + expo: '*' + expo-eas-client@57.0.2: resolution: {integrity: sha512-EfFiqUr0o9TvTOgMbqDiV1oIdG/d7kirhqtwa7roGmm9wF+CpXUz20d9YGzO6KzJWUYgdUgu4B6Ocv0jNJUdnQ==} @@ -6901,6 +6912,13 @@ packages: expo-dev-client: optional: true + expo-video@57.0.3: + resolution: {integrity: sha512-Z+rLdBSzICwoHm/HUxND5fm5nfgqiB+QPWAOKxmA8ScYwvxG1UGjZ6OaayvPc3GkT4aucsbycmFO0uUv/qnIhg==} + peerDependencies: + expo: '*' + react: '*' + react-native: '*' + expo-web-browser@57.0.2: resolution: {integrity: sha512-3vl5kvd7PB48ub6PpNIJUuPxO8xVa6D8RnIgNba6SXRwqFprOfeEZgwTgtm41kz0AAtvMOztUVNEUkwrHKjqMQ==} peerDependencies: @@ -16974,6 +16992,10 @@ snapshots: expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc) ua-parser-js: 0.7.41 + expo-document-picker@57.0.1(expo@57.0.18): + dependencies: + expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc) + expo-eas-client@57.0.2: {} expo-file-system@57.0.6(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)): @@ -17214,6 +17236,12 @@ snapshots: transitivePeerDependencies: - supports-color + expo-video@57.0.3(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3): + dependencies: + expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc) + react: 19.2.3 + react-native: 0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) + expo-web-browser@57.0.2(expo@57.0.18)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6)): dependencies: expo: 57.0.18(f9c992a5d7c53d81398568d3950992dc)