From 5418383a3a47b3b4d996bbbf96856be1f52896b0 Mon Sep 17 00:00:00 2001 From: cyberinferno Date: Tue, 23 Jun 2026 13:56:16 +0530 Subject: [PATCH 1/3] Document file browser chunked uploads --- AGENTS.md | 5 + README.md | 12 +- cmd/omnihance-a3-agent/docs/openapi.yml | 482 ++++++++ .../src/components/file-browser-uploader.tsx | 1082 ++++++++++++++++ .../src/components/file-tree.tsx | 74 +- .../omnihance-a3-agent-ui/src/lib/api.ts | 156 +++ internal/constants/constants.go | 3 + internal/server/file_system_routes.go | 13 + internal/server/file_upload_routes.go | 1091 +++++++++++++++++ internal/server/file_upload_routes_test.go | 275 +++++ internal/server/server.go | 7 + 11 files changed, 3183 insertions(+), 17 deletions(-) create mode 100644 cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx create mode 100644 internal/server/file_upload_routes.go create mode 100644 internal/server/file_upload_routes_test.go diff --git a/AGENTS.md b/AGENTS.md index 63ac35e..19f7d8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,3 +129,8 @@ For multi-step tasks, state a brief plan: - **Magic Numbers:** Avoid using magic numbers; use constants or named variables. - **Redundant Code:** Do NOT write redundant code. - **Hardcoded Values:** Do NOT use hardcoded values; use constants or named variables. + +### Workflow Guidelines + +- You have access to Github CLI. Use it for all Github related actions. +- Before starting to implement anything please check the active branch. If its `master` or `main` pull the latest changes. Then fork a new branch from it with naming convention like `feat/{some-feature}` or `fix/{some-fix}` or `docs/{some-docs}` etc. Keep branch name short but meaningful with conventional commit prefix. If its already in non default branch do not do anything. diff --git a/README.md b/README.md index 0c3f866..a068d50 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Omnihance A3 Agent is a full-stack application consisting of: - Can manage users (list, update status, set passwords) - Cannot have status changed by other admins - **Admin** (`admin`): Administrative access - - Can view and edit files + - Can view, edit, and upload files in the file browser - Can revert file changes - Can upload game client data - Can view metrics and game data @@ -55,7 +55,7 @@ Omnihance A3 Agent is a full-stack application consisting of: - **Permission Actions**: - `view_files`: View file system and file contents (super_admin, admin, viewer) - `download_files`: Create and use one-day user-bound download links for file-browser and backup output files through `POST /api/file-tree/download-link` and `GET /api/file-tree/download/{token}` (super_admin, admin) - - `edit_files`: Edit files (super_admin, admin) + - `edit_files`: Edit files and upload files through the file browser (super_admin, admin) - `revert_files`: Revert files to previous revisions (super_admin, admin) - `upload_game_data`: Upload MON.ull and MC.ull files (super_admin, admin) - `manage_users`: Manage user accounts (super_admin only) @@ -78,6 +78,7 @@ Omnihance A3 Agent is a full-stack application consisting of: - **File Viewing**: View NPC files, quest files, spawn files, drop files, item files, item combination data files, and text files in the browser - **File Duplication**: Right-click any file in the file explorer to duplicate it with a custom name - **Directory Downloads**: Right-click a directory to request a ZIP download. The directory is compressed in the background through a tagged one-time backup job, stored under `DIRECTORY_DOWNLOADS_DIRECTORY`, and delivered through the same secure temp-link download flow used for files and backup outputs. +- **Chunked File Uploads**: Drag files or folders into a real directory, or use the Upload button next to Show dotfiles to open a drop zone with file and folder pickers. Uploads are disabled on the root drive-listing page, run one file at a time per browser tab, auto-rename conflicts with the existing `(copy)` naming pattern, retry transient chunk failures, verify SHA-256 before finalizing, and keep abandoned temp data hidden until server cleanup removes it. - **File Editing**: - **NPC File Editor**: Edit NPC properties including: - ID, Name, Respawn Rate @@ -583,6 +584,11 @@ Only stable GitHub releases are considered because GitHub's latest release endpo - `PUT /api/file-tree/item-combination-data` - Update A3 item combination data - `POST /api/file-tree/revert-file` - Revert file to previous revision - `POST /api/file-tree/duplicate-file` - Duplicate a file in the same directory +- `POST /api/file-tree/uploads` - Start a file-browser upload batch for a non-root destination directory and reserve conflict-safe target names +- `PUT /api/file-tree/uploads/{upload_id}/files/{file_id}/chunks/{chunk_index}` - Upload one binary file chunk with retry support +- `POST /api/file-tree/uploads/{upload_id}/files/{file_id}/complete` - Verify SHA-256 and finalize one uploaded file +- `POST /api/file-tree/uploads/{upload_id}/heartbeat` - Keep an active upload session alive while the browser tab is open +- `DELETE /api/file-tree/uploads/{upload_id}` - Cancel an upload session and remove hidden temp data - `GET /api/file-tree/revision-summary` - Get revision count for a file - `POST /api/file-tree/download-link` - Create or reuse a one-day user-bound download link for a file (requires `download_files` permission) - `POST /api/file-tree/directory-download-link` - Create, reuse, or start a background directory ZIP job and return either a secure download URL or polling metadata (requires `download_files` permission) @@ -732,7 +738,7 @@ The application uses SQLite with the following main tables: 5. **Upload Game Client Data**: Navigate to the Client Data section and upload MON.ull, MC.ull, and IT0.ull through IT3.ull files to populate monster, map, and item databases (requires admin or super admin role). -6. **Navigate Files**: Use the file tree sidebar to browse your server's file system (all authenticated users can view). Pin frequently used directories as shortcuts, right-click files to duplicate them, and download files or directories with admin or super admin access. Directory downloads compress in the background; keep the page open and do not refresh so the download can start automatically when ready. If you refresh, click the same directory download again to resume polling for the in-progress job. +6. **Navigate Files**: Use the file tree sidebar to browse your server's file system (all authenticated users can view). Pin frequently used directories as shortcuts, right-click files to duplicate them, and download files or directories with admin or super admin access. Admins and super admins can upload files or folders inside a selected directory by dragging into the browser or using the Upload button beside Show dotfiles; uploads are queued per tab, conflicts are auto-renamed, and progress stays visible in the bottom-right panel. Directory downloads compress in the background; keep the page open and do not refresh so the download can start automatically when ready. If you refresh, click the same directory download again to resume polling for the in-progress job. 7. **Edit Files**: Click on editable files (NPC files, quest files, spawn files, drop files (monster drop configurations), item files, item combination data files, or text files) to view and edit them (requires admin or super admin role). - **Quest Files**: Edit quest configurations with type-aware objectives, add/remove controls for optional slots, and binary-safe padding preservation diff --git a/cmd/omnihance-a3-agent/docs/openapi.yml b/cmd/omnihance-a3-agent/docs/openapi.yml index 46fc492..8d59b9b 100644 --- a/cmd/omnihance-a3-agent/docs/openapi.yml +++ b/cmd/omnihance-a3-agent/docs/openapi.yml @@ -1362,6 +1362,331 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/uploads: + post: + tags: + - file-system + summary: Start file-browser upload batch + description: Creates a chunked upload session for a non-root destination directory. Requires edit file permission. The server reserves final target names with the existing copy-name pattern so concurrent uploads cannot overwrite each other. + security: + - ApiKeyAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateFileUploadRequest' + responses: + '201': + description: Upload session created successfully + content: + application/json: + schema: + $ref: '#/components/schemas/CreateFileUploadResponse' + '400': + description: Bad Request - Invalid destination path, upload file path, file size, duplicate relative path, or chunk size + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Missing edit file permission or filesystem permission denied + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Destination path not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '507': + description: Insufficient storage while creating upload temp storage + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/uploads/{upload_id}/files/{file_id}/chunks/{chunk_index}: + put: + tags: + - file-system + summary: Upload one file chunk + description: Writes one binary chunk into hidden upload temp storage. Chunk uploads refresh the session activity timestamp and may be retried by the client for network or transient server failures. + security: + - ApiKeyAuth: [] + parameters: + - in: path + name: upload_id + required: true + schema: + type: string + description: Upload session ID returned by the batch creation endpoint. + - in: path + name: file_id + required: true + schema: + type: string + description: Server file ID returned for the file in the upload batch. + - in: path + name: chunk_index + required: true + schema: + type: integer + minimum: 0 + description: Zero-based chunk index. + requestBody: + required: true + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + '200': + description: Chunk uploaded successfully + content: + application/json: + schema: + $ref: '#/components/schemas/FileUploadChunkResponse' + '400': + description: Bad Request - Invalid chunk index, unexpected chunk size, or chunk out of range + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Missing edit file permission or filesystem permission denied + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Upload session or file not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '410': + description: Upload session expired + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '507': + description: Insufficient storage while writing upload chunk + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/uploads/{upload_id}/files/{file_id}/complete: + post: + tags: + - file-system + summary: Complete uploaded file + description: Verifies that all chunks are present, checks the uploaded file SHA-256 against the client hash, resolves any final external path conflict without overwriting, then moves the hidden temp file into place. + security: + - ApiKeyAuth: [] + parameters: + - in: path + name: upload_id + required: true + schema: + type: string + description: Upload session ID returned by the batch creation endpoint. + - in: path + name: file_id + required: true + schema: + type: string + description: Server file ID returned for the file in the upload batch. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CompleteFileUploadRequest' + responses: + '200': + description: File verified and finalized successfully + content: + application/json: + schema: + $ref: '#/components/schemas/CompleteFileUploadResponse' + '400': + description: Bad Request - Invalid SHA-256 value or missing chunks + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Missing edit file permission or filesystem permission denied + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Upload session or file not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '409': + description: Conflict - Uploaded file failed integrity check + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '410': + description: Upload session expired + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '507': + description: Insufficient storage while finalizing uploaded file + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/uploads/{upload_id}/heartbeat: + post: + tags: + - file-system + summary: Refresh upload session heartbeat + description: Refreshes the activity timestamp for an active upload session so hidden temp data is not treated as abandoned while the browser tab is still uploading. + security: + - ApiKeyAuth: [] + parameters: + - in: path + name: upload_id + required: true + schema: + type: string + description: Upload session ID returned by the batch creation endpoint. + responses: + '200': + description: Upload heartbeat accepted + content: + application/json: + schema: + $ref: '#/components/schemas/FileUploadHeartbeatResponse' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Missing edit file permission + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Upload session not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '410': + description: Upload session expired + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /api/file-tree/uploads/{upload_id}: + delete: + tags: + - file-system + summary: Cancel upload session + description: Cancels an active upload session, removes hidden temp upload data, and releases any reserved final paths for files that have not completed. + security: + - ApiKeyAuth: [] + parameters: + - in: path + name: upload_id + required: true + schema: + type: string + description: Upload session ID returned by the batch creation endpoint. + responses: + '200': + description: Upload cancelled successfully + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: 'Upload cancelled' + '401': + description: Unauthorized + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '403': + description: Forbidden - Missing edit file permission + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: Upload session not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' /api/file-tree/revision-summary: get: tags: @@ -4831,6 +5156,163 @@ components: type: string description: Full path of the newly created duplicate file example: "C:\\A3Server\\configs\\server (copy).ini" + CreateFileUploadRequest: + type: object + required: + - destination_path + - chunk_size + - files + properties: + destination_path: + type: string + description: Destination directory path. Upload creation is rejected for the root drive-listing page. + example: "C:\\A3Server" + chunk_size: + type: integer + format: int64 + minimum: 1 + description: Chunk size in bytes that the client will use for each uploaded file. + example: 4194304 + files: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/CreateFileUploadRequestFile' + CreateFileUploadRequestFile: + type: object + required: + - client_file_id + - relative_path + - size + properties: + client_file_id: + type: string + description: Client-generated ID used to map the response file reservation back to the selected browser file. + example: 'task-1-file-0' + relative_path: + type: string + description: Slash-separated upload path relative to the selected upload item. Folder uploads preserve nested paths here. + example: 'ZoneData/map/0.n_ndt' + size: + type: integer + format: int64 + minimum: 0 + description: File size in bytes. + example: 7340032 + CreateFileUploadResponse: + type: object + properties: + upload_id: + type: string + description: Server upload session ID. + example: '01HZXY2M3A4B5C6D7E8F9G0H1J' + expires_at: + type: string + format: date-time + description: Session expiration timestamp if no heartbeat or chunk activity is received. + files: + type: array + items: + $ref: '#/components/schemas/CreateFileUploadResponseFile' + CreateFileUploadResponseFile: + type: object + properties: + client_file_id: + type: string + description: Client file ID from the creation request. + example: 'task-1-file-0' + file_id: + type: string + description: Server-generated file ID used for chunk upload and completion calls. + example: '01HZXY2M3A4B5C6D7E8F9G0H1K' + relative_path: + type: string + description: Original normalized upload relative path. + example: 'ZoneData/map/0.n_ndt' + resolved_relative_path: + type: string + description: Conflict-safe relative path reserved by the server. + example: 'ZoneData/map/0 (copy).n_ndt' + target_path: + type: string + description: Full reserved destination path for the uploaded file. + example: "C:\\A3Server\\ZoneData\\map\\0 (copy).n_ndt" + size: + type: integer + format: int64 + description: File size in bytes. + example: 7340032 + chunk_size: + type: integer + format: int64 + description: Chunk size in bytes for this file. + example: 4194304 + total_chunks: + type: integer + description: Number of chunks expected for this file. + example: 2 + FileUploadChunkResponse: + type: object + properties: + message: + type: string + example: 'Chunk uploaded successfully' + received_chunks: + type: integer + description: Number of chunks received for the file so far. + example: 1 + total_chunks: + type: integer + description: Number of chunks required for the file. + example: 2 + CompleteFileUploadRequest: + type: object + required: + - sha256 + properties: + sha256: + type: string + minLength: 64 + maxLength: 64 + description: Lowercase hexadecimal SHA-256 hash of the original browser file. + example: '6d37795021e544d53d2569bb6139720ee0869d1111d924aed610ea148a84efcf' + CompleteFileUploadResponse: + type: object + properties: + message: + type: string + example: 'File uploaded successfully' + file_id: + type: string + description: Server file ID that was completed. + example: '01HZXY2M3A4B5C6D7E8F9G0H1K' + relative_path: + type: string + description: Original normalized upload relative path. + example: 'ZoneData/map/0.n_ndt' + resolved_relative_path: + type: string + description: Final relative path used after reservation and final conflict checks. + example: 'ZoneData/map/0 (copy).n_ndt' + final_path: + type: string + description: Full destination path where the uploaded file was finalized. + example: "C:\\A3Server\\ZoneData\\map\\0 (copy).n_ndt" + sha256: + type: string + description: Server-calculated SHA-256 hash of the finalized upload. + example: '6d37795021e544d53d2569bb6139720ee0869d1111d924aed610ea148a84efcf' + FileUploadHeartbeatResponse: + type: object + properties: + upload_id: + type: string + description: Upload session ID. + example: '01HZXY2M3A4B5C6D7E8F9G0H1J' + expires_at: + type: string + format: date-time + description: Updated session expiration timestamp. SpawnFileAPIData: type: object description: Parsed binary data from a spawn file (API request/response format). All fields are required when used as a request body. diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx new file mode 100644 index 0000000..3666ee6 --- /dev/null +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx @@ -0,0 +1,1082 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + AlertCircle, + CheckCircle2, + FolderUp, + Loader2, + Upload, + X, +} from 'lucide-react'; +import { toast } from 'sonner'; +import { + cancelFileUpload, + completeFileUpload, + createFileUpload, + heartbeatFileUpload, + uploadFileChunk, + APIError, + type CreateFileUploadResponse, +} from '@/lib/api'; +import { formatBytes, cn } from '@/lib/util'; + +import { Button } from '@/components/ui/button'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; + +const uploadChunkSize = 4 * 1024 * 1024; +const heartbeatIntervalMs = 30 * 1000; +const maxChunkAttempts = 4; + +type UploadTaskStatus = + | 'queued' + | 'uploading' + | 'verifying' + | 'complete' + | 'failed' + | 'cancelled'; + +type UploadSource = { + file: File; + relativePath: string; +}; + +type UploadTask = { + id: string; + destinationPath: string; + files: UploadSource[]; + status: UploadTaskStatus; + uploadedBytes: number; + totalBytes: number; + currentFileName: string | null; + uploadId: string | null; + error: string | null; + createdAt: number; +}; + +type UseFileBrowserUploaderParams = { + destinationPath: string; + canUpload: boolean; + onUploaded: () => void; +}; + +type BrowserFileEntry = FileSystemFileEntry & { + file: ( + successCallback: (file: File) => void, + errorCallback?: (error: DOMException) => void, + ) => void; +}; + +type BrowserDirectoryEntry = FileSystemDirectoryEntry & { + createReader: () => FileSystemDirectoryReader; +}; + +type BrowserEntry = BrowserFileEntry | BrowserDirectoryEntry; + +type DataTransferItemWithEntry = DataTransferItem & { + webkitGetAsEntry?: () => BrowserEntry | null; +}; + +export function useFileBrowserUploader({ + destinationPath, + canUpload, + onUploaded, +}: UseFileBrowserUploaderParams) { + const [tasks, setTasks] = useState([]); + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [isDragging, setIsDragging] = useState(false); + const fileInputRef = useRef(null); + const directoryInputRef = useRef(null); + const workerRunningRef = useRef(false); + const cancelledTaskIdsRef = useRef(new Set()); + const activeAbortRef = useRef(null); + const activeUploadIdsRef = useRef(new Map()); + const currentTaskRef = useRef(null); + + useEffect(() => { + directoryInputRef.current?.setAttribute('webkitdirectory', ''); + }, []); + + const enqueueUpload = useCallback( + (sources: UploadSource[]) => { + if (!canUpload || !destinationPath) { + toast.error('Open a folder before uploading'); + return; + } + + const uniqueSources = dedupeUploadSources(sources); + if (uniqueSources.length === 0) { + toast.error('No files found to upload'); + return; + } + + const task: UploadTask = { + id: crypto.randomUUID(), + destinationPath, + files: uniqueSources, + status: 'queued', + uploadedBytes: 0, + totalBytes: uniqueSources.reduce((total, source) => { + return total + source.file.size; + }, 0), + currentFileName: null, + uploadId: null, + error: null, + createdAt: Date.now(), + }; + + setTasks((current) => [...current, task]); + setIsDialogOpen(false); + toast.success( + uniqueSources.length === 1 + ? `Queued ${uniqueSources[0].file.name}` + : `Queued ${uniqueSources.length} files`, + ); + }, + [canUpload, destinationPath], + ); + + const cancelTask = useCallback((taskId: string) => { + cancelledTaskIdsRef.current.add(taskId); + if (currentTaskRef.current?.id === taskId) { + activeAbortRef.current?.abort(); + } + + const uploadId = activeUploadIdsRef.current.get(taskId); + if (uploadId) { + void cancelFileUpload(uploadId).catch(() => undefined); + } + + setTasks((current) => + current.map((task) => { + if (task.id !== taskId) { + return task; + } + + return { + ...task, + status: 'cancelled', + error: null, + }; + }), + ); + }, []); + + const clearTask = useCallback((taskId: string) => { + setTasks((current) => current.filter((task) => task.id !== taskId)); + cancelledTaskIdsRef.current.delete(taskId); + activeUploadIdsRef.current.delete(taskId); + }, []); + + const openUploadDialog = useCallback(() => { + if (!canUpload) { + toast.error('Open a folder before uploading'); + return; + } + + setIsDialogOpen(true); + }, [canUpload]); + + const handleFilesSelected = useCallback( + (event: React.ChangeEvent) => { + const selectedFiles = Array.from(event.target.files ?? []); + enqueueUpload( + selectedFiles.map((file) => ({ + file, + relativePath: file.name, + })), + ); + event.target.value = ''; + }, + [enqueueUpload], + ); + + const handleDirectorySelected = useCallback( + (event: React.ChangeEvent) => { + const selectedFiles = Array.from(event.target.files ?? []); + enqueueUpload( + selectedFiles.map((file) => ({ + file, + relativePath: + (file as File & { webkitRelativePath?: string }) + .webkitRelativePath || file.name, + })), + ); + event.target.value = ''; + }, + [enqueueUpload], + ); + + const handleDrop = useCallback( + async (event: React.DragEvent) => { + if (!canUpload) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + setIsDragging(false); + + try { + const sources = await uploadSourcesFromDataTransfer(event.dataTransfer); + enqueueUpload(sources); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : 'Failed to read dropped files', + ); + } + }, + [canUpload, enqueueUpload], + ); + + const dropHandlers = useMemo( + () => ({ + onDragOver: (event: React.DragEvent) => { + if (!canUpload || !hasFileDrag(event.dataTransfer)) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + setIsDragging(true); + }, + onDragLeave: (event: React.DragEvent) => { + if (!canUpload) { + return; + } + + event.preventDefault(); + event.stopPropagation(); + if (event.currentTarget.contains(event.relatedTarget as Node | null)) { + return; + } + + setIsDragging(false); + }, + onDrop: handleDrop, + }), + [canUpload, handleDrop], + ); + + const runNextQueuedTask = useCallback(async () => { + if (workerRunningRef.current) { + return; + } + + const nextTask = tasks.find((task) => task.status === 'queued'); + if (!nextTask) { + return; + } + + workerRunningRef.current = true; + currentTaskRef.current = nextTask; + + try { + await uploadTask( + nextTask, + setTasks, + cancelledTaskIdsRef, + activeAbortRef, + activeUploadIdsRef, + ); + onUploaded(); + } catch (error) { + if (!cancelledTaskIdsRef.current.has(nextTask.id)) { + setTasks((current) => + current.map((task) => { + if (task.id !== nextTask.id) { + return task; + } + + return { + ...task, + status: 'failed', + error: uploadErrorMessage(error), + }; + }), + ); + } + } finally { + activeAbortRef.current = null; + currentTaskRef.current = null; + workerRunningRef.current = false; + } + }, [onUploaded, tasks]); + + useEffect(() => { + void runNextQueuedTask(); + }, [runNextQueuedTask, tasks]); + + const activeTasks = tasks.filter((task) => task.status !== 'complete'); + const completedTasks = tasks.filter((task) => task.status === 'complete'); + const visibleTasks = [...activeTasks, ...completedTasks].slice(-6); + + const dialog = ( + + + + Upload + + Add files or a folder to the current directory. + + +
{ + event.preventDefault(); + event.stopPropagation(); + }} + onDrop={async (event) => { + event.preventDefault(); + event.stopPropagation(); + try { + const sources = await uploadSourcesFromDataTransfer( + event.dataTransfer, + ); + enqueueUpload(sources); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : 'Failed to read dropped files', + ); + } + }} + > + + + +

Drop files here

+
+ + +
+
+ + + ); + + const progressPanel = + visibleTasks.length > 0 ? ( +
+
+
+

Uploads

+

+ {activeTasks.length} active, {completedTasks.length} complete +

+
+ +
+
+ {visibleTasks.map((task) => ( + cancelTask(task.id)} + onClear={() => clearTask(task.id)} + /> + ))} +
+
+ ) : null; + + return { + canUpload, + isDragging, + openUploadDialog, + dropHandlers, + dialog, + progressPanel, + }; +} + +function UploadTaskRow({ + task, + onCancel, + onClear, +}: { + task: UploadTask; + onCancel: () => void; + onClear: () => void; +}) { + const progress = + task.totalBytes > 0 + ? Math.min(100, Math.round((task.uploadedBytes / task.totalBytes) * 100)) + : task.status === 'complete' + ? 100 + : 0; + const label = + task.files.length === 1 + ? task.files[0].file.name + : `${task.files.length} files`; + const canCancel = + task.status === 'queued' || + task.status === 'uploading' || + task.status === 'verifying'; + const canClear = + task.status === 'complete' || + task.status === 'failed' || + task.status === 'cancelled'; + + return ( +
+
+
+

{label}

+

+ {task.status === 'queued' + ? 'Queued' + : task.status === 'uploading' + ? task.currentFileName || 'Uploading' + : task.status === 'verifying' + ? 'Verifying' + : task.status === 'failed' + ? task.error || 'Upload failed' + : task.status === 'cancelled' + ? 'Cancelled' + : 'Complete'} +

+
+ {task.status === 'complete' && ( + + )} + {task.status === 'failed' && ( + + )} + {(task.status === 'queued' || + task.status === 'uploading' || + task.status === 'verifying') && ( + + )} +
+
+
+
+
+ + {formatBytes(task.uploadedBytes)} / {formatBytes(task.totalBytes)} + + {canCancel && ( + + )} + {canClear && ( + + )} +
+
+ ); +} + +async function uploadTask( + task: UploadTask, + setTasks: React.Dispatch>, + cancelledTaskIdsRef: React.MutableRefObject>, + activeAbortRef: React.MutableRefObject, + activeUploadIdsRef: React.MutableRefObject>, +) { + updateTask(setTasks, task.id, { + status: 'uploading', + uploadedBytes: 0, + currentFileName: null, + error: null, + }); + + const session = await createFileUpload({ + destination_path: task.destinationPath, + chunk_size: uploadChunkSize, + files: task.files.map((source, index) => ({ + client_file_id: `${task.id}-${index}`, + relative_path: source.relativePath, + size: source.file.size, + })), + }); + + activeUploadIdsRef.current.set(task.id, session.upload_id); + updateTask(setTasks, task.id, { uploadId: session.upload_id }); + + if (cancelledTaskIdsRef.current.has(task.id)) { + await cancelFileUpload(session.upload_id).catch(() => undefined); + throw new Error('Upload cancelled'); + } + + const heartbeatId = window.setInterval(() => { + void heartbeatFileUpload(session.upload_id).catch(() => undefined); + }, heartbeatIntervalMs); + + try { + await uploadSessionFiles( + task, + session, + setTasks, + cancelledTaskIdsRef, + activeAbortRef, + ); + + updateTask(setTasks, task.id, { + status: 'complete', + uploadedBytes: task.totalBytes, + currentFileName: null, + }); + } catch (error) { + if (cancelledTaskIdsRef.current.has(task.id)) { + updateTask(setTasks, task.id, { + status: 'cancelled', + currentFileName: null, + }); + throw error; + } + + await cancelFileUpload(session.upload_id).catch(() => undefined); + throw error; + } finally { + window.clearInterval(heartbeatId); + activeUploadIdsRef.current.delete(task.id); + } +} + +async function uploadSessionFiles( + task: UploadTask, + session: CreateFileUploadResponse, + setTasks: React.Dispatch>, + cancelledTaskIdsRef: React.MutableRefObject>, + activeAbortRef: React.MutableRefObject, +) { + let completedBytes = 0; + + for (let index = 0; index < task.files.length; index++) { + throwIfCancelled(task.id, cancelledTaskIdsRef); + + const source = task.files[index]; + const serverFile = session.files.find( + (file) => file.client_file_id === `${task.id}-${index}`, + ); + if (!serverFile) { + throw new Error('Upload server did not return file metadata'); + } + + updateTask(setTasks, task.id, { + status: 'uploading', + currentFileName: source.relativePath, + }); + + const hasher = new IncrementalSha256(); + let fileUploadedBytes = 0; + + for ( + let chunkIndex = 0; + chunkIndex < serverFile.total_chunks; + chunkIndex++ + ) { + throwIfCancelled(task.id, cancelledTaskIdsRef); + + const start = chunkIndex * serverFile.chunk_size; + const end = Math.min(source.file.size, start + serverFile.chunk_size); + const chunk = source.file.slice(start, end); + const bytes = new Uint8Array(await chunk.arrayBuffer()); + hasher.update(bytes); + + await uploadChunkWithRetry({ + taskId: task.id, + uploadId: session.upload_id, + fileId: serverFile.file_id, + chunkIndex, + chunk, + completedBytes, + fileUploadedBytes, + setTasks, + cancelledTaskIdsRef, + activeAbortRef, + }); + + fileUploadedBytes += chunk.size; + updateTask(setTasks, task.id, { + uploadedBytes: completedBytes + fileUploadedBytes, + }); + } + + updateTask(setTasks, task.id, { + status: 'verifying', + currentFileName: source.relativePath, + }); + const sha256 = hasher.digestHex(); + throwIfCancelled(task.id, cancelledTaskIdsRef); + + const controller = new AbortController(); + activeAbortRef.current = controller; + try { + await completeFileUpload({ + uploadId: session.upload_id, + fileId: serverFile.file_id, + sha256, + signal: controller.signal, + }); + } finally { + if (activeAbortRef.current === controller) { + activeAbortRef.current = null; + } + } + + completedBytes += source.file.size; + updateTask(setTasks, task.id, { + uploadedBytes: completedBytes, + }); + } +} + +async function uploadChunkWithRetry({ + taskId, + uploadId, + fileId, + chunkIndex, + chunk, + completedBytes, + fileUploadedBytes, + setTasks, + cancelledTaskIdsRef, + activeAbortRef, +}: { + taskId: string; + uploadId: string; + fileId: string; + chunkIndex: number; + chunk: Blob; + completedBytes: number; + fileUploadedBytes: number; + setTasks: React.Dispatch>; + cancelledTaskIdsRef: React.MutableRefObject>; + activeAbortRef: React.MutableRefObject; +}) { + let lastError: unknown; + for (let attempt = 1; attempt <= maxChunkAttempts; attempt++) { + throwIfCancelled(taskId, cancelledTaskIdsRef); + + const controller = new AbortController(); + activeAbortRef.current = controller; + + try { + await uploadFileChunk({ + uploadId, + fileId, + chunkIndex, + chunk, + signal: controller.signal, + onUploadProgress: (loaded) => { + updateTask(setTasks, taskId, { + uploadedBytes: completedBytes + fileUploadedBytes + loaded, + }); + }, + }); + return; + } catch (error) { + lastError = error; + if ( + cancelledTaskIdsRef.current.has(taskId) || + !shouldRetryUploadError(error) || + attempt === maxChunkAttempts + ) { + break; + } + + await wait(500 * 2 ** (attempt - 1)); + } finally { + activeAbortRef.current = null; + } + } + + throw lastError; +} + +function updateTask( + setTasks: React.Dispatch>, + taskId: string, + patch: Partial, +) { + setTasks((current) => + current.map((task) => { + if (task.id !== taskId) { + return task; + } + + return { + ...task, + ...patch, + }; + }), + ); +} + +function throwIfCancelled( + taskId: string, + cancelledTaskIdsRef: React.MutableRefObject>, +) { + if (cancelledTaskIdsRef.current.has(taskId)) { + throw new Error('Upload cancelled'); + } +} + +function shouldRetryUploadError(error: unknown) { + if (error instanceof APIError) { + if (error.status === 0) { + return true; + } + + return error.status >= 500 && error.status !== 507; + } + + return false; +} + +function uploadErrorMessage(error: unknown) { + if (error instanceof APIError) { + if (error.status === 0) { + return 'Unable to reach upload server after multiple retries'; + } + + return error.getErrorMessage(); + } + + if (error instanceof Error) { + return error.message; + } + + return 'Upload failed'; +} + +function wait(ms: number) { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +function dedupeUploadSources(sources: UploadSource[]) { + const seen = new Set(); + const result: UploadSource[] = []; + for (const source of sources) { + const relativePath = normalizeRelativeUploadPath(source.relativePath); + if (!relativePath || seen.has(relativePath.toLowerCase())) { + continue; + } + + seen.add(relativePath.toLowerCase()); + result.push({ + file: source.file, + relativePath, + }); + } + + return result; +} + +function normalizeRelativeUploadPath(path: string) { + return path + .replace(/\\/g, '/') + .split('/') + .map((part) => part.trim()) + .filter(Boolean) + .join('/'); +} + +function hasFileDrag(dataTransfer: DataTransfer) { + return Array.from(dataTransfer.types).includes('Files'); +} + +async function uploadSourcesFromDataTransfer(dataTransfer: DataTransfer) { + const itemEntries = Array.from(dataTransfer.items) + .map((item) => (item as DataTransferItemWithEntry).webkitGetAsEntry?.()) + .filter((entry): entry is BrowserEntry => Boolean(entry)); + + if (itemEntries.length > 0) { + const sources: UploadSource[] = []; + for (const entry of itemEntries) { + sources.push(...(await uploadSourcesFromEntry(entry, ''))); + } + + return sources; + } + + return Array.from(dataTransfer.files).map((file) => ({ + file, + relativePath: file.name, + })); +} + +async function uploadSourcesFromEntry( + entry: BrowserEntry, + parentPath: string, +): Promise { + if (entry.isFile) { + const file = await readEntryFile(entry as BrowserFileEntry); + return [ + { + file, + relativePath: parentPath ? `${parentPath}/${file.name}` : file.name, + }, + ]; + } + + const directory = entry as BrowserDirectoryEntry; + const nextParentPath = parentPath + ? `${parentPath}/${directory.name}` + : directory.name; + const entries = await readDirectoryEntries(directory); + const sources: UploadSource[] = []; + for (const child of entries) { + sources.push( + ...(await uploadSourcesFromEntry(child as BrowserEntry, nextParentPath)), + ); + } + + return sources; +} + +function readEntryFile(entry: BrowserFileEntry) { + return new Promise((resolve, reject) => { + entry.file(resolve, reject); + }); +} + +async function readDirectoryEntries(directory: BrowserDirectoryEntry) { + const reader = directory.createReader(); + const entries: FileSystemEntry[] = []; + + for (;;) { + const batch = await new Promise((resolve, reject) => { + reader.readEntries(resolve, reject); + }); + if (batch.length === 0) { + break; + } + + entries.push(...batch); + } + + return entries; +} + +class IncrementalSha256 { + private hash = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, + 0x1f83d9ab, 0x5be0cd19, + ]); + + private buffer = new Uint8Array(64); + private bufferLength = 0; + private bytesHashed = 0; + private finished = false; + private temp = new Uint32Array(64); + + update(data: Uint8Array) { + if (this.finished) { + throw new Error('Hash is already finalized'); + } + + let position = 0; + this.bytesHashed += data.length; + + while (position < data.length) { + const take = Math.min(data.length - position, 64 - this.bufferLength); + this.buffer.set( + data.subarray(position, position + take), + this.bufferLength, + ); + this.bufferLength += take; + position += take; + + if (this.bufferLength === 64) { + this.processBlock(this.buffer); + this.bufferLength = 0; + } + } + } + + digestHex() { + this.finish(); + const bytes = new Uint8Array(32); + for (let i = 0; i < this.hash.length; i++) { + bytes[i * 4] = this.hash[i] >>> 24; + bytes[i * 4 + 1] = this.hash[i] >>> 16; + bytes[i * 4 + 2] = this.hash[i] >>> 8; + bytes[i * 4 + 3] = this.hash[i]; + } + + return Array.from(bytes) + .map((byte) => byte.toString(16).padStart(2, '0')) + .join(''); + } + + private finish() { + if (this.finished) { + return; + } + + const bytesHashed = this.bytesHashed; + this.buffer[this.bufferLength++] = 0x80; + + if (this.bufferLength > 56) { + this.buffer.fill(0, this.bufferLength, 64); + this.processBlock(this.buffer); + this.bufferLength = 0; + } + + this.buffer.fill(0, this.bufferLength, 56); + const bitsHigh = Math.floor(bytesHashed / 0x20000000); + const bitsLow = (bytesHashed << 3) >>> 0; + this.buffer[56] = bitsHigh >>> 24; + this.buffer[57] = bitsHigh >>> 16; + this.buffer[58] = bitsHigh >>> 8; + this.buffer[59] = bitsHigh; + this.buffer[60] = bitsLow >>> 24; + this.buffer[61] = bitsLow >>> 16; + this.buffer[62] = bitsLow >>> 8; + this.buffer[63] = bitsLow; + this.processBlock(this.buffer); + this.finished = true; + } + + private processBlock(block: Uint8Array) { + const words = this.temp; + for (let i = 0; i < 16; i++) { + const offset = i * 4; + words[i] = + (block[offset] << 24) | + (block[offset + 1] << 16) | + (block[offset + 2] << 8) | + block[offset + 3]; + } + + for (let i = 16; i < 64; i++) { + const s0 = + rotateRight(words[i - 15], 7) ^ + rotateRight(words[i - 15], 18) ^ + (words[i - 15] >>> 3); + const s1 = + rotateRight(words[i - 2], 17) ^ + rotateRight(words[i - 2], 19) ^ + (words[i - 2] >>> 10); + words[i] = (words[i - 16] + s0 + words[i - 7] + s1) >>> 0; + } + + let a = this.hash[0]; + let b = this.hash[1]; + let c = this.hash[2]; + let d = this.hash[3]; + let e = this.hash[4]; + let f = this.hash[5]; + let g = this.hash[6]; + let h = this.hash[7]; + + for (let i = 0; i < 64; i++) { + const s1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25); + const ch = (e & f) ^ (~e & g); + const temp1 = (h + s1 + ch + sha256K[i] + words[i]) >>> 0; + const s0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22); + const maj = (a & b) ^ (a & c) ^ (b & c); + const temp2 = (s0 + maj) >>> 0; + + h = g; + g = f; + f = e; + e = (d + temp1) >>> 0; + d = c; + c = b; + b = a; + a = (temp1 + temp2) >>> 0; + } + + this.hash[0] = (this.hash[0] + a) >>> 0; + this.hash[1] = (this.hash[1] + b) >>> 0; + this.hash[2] = (this.hash[2] + c) >>> 0; + this.hash[3] = (this.hash[3] + d) >>> 0; + this.hash[4] = (this.hash[4] + e) >>> 0; + this.hash[5] = (this.hash[5] + f) >>> 0; + this.hash[6] = (this.hash[6] + g) >>> 0; + this.hash[7] = (this.hash[7] + h) >>> 0; + } +} + +function rotateRight(value: number, bits: number) { + return (value >>> bits) | (value << (32 - bits)); +} + +const sha256K = new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]); diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx index df5ffdb..81acd5c 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-tree.tsx @@ -17,6 +17,7 @@ import { Eye, Pin, Download, + Upload as UploadIcon, } from 'lucide-react'; import { Breadcrumb, @@ -69,6 +70,7 @@ import { formatBytes, formatDate, cn } from '@/lib/util'; import { queryKeys } from '@/constants'; import { toast } from 'sonner'; import { usePermissions } from '@/hooks/use-permissions'; +import { useFileBrowserUploader } from '@/components/file-browser-uploader'; interface FileTreeProps { initialPath?: string; @@ -146,6 +148,7 @@ export function FileTree({ initialPath }: FileTreeProps) { const { hasPermission } = usePermissions(); const canManageServer = hasPermission('manage_server'); const canDownloadFiles = hasPermission('download_files'); + const canEditFiles = hasPermission('edit_files'); const [internalPath, setInternalPath] = useState(null); const [showDotfiles, setShowDotfiles] = useState(false); const contextMenuFileRef = useRef(null); @@ -640,10 +643,14 @@ export function FileTree({ initialPath }: FileTreeProps) { return; } - handleDirectoryDownloadResponse( - directoryDownloadStatus, - directoryDownloadRun.path, - ); + const timeoutId = window.setTimeout(() => { + handleDirectoryDownloadResponse( + directoryDownloadStatus, + directoryDownloadRun.path, + ); + }, 0); + + return () => window.clearTimeout(timeoutId); }, [ directoryDownloadStatus, directoryDownloadRun, @@ -655,17 +662,21 @@ export function FileTree({ initialPath }: FileTreeProps) { return; } - clearDirectoryDownloadToast(); - setDirectoryDownloadRun(null); + const timeoutId = window.setTimeout(() => { + clearDirectoryDownloadToast(); + setDirectoryDownloadRun(null); - const errorMessage = - directoryDownloadError instanceof APIError - ? directoryDownloadError.getErrorMessage() - : directoryDownloadError instanceof Error - ? directoryDownloadError.message - : 'Failed to get directory download status'; + const errorMessage = + directoryDownloadError instanceof APIError + ? directoryDownloadError.getErrorMessage() + : directoryDownloadError instanceof Error + ? directoryDownloadError.message + : 'Failed to get directory download status'; + + toast.error(errorMessage); + }, 0); - toast.error(errorMessage); + return () => window.clearTimeout(timeoutId); }, [ clearDirectoryDownloadToast, directoryDownloadError, @@ -806,6 +817,16 @@ export function FileTree({ initialPath }: FileTreeProps) { return true; }; + const handleUploadComplete = useCallback(() => { + queryClient.invalidateQueries({ queryKey: queryKeys.fileTree() }); + }, [queryClient]); + + const uploader = useFileBrowserUploader({ + destinationPath: currentPath, + canUpload: canEditFiles && Boolean(currentPath), + onUploaded: handleUploadComplete, + }); + if (error) { const errorMessage = error instanceof APIError @@ -822,7 +843,18 @@ export function FileTree({ initialPath }: FileTreeProps) { } return ( -
+
+ {uploader.isDragging && ( +
+
+ + Drop to upload +
+
+ )} {/* Breadcrumb Navigation and Show Dotfiles Checkbox */}
@@ -875,6 +907,18 @@ export function FileTree({ initialPath }: FileTreeProps) { > Show dotfiles + {uploader.canUpload && ( + + )} {canPinCurrentPath() && (
); } diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts index 570e2d9..9b8f047 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts @@ -21,6 +21,15 @@ export const API_ROUTES = { REVERT_FILE: '/api/file-tree/revert-file', DUPLICATE_FILE: '/api/file-tree/duplicate-file', REVISION_COUNT: '/api/file-tree/revision-summary', + FILE_UPLOADS: '/api/file-tree/uploads', + FILE_UPLOAD_CHUNK: (uploadId: string, fileId: string, chunkIndex: number) => + `/api/file-tree/uploads/${encodeURIComponent(uploadId)}/files/${encodeURIComponent(fileId)}/chunks/${chunkIndex}`, + FILE_UPLOAD_COMPLETE: (uploadId: string, fileId: string) => + `/api/file-tree/uploads/${encodeURIComponent(uploadId)}/files/${encodeURIComponent(fileId)}/complete`, + FILE_UPLOAD_HEARTBEAT: (uploadId: string) => + `/api/file-tree/uploads/${encodeURIComponent(uploadId)}/heartbeat`, + FILE_UPLOAD_CANCEL: (uploadId: string) => + `/api/file-tree/uploads/${encodeURIComponent(uploadId)}`, FILE_DOWNLOAD_LINK: '/api/file-tree/download-link', DIRECTORY_DOWNLOAD_LINK: '/api/file-tree/directory-download-link', DIRECTORY_DOWNLOAD_STATUS: (runId: number) => @@ -374,6 +383,79 @@ const DuplicateFileResponseSchema = z.object({ export type DuplicateFileResponse = z.infer; +const CreateFileUploadRequestFileSchema = z.object({ + client_file_id: z.string().min(1), + relative_path: z.string().min(1), + size: z.number().int().nonnegative(), +}); + +const CreateFileUploadRequestSchema = z.object({ + destination_path: z.string().min(1), + chunk_size: z.number().int().positive(), + files: z.array(CreateFileUploadRequestFileSchema).min(1), +}); + +export type CreateFileUploadRequest = z.infer< + typeof CreateFileUploadRequestSchema +>; + +const CreateFileUploadResponseFileSchema = z.object({ + client_file_id: z.string(), + file_id: z.string(), + relative_path: z.string(), + resolved_relative_path: z.string(), + target_path: z.string(), + size: z.number().int().nonnegative(), + chunk_size: z.number().int().positive(), + total_chunks: z.number().int().nonnegative(), +}); + +const CreateFileUploadResponseSchema = z.object({ + upload_id: z.string(), + expires_at: z.string(), + files: z.array(CreateFileUploadResponseFileSchema), +}); + +export type CreateFileUploadResponse = z.infer< + typeof CreateFileUploadResponseSchema +>; + +const FileUploadChunkResponseSchema = z.object({ + message: z.string(), + received_chunks: z.number().int().nonnegative(), + total_chunks: z.number().int().nonnegative(), +}); + +export type FileUploadChunkResponse = z.infer< + typeof FileUploadChunkResponseSchema +>; + +const CompleteFileUploadRequestSchema = z.object({ + sha256: z.string().length(64), +}); + +const CompleteFileUploadResponseSchema = z.object({ + message: z.string(), + file_id: z.string(), + relative_path: z.string(), + resolved_relative_path: z.string(), + final_path: z.string(), + sha256: z.string(), +}); + +export type CompleteFileUploadResponse = z.infer< + typeof CompleteFileUploadResponseSchema +>; + +const FileUploadHeartbeatResponseSchema = z.object({ + upload_id: z.string(), + expires_at: z.string(), +}); + +export type FileUploadHeartbeatResponse = z.infer< + typeof FileUploadHeartbeatResponseSchema +>; + const DownloadLinkResponseSchema = z.object({ download_url: z.string(), expires_at: z.string(), @@ -1160,6 +1242,80 @@ export async function duplicateFile( ); } +export async function createFileUpload( + data: CreateFileUploadRequest, +): Promise { + const response = await axiosInstance.post( + API_ROUTES.FILE_UPLOADS, + CreateFileUploadRequestSchema.parse(data), + ); + return validateResponse( + CreateFileUploadResponseSchema, + response.data, + API_ROUTES.FILE_UPLOADS, + ); +} + +export async function uploadFileChunk(params: { + uploadId: string; + fileId: string; + chunkIndex: number; + chunk: Blob; + signal?: AbortSignal; + onUploadProgress?: (loaded: number, total: number) => void; +}): Promise { + const route = API_ROUTES.FILE_UPLOAD_CHUNK( + params.uploadId, + params.fileId, + params.chunkIndex, + ); + const response = await axiosInstance.put(route, params.chunk, { + headers: { + 'Content-Type': 'application/octet-stream', + }, + signal: params.signal, + onUploadProgress: (event) => { + params.onUploadProgress?.(event.loaded, event.total || params.chunk.size); + }, + }); + return validateResponse(FileUploadChunkResponseSchema, response.data, route); +} + +export async function completeFileUpload(params: { + uploadId: string; + fileId: string; + sha256: string; + signal?: AbortSignal; +}): Promise { + const route = API_ROUTES.FILE_UPLOAD_COMPLETE(params.uploadId, params.fileId); + const response = await axiosInstance.post( + route, + CompleteFileUploadRequestSchema.parse({ sha256: params.sha256 }), + { signal: params.signal }, + ); + return validateResponse( + CompleteFileUploadResponseSchema, + response.data, + route, + ); +} + +export async function heartbeatFileUpload( + uploadId: string, +): Promise { + const route = API_ROUTES.FILE_UPLOAD_HEARTBEAT(uploadId); + const response = await axiosInstance.post(route); + return validateResponse( + FileUploadHeartbeatResponseSchema, + response.data, + route, + ); +} + +export async function cancelFileUpload(uploadId: string): Promise { + await axiosInstance.delete(API_ROUTES.FILE_UPLOAD_CANCEL(uploadId)); +} + export async function createFileDownloadLink(params: { path: string; }): Promise { diff --git a/internal/constants/constants.go b/internal/constants/constants.go index af02881..679a68c 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -14,6 +14,9 @@ const ( ErrorCodePathIsDirectory = "PATH_IS_DIRECTORY" ErrorCodeFileNotViewable = "FILE_NOT_VIEWABLE" ErrorCodeFileReadError = "FILE_READ_ERROR" + ErrorCodeDiskFull = "DISK_FULL" + ErrorCodeHashMismatch = "HASH_MISMATCH" + ErrorCodeUploadExpired = "UPLOAD_EXPIRED" ) const ( diff --git a/internal/server/file_system_routes.go b/internal/server/file_system_routes.go index df996e6..3435ae4 100644 --- a/internal/server/file_system_routes.go +++ b/internal/server/file_system_routes.go @@ -61,6 +61,11 @@ func (s *Server) InitializeFileSystemRoutes(r *chi.Mux) { r.Post("/revert-file", s.handleRevertFile) r.Post("/duplicate-file", s.handleDuplicateFile) r.Get("/revision-summary", s.handleRevisionSummary) + r.Post("/uploads", s.handleCreateFileUpload) + r.Put("/uploads/{upload_id}/files/{file_id}/chunks/{chunk_index}", s.handleUploadFileChunk) + r.Post("/uploads/{upload_id}/files/{file_id}/complete", s.handleCompleteUploadedFile) + r.Post("/uploads/{upload_id}/heartbeat", s.handleFileUploadHeartbeat) + r.Delete("/uploads/{upload_id}", s.handleCancelFileUpload) r.Post("/download-link", s.handleCreateFileDownloadLink) r.Post("/directory-download-link", s.handleCreateDirectoryDownloadLink) r.Get("/directory-downloads/{run_id}", s.handleGetDirectoryDownloadStatus) @@ -147,6 +152,10 @@ func (s *Server) getSystemRoots(showDotfiles bool) (*FileNode, error) { } for _, entry := range entries { + if entry.Name() == fileUploadTempDirName { + continue + } + if !showDotfiles && len(entry.Name()) > 0 && entry.Name()[0] == '.' { continue } @@ -186,6 +195,10 @@ func (s *Server) getDirectoryNode(path string, showDotfiles bool) (*FileNode, er entries, err := s.fileEditor.ReadDir(path) if err == nil { for _, entry := range entries { + if entry.Name() == fileUploadTempDirName { + continue + } + if !showDotfiles && len(entry.Name()) > 0 && entry.Name()[0] == '.' { continue } diff --git a/internal/server/file_upload_routes.go b/internal/server/file_upload_routes.go new file mode 100644 index 0000000..982949b --- /dev/null +++ b/internal/server/file_upload_routes.go @@ -0,0 +1,1091 @@ +package server + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-playground/validator/v10" + + "github.com/omnihance/omnihance-a3-agent/internal/constants" + "github.com/omnihance/omnihance-a3-agent/internal/logger" + "github.com/omnihance/omnihance-a3-agent/internal/permissions" + "github.com/omnihance/omnihance-a3-agent/internal/services" + "github.com/omnihance/omnihance-a3-agent/internal/utils" +) + +const ( + fileUploadTempDirName = ".omnihance-upload-temp" + fileUploadSessionTTL = 2 * time.Hour + fileUploadCleanupInterval = 15 * time.Minute + fileUploadDefaultRegistryDirName = ".revisions" + fileUploadRegistryFileName = "file-upload-temp-registry.json" +) + +type fileUploadManager struct { + fileEditor services.FileEditorService + log logger.Logger + registryPath string + + registryMu sync.Mutex + mu sync.Mutex + sessions map[string]*fileUploadSession + reservations map[string]string + stopCh chan struct{} + started bool +} + +type fileUploadSession struct { + ID string + DestinationPath string + TempRoot string + CreatedAt time.Time + ExpiresAt time.Time + LastSeenAt time.Time + Files map[string]*fileUploadFile + OrderedFiles []*fileUploadFile + Reservations []string + Cancelled bool +} + +type fileUploadFile struct { + ID string + ClientFileID string + RelativePath string + ResolvedRelativePath string + TargetPath string + TempPath string + Size int64 + ChunkSize int64 + TotalChunks int + ReceivedChunks map[int]int64 + Completed bool + FinalPath string +} + +func newFileUploadManager(registryDir string, fileEditor services.FileEditorService, log logger.Logger) *fileUploadManager { + if registryDir == "" { + registryDir = fileUploadDefaultRegistryDirName + } + + return &fileUploadManager{ + fileEditor: fileEditor, + log: log, + registryPath: filepath.Join(registryDir, fileUploadRegistryFileName), + sessions: make(map[string]*fileUploadSession), + reservations: make(map[string]string), + stopCh: make(chan struct{}), + } +} + +func (m *fileUploadManager) Start() error { + m.mu.Lock() + if m.started { + m.mu.Unlock() + return nil + } + + m.started = true + m.mu.Unlock() + + if err := m.cleanupRegisteredTempRoots(); err != nil { + return err + } + + go func() { + ticker := time.NewTicker(fileUploadCleanupInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + m.cleanupExpiredSessions(time.Now()) + case <-m.stopCh: + return + } + } + }() + + return nil +} + +func (m *fileUploadManager) Stop() { + m.mu.Lock() + if !m.started { + m.mu.Unlock() + return + } + + m.started = false + close(m.stopCh) + m.stopCh = make(chan struct{}) + m.mu.Unlock() +} + +func (s *Server) ensureUploadManager() *fileUploadManager { + if s.uploadManager != nil { + return s.uploadManager + } + + registryDir := fileUploadDefaultRegistryDirName + if s.cfg != nil && s.cfg.RevisionsDirectory != "" { + registryDir = s.cfg.RevisionsDirectory + } + + s.uploadManager = newFileUploadManager(registryDir, s.fileEditor, s.log) + if err := s.uploadManager.Start(); err != nil && s.log != nil { + s.log.Error("Could not start file upload manager", logger.Field{Key: "error", Value: err}) + } + + return s.uploadManager +} + +func (s *Server) handleCreateFileUpload(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionEditFiles) { + return + } + + var req CreateFileUploadRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeFileUploadError(w, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Invalid request body: "+err.Error())) + return + } + + validate := validator.New() + if err := validate.Struct(req); err != nil { + var validationErrors []string + for _, fieldErr := range err.(validator.ValidationErrors) { + validationErrors = append(validationErrors, fieldErr.Field()+" is required") + } + + writeFileUploadError(w, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, strings.Join(validationErrors, ", "))) + return + } + + response, err := s.ensureUploadManager().CreateSession(req) + if err != nil { + writeFileUploadError(w, fileUploadErrorFor(err)) + return + } + + _ = utils.WriteJSONResponseWithStatus(w, http.StatusCreated, response) +} + +func (s *Server) handleUploadFileChunk(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionEditFiles) { + return + } + + chunkIndex, err := strconv.Atoi(chi.URLParam(r, "chunk_index")) + if err != nil || chunkIndex < 0 { + writeFileUploadError(w, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Invalid chunk index")) + return + } + + response, err := s.ensureUploadManager().UploadChunk( + chi.URLParam(r, "upload_id"), + chi.URLParam(r, "file_id"), + chunkIndex, + r.Body, + ) + if err != nil { + writeFileUploadError(w, fileUploadErrorFor(err)) + return + } + + _ = utils.WriteJSONResponse(w, response) +} + +func (s *Server) handleCompleteUploadedFile(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionEditFiles) { + return + } + + var req CompleteFileUploadRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeFileUploadError(w, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Invalid request body: "+err.Error())) + return + } + + validate := validator.New() + if err := validate.Struct(req); err != nil { + writeFileUploadError(w, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "sha256 is required")) + return + } + + response, err := s.ensureUploadManager().CompleteFile(chi.URLParam(r, "upload_id"), chi.URLParam(r, "file_id"), req.SHA256) + if err != nil { + writeFileUploadError(w, fileUploadErrorFor(err)) + return + } + + _ = utils.WriteJSONResponse(w, response) +} + +func (s *Server) handleFileUploadHeartbeat(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionEditFiles) { + return + } + + response, err := s.ensureUploadManager().Heartbeat(chi.URLParam(r, "upload_id")) + if err != nil { + writeFileUploadError(w, fileUploadErrorFor(err)) + return + } + + _ = utils.WriteJSONResponse(w, response) +} + +func (s *Server) handleCancelFileUpload(w http.ResponseWriter, r *http.Request) { + if !s.requireUserPermission(w, r, permissions.ActionEditFiles) { + return + } + + if err := s.ensureUploadManager().CancelSession(chi.URLParam(r, "upload_id")); err != nil { + writeFileUploadError(w, fileUploadErrorFor(err)) + return + } + + _ = utils.WriteJSONResponse(w, map[string]interface{}{ + "message": "Upload cancelled", + }) +} + +func (m *fileUploadManager) CreateSession(req CreateFileUploadRequest) (*CreateFileUploadResponse, error) { + now := time.Now() + m.cleanupExpiredSessions(now) + + destinationPath := filepath.Clean(strings.TrimSpace(req.DestinationPath)) + if destinationPath == "" || destinationPath == "." { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Destination path is required") + } + + info, err := m.fileEditor.Stat(destinationPath) + if err != nil { + if m.fileEditor.IsNotExist(err) { + return nil, newFileUploadHTTPError(http.StatusNotFound, constants.ErrorCodeNotFound, "Destination path not found") + } + + return nil, classifyFileUploadError(err, "Cannot read destination path") + } + + if !info.IsDir() { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Destination path must be a directory") + } + + cleanFiles, err := cleanUploadFiles(req) + if err != nil { + return nil, err + } + + uploadID, err := generateUploadID() + if err != nil { + return nil, classifyFileUploadError(err, "Failed to create upload ID") + } + + tempRoot := filepath.Join(destinationPath, fileUploadTempDirName, uploadID) + if err := m.fileEditor.MkdirAll(tempRoot, 0700); err != nil { + return nil, classifyFileUploadError(err, "Failed to create upload temp directory") + } + + if err := m.registerTempRoot(tempRoot); err != nil { + _ = m.fileEditor.RemoveAll(tempRoot) + return nil, classifyFileUploadError(err, "Failed to register upload temp directory") + } + + session := &fileUploadSession{ + ID: uploadID, + DestinationPath: destinationPath, + TempRoot: tempRoot, + CreatedAt: now, + ExpiresAt: now.Add(fileUploadSessionTTL), + LastSeenAt: now, + Files: make(map[string]*fileUploadFile, len(cleanFiles)), + OrderedFiles: make([]*fileUploadFile, 0, len(cleanFiles)), + Reservations: []string{}, + } + + response := &CreateFileUploadResponse{ + UploadID: uploadID, + ExpiresAt: session.ExpiresAt.Format(time.RFC3339), + Files: make([]CreateFileUploadResponseFile, 0, len(cleanFiles)), + } + + m.mu.Lock() + defer m.mu.Unlock() + + topLevelResolvedNames := map[string]string{} + requestReservations := map[string]bool{} + for _, file := range cleanFiles { + topLevelName := uploadTopLevelName(file.CleanRelativePath) + if _, ok := topLevelResolvedNames[topLevelName]; ok { + continue + } + + targetPath, resolvedName, err := m.resolveAvailableTargetLocked(destinationPath, topLevelName, requestReservations) + if err != nil { + m.cancelSessionLocked(session) + return nil, err + } + + topLevelResolvedNames[topLevelName] = resolvedName + reservationKey := m.reserveUploadPathLocked(session, targetPath) + requestReservations[reservationKey] = true + } + + for index, cleanFile := range cleanFiles { + fileID := strconv.Itoa(index + 1) + resolvedRelativePath := resolveUploadRelativePath(cleanFile.CleanRelativePath, topLevelResolvedNames) + targetPath := filepath.Join(destinationPath, resolvedRelativePath) + reservationKey := normalizeUploadReservationPath(targetPath) + + if reservedUploadID, reserved := m.reservations[reservationKey]; reserved && reservedUploadID != uploadID { + targetPath, resolvedRelativePath, err = m.resolveAvailableNestedTargetLocked(destinationPath, resolvedRelativePath) + if err != nil { + m.cancelSessionLocked(session) + return nil, err + } + } + + m.reserveUploadPathLocked(session, targetPath) + + uploadFile := &fileUploadFile{ + ID: fileID, + ClientFileID: cleanFile.ClientFileID, + RelativePath: cleanFile.OriginalRelativePath, + ResolvedRelativePath: resolvedRelativePath, + TargetPath: targetPath, + TempPath: filepath.Join(tempRoot, fileID+".part"), + Size: cleanFile.Size, + ChunkSize: req.ChunkSize, + TotalChunks: totalUploadChunks(cleanFile.Size, req.ChunkSize), + ReceivedChunks: map[int]int64{}, + } + + session.Files[fileID] = uploadFile + session.OrderedFiles = append(session.OrderedFiles, uploadFile) + response.Files = append(response.Files, CreateFileUploadResponseFile{ + ClientFileID: uploadFile.ClientFileID, + FileID: uploadFile.ID, + RelativePath: uploadFile.RelativePath, + ResolvedRelativePath: uploadFile.ResolvedRelativePath, + TargetPath: uploadFile.TargetPath, + Size: uploadFile.Size, + ChunkSize: uploadFile.ChunkSize, + TotalChunks: uploadFile.TotalChunks, + }) + } + + m.sessions[uploadID] = session + return response, nil +} + +func (m *fileUploadManager) UploadChunk(uploadID string, fileID string, chunkIndex int, body io.Reader) (*FileUploadChunkResponse, error) { + m.mu.Lock() + session, file, err := m.activeFileLocked(uploadID, fileID, time.Now()) + if err != nil { + m.mu.Unlock() + return nil, err + } + + if file.Completed { + m.mu.Unlock() + return nil, newFileUploadHTTPError(http.StatusConflict, constants.ErrorCodeBadRequest, "File is already complete") + } + + if chunkIndex >= file.TotalChunks { + m.mu.Unlock() + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Chunk index is out of range") + } + + expectedSize := expectedUploadChunkSize(file.Size, file.ChunkSize, chunkIndex) + chunkData, err := io.ReadAll(io.LimitReader(body, expectedSize+1)) + if err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to read upload chunk") + } + + if int64(len(chunkData)) != expectedSize { + m.mu.Unlock() + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Chunk size does not match expected size") + } + + if err := m.fileEditor.MkdirAll(filepath.Dir(file.TempPath), 0700); err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to create upload temp directory") + } + + tempFile, err := m.fileEditor.OpenFile(file.TempPath, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to open upload temp file") + } + + _, writeErr := tempFile.WriteAt(chunkData, int64(chunkIndex)*file.ChunkSize) + closeErr := tempFile.Close() + if writeErr != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(writeErr, "Failed to write upload chunk") + } + + if closeErr != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(closeErr, "Failed to close upload temp file") + } + + file.ReceivedChunks[chunkIndex] = expectedSize + session.LastSeenAt = time.Now() + session.ExpiresAt = session.LastSeenAt.Add(fileUploadSessionTTL) + response := &FileUploadChunkResponse{ + Message: "Chunk uploaded", + ReceivedChunks: len(file.ReceivedChunks), + TotalChunks: file.TotalChunks, + } + m.mu.Unlock() + + return response, nil +} + +func (m *fileUploadManager) CompleteFile(uploadID string, fileID string, clientSHA256 string) (*CompleteFileUploadResponse, error) { + m.mu.Lock() + session, file, err := m.activeFileLocked(uploadID, fileID, time.Now()) + if err != nil { + m.mu.Unlock() + return nil, err + } + + if file.Completed { + response := &CompleteFileUploadResponse{ + Message: "File already uploaded", + FileID: file.ID, + RelativePath: file.RelativePath, + ResolvedRelativePath: file.ResolvedRelativePath, + FinalPath: file.FinalPath, + SHA256: clientSHA256, + } + m.mu.Unlock() + return response, nil + } + + if len(file.ReceivedChunks) != file.TotalChunks { + m.mu.Unlock() + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Not all chunks have been uploaded") + } + + if file.Size == 0 { + emptyFile, err := m.fileEditor.OpenFile(file.TempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to create empty upload temp file") + } + + if err := emptyFile.Close(); err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to close empty upload temp file") + } + } + + serverSHA256, err := hashUploadFile(file.TempPath) + if err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to hash uploaded file") + } + + normalizedClientHash := strings.ToLower(strings.TrimSpace(clientSHA256)) + if serverSHA256 != normalizedClientHash { + _ = m.fileEditor.Remove(file.TempPath) + m.releaseFileReservationLocked(session, file) + m.mu.Unlock() + return nil, newFileUploadHTTPError(http.StatusConflict, constants.ErrorCodeHashMismatch, "Uploaded file failed integrity check") + } + + finalPath := file.TargetPath + if _, err := m.fileEditor.Stat(finalPath); err == nil { + resolvedPath, resolvedRelativePath, err := m.resolveAvailableNestedTargetLocked(session.DestinationPath, file.ResolvedRelativePath) + if err != nil { + m.mu.Unlock() + return nil, err + } + + m.releaseFileReservationLocked(session, file) + finalPath = resolvedPath + file.TargetPath = resolvedPath + file.ResolvedRelativePath = resolvedRelativePath + m.reserveUploadPathLocked(session, finalPath) + } else if !m.fileEditor.IsNotExist(err) { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Cannot check final upload path") + } + + if err := m.fileEditor.MkdirAll(filepath.Dir(finalPath), 0755); err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to create upload destination directory") + } + + if err := os.Rename(file.TempPath, finalPath); err != nil { + m.mu.Unlock() + return nil, classifyFileUploadError(err, "Failed to finalize uploaded file") + } + + file.Completed = true + file.FinalPath = finalPath + session.LastSeenAt = time.Now() + session.ExpiresAt = session.LastSeenAt.Add(fileUploadSessionTTL) + m.releaseFileReservationLocked(session, file) + m.removeSessionIfCompleteLocked(session) + response := &CompleteFileUploadResponse{ + Message: "File uploaded successfully", + FileID: file.ID, + RelativePath: file.RelativePath, + ResolvedRelativePath: file.ResolvedRelativePath, + FinalPath: finalPath, + SHA256: serverSHA256, + } + m.mu.Unlock() + + return response, nil +} + +func (m *fileUploadManager) Heartbeat(uploadID string) (*FileUploadHeartbeatResponse, error) { + now := time.Now() + m.mu.Lock() + defer m.mu.Unlock() + + session, ok := m.sessions[uploadID] + if !ok { + return nil, newFileUploadHTTPError(http.StatusNotFound, constants.ErrorCodeNotFound, "Upload session not found") + } + + if now.Sub(session.LastSeenAt) > fileUploadSessionTTL { + m.cancelSessionLocked(session) + return nil, newFileUploadHTTPError(http.StatusGone, constants.ErrorCodeUploadExpired, "Upload session expired") + } + + session.LastSeenAt = now + session.ExpiresAt = now.Add(fileUploadSessionTTL) + return &FileUploadHeartbeatResponse{ + UploadID: session.ID, + ExpiresAt: session.ExpiresAt.Format(time.RFC3339), + }, nil +} + +func (m *fileUploadManager) CancelSession(uploadID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + session, ok := m.sessions[uploadID] + if !ok { + return newFileUploadHTTPError(http.StatusNotFound, constants.ErrorCodeNotFound, "Upload session not found") + } + + m.cancelSessionLocked(session) + return nil +} + +func (m *fileUploadManager) cleanupExpiredSessions(now time.Time) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, session := range m.sessions { + if now.Sub(session.LastSeenAt) > fileUploadSessionTTL { + m.cancelSessionLocked(session) + } + } +} + +func (m *fileUploadManager) cleanupRegisteredTempRoots() error { + m.registryMu.Lock() + defer m.registryMu.Unlock() + + tempRoots, err := m.readRegisteredTempRoots() + if err != nil { + return err + } + + for _, tempRoot := range tempRoots { + if err := m.fileEditor.RemoveAll(tempRoot); err != nil && !m.fileEditor.IsNotExist(err) && m.log != nil { + m.log.Warn("Failed to cleanup upload temp directory", logger.Field{Key: "path", Value: tempRoot}, logger.Field{Key: "error", Value: err}) + } + } + + return m.writeRegisteredTempRoots(nil) +} + +func (m *fileUploadManager) activeFileLocked(uploadID string, fileID string, now time.Time) (*fileUploadSession, *fileUploadFile, error) { + session, ok := m.sessions[uploadID] + if !ok { + return nil, nil, newFileUploadHTTPError(http.StatusNotFound, constants.ErrorCodeNotFound, "Upload session not found") + } + + if now.Sub(session.LastSeenAt) > fileUploadSessionTTL { + m.cancelSessionLocked(session) + return nil, nil, newFileUploadHTTPError(http.StatusGone, constants.ErrorCodeUploadExpired, "Upload session expired") + } + + file, ok := session.Files[fileID] + if !ok { + return nil, nil, newFileUploadHTTPError(http.StatusNotFound, constants.ErrorCodeNotFound, "Upload file not found") + } + + session.LastSeenAt = now + session.ExpiresAt = now.Add(fileUploadSessionTTL) + return session, file, nil +} + +func (m *fileUploadManager) cancelSessionLocked(session *fileUploadSession) { + session.Cancelled = true + m.releaseSessionReservationsLocked(session) + + delete(m.sessions, session.ID) + m.removeTempRoot(session.TempRoot) +} + +func (m *fileUploadManager) removeSessionIfCompleteLocked(session *fileUploadSession) { + for _, file := range session.OrderedFiles { + if !file.Completed { + return + } + } + + m.releaseSessionReservationsLocked(session) + delete(m.sessions, session.ID) + m.removeTempRoot(session.TempRoot) +} + +func (m *fileUploadManager) reserveUploadPathLocked(session *fileUploadSession, path string) string { + reservation := normalizeUploadReservationPath(path) + if m.reservations[reservation] != session.ID { + m.reservations[reservation] = session.ID + session.Reservations = append(session.Reservations, reservation) + } + + return reservation +} + +func (m *fileUploadManager) releaseFileReservationLocked(session *fileUploadSession, file *fileUploadFile) { + reservation := normalizeUploadReservationPath(file.TargetPath) + if m.reservations[reservation] == session.ID { + delete(m.reservations, reservation) + } +} + +func (m *fileUploadManager) releaseSessionReservationsLocked(session *fileUploadSession) { + for _, reservation := range session.Reservations { + if m.reservations[reservation] == session.ID { + delete(m.reservations, reservation) + } + } +} + +func (m *fileUploadManager) resolveAvailableTargetLocked(parentPath string, name string, requestReservations map[string]bool) (string, string, error) { + trimmedName := strings.TrimSpace(name) + if trimmedName == "" { + return "", "", newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload file name is required") + } + + baseName, extension := splitUploadFileName(trimmedName) + candidateName := trimmedName + counter := 0 + for { + candidatePath := filepath.Join(parentPath, candidateName) + available, err := m.uploadPathAvailableLocked(candidatePath, requestReservations) + if err != nil { + return "", "", err + } + + if available { + return candidatePath, candidateName, nil + } + + counter++ + if counter == 1 { + candidateName = fmt.Sprintf("%s (copy)%s", baseName, extension) + } else { + candidateName = fmt.Sprintf("%s (copy %d)%s", baseName, counter, extension) + } + } +} + +func (m *fileUploadManager) resolveAvailableNestedTargetLocked(destinationPath string, relativePath string) (string, string, error) { + parentRelativePath := filepath.Dir(relativePath) + fileName := filepath.Base(relativePath) + parentPath := destinationPath + if parentRelativePath != "." { + parentPath = filepath.Join(destinationPath, parentRelativePath) + } + + targetPath, resolvedName, err := m.resolveAvailableTargetLocked(parentPath, fileName, map[string]bool{}) + if err != nil { + return "", "", err + } + + if parentRelativePath == "." { + return targetPath, resolvedName, nil + } + + return targetPath, filepath.Join(parentRelativePath, resolvedName), nil +} + +func (m *fileUploadManager) uploadPathAvailableLocked(path string, requestReservations map[string]bool) (bool, error) { + reservationKey := normalizeUploadReservationPath(path) + if m.reservations[reservationKey] != "" || requestReservations[reservationKey] { + return false, nil + } + + if _, err := m.fileEditor.Stat(path); err == nil { + return false, nil + } else if !m.fileEditor.IsNotExist(err) { + return false, classifyFileUploadError(err, "Cannot check upload destination") + } + + return true, nil +} + +func (m *fileUploadManager) registerTempRoot(tempRoot string) error { + m.registryMu.Lock() + defer m.registryMu.Unlock() + + tempRoots, err := m.readRegisteredTempRoots() + if err != nil { + return err + } + + for _, existing := range tempRoots { + if filepath.Clean(existing) == filepath.Clean(tempRoot) { + return nil + } + } + + tempRoots = append(tempRoots, tempRoot) + return m.writeRegisteredTempRoots(tempRoots) +} + +func (m *fileUploadManager) unregisterTempRoot(tempRoot string) error { + m.registryMu.Lock() + defer m.registryMu.Unlock() + + tempRoots, err := m.readRegisteredTempRoots() + if err != nil { + return err + } + + filteredTempRoots := make([]string, 0, len(tempRoots)) + for _, existing := range tempRoots { + if filepath.Clean(existing) == filepath.Clean(tempRoot) { + continue + } + + filteredTempRoots = append(filteredTempRoots, existing) + } + + return m.writeRegisteredTempRoots(filteredTempRoots) +} + +func (m *fileUploadManager) removeTempRoot(tempRoot string) { + if err := m.fileEditor.RemoveAll(tempRoot); err != nil && !m.fileEditor.IsNotExist(err) && m.log != nil { + m.log.Warn("Failed to remove upload temp directory", logger.Field{Key: "path", Value: tempRoot}, logger.Field{Key: "error", Value: err}) + } + + if err := m.unregisterTempRoot(tempRoot); err != nil && m.log != nil { + m.log.Warn("Failed to update upload temp registry", logger.Field{Key: "path", Value: tempRoot}, logger.Field{Key: "error", Value: err}) + } +} + +func (m *fileUploadManager) readRegisteredTempRoots() ([]string, error) { + content, err := os.ReadFile(m.registryPath) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + + return nil, err + } + + var tempRoots []string + if err := json.Unmarshal(content, &tempRoots); err != nil { + return []string{}, nil + } + + return tempRoots, nil +} + +func (m *fileUploadManager) writeRegisteredTempRoots(tempRoots []string) error { + if err := os.MkdirAll(filepath.Dir(m.registryPath), 0700); err != nil { + return err + } + + if tempRoots == nil { + tempRoots = []string{} + } + + content, err := json.Marshal(tempRoots) + if err != nil { + return err + } + + return os.WriteFile(m.registryPath, content, 0600) +} + +func cleanUploadFiles(req CreateFileUploadRequest) ([]cleanUploadFile, error) { + if len(req.Files) == 0 { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "At least one file is required") + } + + if req.ChunkSize <= 0 { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "chunk_size must be greater than zero") + } + + cleanFiles := make([]cleanUploadFile, 0, len(req.Files)) + seenRelativePaths := map[string]bool{} + for _, file := range req.Files { + cleanRelativePath, err := cleanUploadRelativePath(file.RelativePath) + if err != nil { + return nil, err + } + + relativeKey := normalizeUploadReservationPath(cleanRelativePath) + if seenRelativePaths[relativeKey] { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Duplicate upload relative path: "+file.RelativePath) + } + + if file.Size < 0 { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload file size cannot be negative") + } + + clientFileID := strings.TrimSpace(file.ClientFileID) + if clientFileID == "" { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload client file ID is required") + } + + seenRelativePaths[relativeKey] = true + cleanFiles = append(cleanFiles, cleanUploadFile{ + ClientFileID: clientFileID, + OriginalRelativePath: file.RelativePath, + CleanRelativePath: cleanRelativePath, + Size: file.Size, + }) + } + + return cleanFiles, nil +} + +func cleanUploadRelativePath(relativePath string) (string, error) { + trimmedPath := strings.TrimSpace(relativePath) + if trimmedPath == "" { + return "", newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload relative path is required") + } + + normalizedPath := strings.ReplaceAll(trimmedPath, "\\", string(filepath.Separator)) + normalizedPath = filepath.FromSlash(normalizedPath) + cleanPath := filepath.Clean(normalizedPath) + if cleanPath == "." || filepath.IsAbs(cleanPath) || filepath.VolumeName(cleanPath) != "" { + return "", newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload relative path is invalid") + } + + parts := strings.Split(cleanPath, string(filepath.Separator)) + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return "", newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Upload relative path is invalid") + } + } + + return filepath.Join(parts...), nil +} + +func resolveUploadRelativePath(cleanRelativePath string, topLevelResolvedNames map[string]string) string { + parts := strings.Split(cleanRelativePath, string(filepath.Separator)) + if len(parts) == 0 { + return cleanRelativePath + } + + resolvedTopLevel, ok := topLevelResolvedNames[parts[0]] + if !ok { + return cleanRelativePath + } + + parts[0] = resolvedTopLevel + return filepath.Join(parts...) +} + +func uploadTopLevelName(cleanRelativePath string) string { + parts := strings.Split(cleanRelativePath, string(filepath.Separator)) + return parts[0] +} + +func splitUploadFileName(name string) (string, string) { + lastDotIndex := strings.LastIndex(name, ".") + if lastDotIndex <= 0 { + return name, "" + } + + return name[:lastDotIndex], name[lastDotIndex:] +} + +func totalUploadChunks(size int64, chunkSize int64) int { + if size == 0 { + return 0 + } + + return int(math.Ceil(float64(size) / float64(chunkSize))) +} + +func expectedUploadChunkSize(size int64, chunkSize int64, chunkIndex int) int64 { + offset := int64(chunkIndex) * chunkSize + remaining := size - offset + if remaining < chunkSize { + return remaining + } + + return chunkSize +} + +func hashUploadFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { + _ = file.Close() + }() + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func generateUploadID() (string, error) { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return "", err + } + + return hex.EncodeToString(raw), nil +} + +func normalizeUploadReservationPath(path string) string { + return strings.ToLower(filepath.Clean(path)) +} + +func classifyFileUploadError(err error, prefix string) error { + if os.IsPermission(err) { + return newFileUploadHTTPError(http.StatusForbidden, constants.ErrorCodeForbidden, prefix+": permission denied") + } + + if errors.Is(err, syscall.ENOSPC) || strings.Contains(strings.ToLower(err.Error()), "no space") || strings.Contains(strings.ToLower(err.Error()), "disk full") { + return newFileUploadHTTPError(http.StatusInsufficientStorage, constants.ErrorCodeDiskFull, prefix+": not enough disk space") + } + + return newFileUploadHTTPError(http.StatusInternalServerError, constants.ErrorCodeInternalServerError, prefix+": "+err.Error()) +} + +func writeFileUploadError(w http.ResponseWriter, err *fileUploadHTTPError) { + _ = utils.WriteJSONResponseWithStatus(w, err.status, map[string]interface{}{ + "errorCode": err.errorCode, + "context": "file-upload", + "errors": []string{err.message}, + }) +} + +func fileUploadErrorFor(err error) *fileUploadHTTPError { + var uploadErr *fileUploadHTTPError + if errors.As(err, &uploadErr) { + return uploadErr + } + + return newFileUploadHTTPError(http.StatusInternalServerError, constants.ErrorCodeInternalServerError, err.Error()) +} + +func newFileUploadHTTPError(status int, errorCode string, message string) *fileUploadHTTPError { + return &fileUploadHTTPError{ + status: status, + errorCode: errorCode, + message: message, + } +} + +func (e *fileUploadHTTPError) Error() string { + return e.message +} + +type fileUploadHTTPError struct { + status int + errorCode string + message string +} + +type cleanUploadFile struct { + ClientFileID string + OriginalRelativePath string + CleanRelativePath string + Size int64 +} + +type CreateFileUploadRequest struct { + DestinationPath string `json:"destination_path" validate:"required"` + ChunkSize int64 `json:"chunk_size" validate:"required,min=1"` + Files []CreateFileUploadRequestFile `json:"files" validate:"required,min=1,dive"` +} + +type CreateFileUploadRequestFile struct { + ClientFileID string `json:"client_file_id" validate:"required"` + RelativePath string `json:"relative_path" validate:"required"` + Size int64 `json:"size" validate:"min=0"` +} + +type CreateFileUploadResponse struct { + UploadID string `json:"upload_id"` + ExpiresAt string `json:"expires_at"` + Files []CreateFileUploadResponseFile `json:"files"` +} + +type CreateFileUploadResponseFile struct { + ClientFileID string `json:"client_file_id"` + FileID string `json:"file_id"` + RelativePath string `json:"relative_path"` + ResolvedRelativePath string `json:"resolved_relative_path"` + TargetPath string `json:"target_path"` + Size int64 `json:"size"` + ChunkSize int64 `json:"chunk_size"` + TotalChunks int `json:"total_chunks"` +} + +type FileUploadChunkResponse struct { + Message string `json:"message"` + ReceivedChunks int `json:"received_chunks"` + TotalChunks int `json:"total_chunks"` +} + +type CompleteFileUploadRequest struct { + SHA256 string `json:"sha256" validate:"required,len=64,hexadecimal"` +} + +type CompleteFileUploadResponse struct { + Message string `json:"message"` + FileID string `json:"file_id"` + RelativePath string `json:"relative_path"` + ResolvedRelativePath string `json:"resolved_relative_path"` + FinalPath string `json:"final_path"` + SHA256 string `json:"sha256"` +} + +type FileUploadHeartbeatResponse struct { + UploadID string `json:"upload_id"` + ExpiresAt string `json:"expires_at"` +} diff --git a/internal/server/file_upload_routes_test.go b/internal/server/file_upload_routes_test.go new file mode 100644 index 0000000..b0a31c7 --- /dev/null +++ b/internal/server/file_upload_routes_test.go @@ -0,0 +1,275 @@ +package server + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/omnihance/omnihance-a3-agent/internal/config" + "github.com/omnihance/omnihance-a3-agent/internal/constants" + "github.com/omnihance/omnihance-a3-agent/internal/services" + "github.com/omnihance/omnihance-a3-agent/internal/utils" + "github.com/stretchr/testify/require" +) + +func TestCreateFileUploadRejectsViewerAndRootDestination(t *testing.T) { + server := newFileUploadTestServer(t) + + req := fileUploadRequest(t, http.MethodPost, "/api/file-tree/uploads", CreateFileUploadRequest{ + DestinationPath: t.TempDir(), + ChunkSize: 4, + Files: []CreateFileUploadRequestFile{ + {ClientFileID: "file-1", RelativePath: "server.txt", Size: 4}, + }, + }, constants.RoleUser) + rr := httptest.NewRecorder() + server.handleCreateFileUpload(rr, req) + require.Equal(t, http.StatusForbidden, rr.Code) + + req = fileUploadRequest(t, http.MethodPost, "/api/file-tree/uploads", CreateFileUploadRequest{ + DestinationPath: "", + ChunkSize: 4, + Files: []CreateFileUploadRequestFile{ + {ClientFileID: "file-1", RelativePath: "server.txt", Size: 4}, + }, + }, constants.RoleAdmin) + rr = httptest.NewRecorder() + server.handleCreateFileUpload(rr, req) + require.Equal(t, http.StatusBadRequest, rr.Code) +} + +func TestFileUploadSessionReservesDuplicateNamesAcrossTabs(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + content := []byte("large upload body") + + first := createUploadSessionForTest(t, manager, destination, "patch.bin", int64(len(content)), 5) + second := createUploadSessionForTest(t, manager, destination, "patch.bin", int64(len(content)), 5) + + require.Equal(t, "patch.bin", first.Files[0].ResolvedRelativePath) + require.Equal(t, "patch (copy).bin", second.Files[0].ResolvedRelativePath) + + completeUploadForTest(t, manager, first, content) + completeUploadForTest(t, manager, second, content) + + require.FileExists(t, filepath.Join(destination, "patch.bin")) + require.FileExists(t, filepath.Join(destination, "patch (copy).bin")) +} + +func TestFileUploadDirectoryTopLevelConflictUsesCopyName(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(destination, "Data"), 0755)) + + response := createUploadSessionForTest(t, manager, destination, filepath.Join("Data", "zone.txt"), 4, 4) + + require.Equal(t, filepath.Join("Data (copy)", "zone.txt"), response.Files[0].ResolvedRelativePath) +} + +func TestFileUploadDirectoryTopLevelReservationAcrossSessions(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + + first := createUploadSessionForTest(t, manager, destination, filepath.Join("Data", "a.txt"), 1, 1) + second := createUploadSessionForTest(t, manager, destination, filepath.Join("Data", "b.txt"), 1, 1) + + require.Equal(t, filepath.Join("Data", "a.txt"), first.Files[0].ResolvedRelativePath) + require.Equal(t, filepath.Join("Data (copy)", "b.txt"), second.Files[0].ResolvedRelativePath) +} + +func TestFileUploadCopyNameUsesCopyTwoAfterCopyExists(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(destination, "server.txt"), []byte("original"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(destination, "server (copy).txt"), []byte("copy"), 0600)) + + response := createUploadSessionForTest(t, manager, destination, "server.txt", 1, 1) + + require.Equal(t, "server (copy 2).txt", response.Files[0].ResolvedRelativePath) +} + +func TestFileUploadConcurrentSessionsRegisterAllTempRoots(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + sessionCount := 12 + errors := make(chan error, sessionCount) + var wg sync.WaitGroup + + for index := 0; index < sessionCount; index++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + + _, err := manager.CreateSession(CreateFileUploadRequest{ + DestinationPath: destination, + ChunkSize: 1, + Files: []CreateFileUploadRequestFile{ + { + ClientFileID: "file-1", + RelativePath: "server-" + strconv.Itoa(index) + ".txt", + Size: 1, + }, + }, + }) + errors <- err + }(index) + } + + wg.Wait() + close(errors) + for err := range errors { + require.NoError(t, err) + } + + tempRoots, err := manager.readRegisteredTempRoots() + require.NoError(t, err) + require.Len(t, tempRoots, sessionCount) +} + +func TestFileUploadCancelRemovesTempAndReleasesReservation(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + content := []byte("cancel me") + response := createUploadSessionForTest(t, manager, destination, "server.txt", int64(len(content)), 4) + + uploadChunksForTest(t, manager, response, content) + tempRoot := manager.sessions[response.UploadID].TempRoot + require.DirExists(t, tempRoot) + + require.NoError(t, manager.CancelSession(response.UploadID)) + require.NoDirExists(t, tempRoot) + + next := createUploadSessionForTest(t, manager, destination, "server.txt", int64(len(content)), 4) + require.Equal(t, "server.txt", next.Files[0].ResolvedRelativePath) +} + +func TestFileUploadExpiredCleanupRemovesTempAndReleasesReservation(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + response := createUploadSessionForTest(t, manager, destination, "server.txt", 4, 4) + tempRoot := manager.sessions[response.UploadID].TempRoot + require.DirExists(t, tempRoot) + + manager.sessions[response.UploadID].LastSeenAt = time.Now().Add(-fileUploadSessionTTL - time.Second) + manager.cleanupExpiredSessions(time.Now()) + + require.NoDirExists(t, tempRoot) + _, ok := manager.sessions[response.UploadID] + require.False(t, ok) + + next := createUploadSessionForTest(t, manager, destination, "server.txt", 4, 4) + require.Equal(t, "server.txt", next.Files[0].ResolvedRelativePath) +} + +func TestFileUploadHashMismatchRemovesTempAndReleasesReservation(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + content := []byte("corrupt check") + response := createUploadSessionForTest(t, manager, destination, "server.txt", int64(len(content)), 4) + + uploadChunksForTest(t, manager, response, content) + tempPath := manager.sessions[response.UploadID].Files[response.Files[0].FileID].TempPath + require.FileExists(t, tempPath) + + _, err := manager.CompleteFile(response.UploadID, response.Files[0].FileID, strings.Repeat("0", 64)) + require.Error(t, err) + var uploadErr *fileUploadHTTPError + require.ErrorAs(t, err, &uploadErr) + require.Equal(t, http.StatusConflict, uploadErr.status) + require.NoFileExists(t, tempPath) + + next := createUploadSessionForTest(t, manager, destination, "server.txt", int64(len(content)), 4) + require.Equal(t, "server.txt", next.Files[0].ResolvedRelativePath) +} + +func TestFileUploadCompleteRenamesWhenFinalPathAppearsExternally(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + content := []byte("server data") + response := createUploadSessionForTest(t, manager, destination, "server.txt", int64(len(content)), 4) + + uploadChunksForTest(t, manager, response, content) + require.NoError(t, os.WriteFile(filepath.Join(destination, "server.txt"), []byte("external"), 0600)) + + complete := completeUploadForTest(t, manager, response, content) + + require.Equal(t, filepath.Join(destination, "server (copy).txt"), complete.FinalPath) + require.FileExists(t, filepath.Join(destination, "server.txt")) + require.FileExists(t, filepath.Join(destination, "server (copy).txt")) +} + +func newFileUploadTestServer(t *testing.T) *Server { + t.Helper() + + return &Server{ + cfg: &config.EnvVars{CookieSecret: "test-secret", RevisionsDirectory: t.TempDir()}, + fileEditor: services.NewFileEditorService(nil), + } +} + +func newFileUploadTestManager(t *testing.T) *fileUploadManager { + t.Helper() + + return newFileUploadManager(t.TempDir(), services.NewFileEditorService(nil), nil) +} + +func fileUploadRequest(t *testing.T, method string, target string, body any, role string) *http.Request { + t.Helper() + + var payload bytes.Buffer + require.NoError(t, json.NewEncoder(&payload).Encode(body)) + req := httptest.NewRequest(method, target, &payload) + ctx := utils.SetUserRolesInContext(req.Context(), []string{role}) + return req.WithContext(ctx) +} + +func createUploadSessionForTest(t *testing.T, manager *fileUploadManager, destination string, relativePath string, size int64, chunkSize int64) *CreateFileUploadResponse { + t.Helper() + + response, err := manager.CreateSession(CreateFileUploadRequest{ + DestinationPath: destination, + ChunkSize: chunkSize, + Files: []CreateFileUploadRequestFile{ + {ClientFileID: "file-1", RelativePath: relativePath, Size: size}, + }, + }) + require.NoError(t, err) + require.Len(t, response.Files, 1) + return response +} + +func uploadChunksForTest(t *testing.T, manager *fileUploadManager, response *CreateFileUploadResponse, content []byte) { + t.Helper() + + file := response.Files[0] + for chunkIndex := 0; chunkIndex < file.TotalChunks; chunkIndex++ { + start := int64(chunkIndex) * file.ChunkSize + end := start + file.ChunkSize + if end > int64(len(content)) { + end = int64(len(content)) + } + + _, err := manager.UploadChunk(response.UploadID, file.FileID, chunkIndex, bytes.NewReader(content[start:end])) + require.NoError(t, err) + } +} + +func completeUploadForTest(t *testing.T, manager *fileUploadManager, response *CreateFileUploadResponse, content []byte) *CompleteFileUploadResponse { + t.Helper() + + uploadChunksForTest(t, manager, response, content) + hash := sha256.Sum256(content) + complete, err := manager.CompleteFile(response.UploadID, response.Files[0].FileID, hex.EncodeToString(hash[:])) + require.NoError(t, err) + return complete +} diff --git a/internal/server/server.go b/internal/server/server.go index d8ac864..8018bd5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -25,6 +25,7 @@ type Server struct { versionChecker services.VersionCheckerService backupService services.BackupService serverViewService services.ServerViewService + uploadManager *fileUploadManager } func NewServer( @@ -56,6 +57,11 @@ func NewServer( serverViewService: serverViewService, } + newServer.uploadManager = newFileUploadManager(cfg.RevisionsDirectory, fileEditor, log) + if err := newServer.uploadManager.Start(); err != nil && log != nil { + log.Error("Could not start file upload manager", logger.Field{Key: "error", Value: err}) + } + server := &http.Server{ Addr: fmt.Sprintf(":%s", newServer.cfg.Port), Handler: newServer.RegisterRoutes(), @@ -64,6 +70,7 @@ func NewServer( WriteTimeout: 30 * time.Second, ReadHeaderTimeout: 9 * time.Minute, } + server.RegisterOnShutdown(newServer.uploadManager.Stop) return server } From 4936ec55eb465a5098d12e53814c9aafe6a00d2f Mon Sep 17 00:00:00 2001 From: cyberinferno Date: Tue, 23 Jun 2026 14:21:10 +0530 Subject: [PATCH 2/3] fix: validate file upload chunks and restart manager cleanly --- AGENTS.md | 6 +- cmd/omnihance-a3-agent/docs/openapi.yml | 2 + .../src/components/file-browser-uploader.tsx | 51 ++++++------- .../omnihance-a3-agent-ui/src/lib/api.ts | 6 +- internal/server/file_upload_routes.go | 72 +++++++++++++++---- internal/server/file_upload_routes_test.go | 30 +++++++- 6 files changed, 123 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 19f7d8c..ac12dbb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,5 +132,7 @@ For multi-step tasks, state a brief plan: ### Workflow Guidelines -- You have access to Github CLI. Use it for all Github related actions. -- Before starting to implement anything please check the active branch. If its `master` or `main` pull the latest changes. Then fork a new branch from it with naming convention like `feat/{some-feature}` or `fix/{some-fix}` or `docs/{some-docs}` etc. Keep branch name short but meaningful with conventional commit prefix. If its already in non default branch do not do anything. +- You have access to GitHub CLI. Use it for all GitHub related actions. +- Before implementing, check the active branch: + 1. On `master` or `main`: pull latest, then create a short branch named `feat/{feature}`, `fix/{fix}`, or `docs/{docs}`. + 2. Already on a non-default branch: no action needed. diff --git a/cmd/omnihance-a3-agent/docs/openapi.yml b/cmd/omnihance-a3-agent/docs/openapi.yml index 8d59b9b..28f600d 100644 --- a/cmd/omnihance-a3-agent/docs/openapi.yml +++ b/cmd/omnihance-a3-agent/docs/openapi.yml @@ -5171,6 +5171,7 @@ components: type: integer format: int64 minimum: 1 + maximum: 8388608 description: Chunk size in bytes that the client will use for each uploaded file. example: 4194304 files: @@ -5274,6 +5275,7 @@ components: type: string minLength: 64 maxLength: 64 + pattern: '^[a-f0-9]{64}$' description: Lowercase hexadecimal SHA-256 hash of the original browser file. example: '6d37795021e544d53d2569bb6139720ee0869d1111d924aed610ea148a84efcf' CompleteFileUploadResponse: diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx index 3666ee6..3fa0948 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/file-browser-uploader.tsx @@ -211,6 +211,22 @@ export function useFileBrowserUploader({ [enqueueUpload], ); + const enqueueDroppedDataTransfer = useCallback( + async (dataTransfer: DataTransfer) => { + try { + const sources = await uploadSourcesFromDataTransfer(dataTransfer); + enqueueUpload(sources); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : 'Failed to read dropped files', + ); + } + }, + [enqueueUpload], + ); + const handleDrop = useCallback( async (event: React.DragEvent) => { if (!canUpload) { @@ -221,18 +237,9 @@ export function useFileBrowserUploader({ event.stopPropagation(); setIsDragging(false); - try { - const sources = await uploadSourcesFromDataTransfer(event.dataTransfer); - enqueueUpload(sources); - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : 'Failed to read dropped files', - ); - } + await enqueueDroppedDataTransfer(event.dataTransfer); }, - [canUpload, enqueueUpload], + [canUpload, enqueueDroppedDataTransfer], ); const dropHandlers = useMemo( @@ -338,18 +345,7 @@ export function useFileBrowserUploader({ onDrop={async (event) => { event.preventDefault(); event.stopPropagation(); - try { - const sources = await uploadSourcesFromDataTransfer( - event.dataTransfer, - ); - enqueueUpload(sources); - } catch (error) { - toast.error( - error instanceof Error - ? error.message - : 'Failed to read dropped files', - ); - } + await enqueueDroppedDataTransfer(event.dataTransfer); }} >
@@ -822,11 +823,11 @@ function dedupeUploadSources(sources: UploadSource[]) { const result: UploadSource[] = []; for (const source of sources) { const relativePath = normalizeRelativeUploadPath(source.relativePath); - if (!relativePath || seen.has(relativePath.toLowerCase())) { + if (!relativePath || seen.has(relativePath)) { continue; } - seen.add(relativePath.toLowerCase()); + seen.add(relativePath); result.push({ file: source.file, relativePath, diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts index 9b8f047..901bb33 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/api.ts @@ -3,6 +3,8 @@ import type { AxiosError } from 'axios'; import type { EChartsOption } from 'echarts'; import { z } from 'zod'; +const maxFileUploadChunkSize = 8 * 1024 * 1024; + export const API_ROUTES = { AUTH_SIGN_IN: '/api/auth/sign-in', AUTH_SIGN_UP: '/api/auth/sign-up', @@ -391,7 +393,7 @@ const CreateFileUploadRequestFileSchema = z.object({ const CreateFileUploadRequestSchema = z.object({ destination_path: z.string().min(1), - chunk_size: z.number().int().positive(), + chunk_size: z.number().int().positive().max(maxFileUploadChunkSize), files: z.array(CreateFileUploadRequestFileSchema).min(1), }); @@ -431,7 +433,7 @@ export type FileUploadChunkResponse = z.infer< >; const CompleteFileUploadRequestSchema = z.object({ - sha256: z.string().length(64), + sha256: z.string().regex(/^[a-f0-9]{64}$/), }); const CompleteFileUploadResponseSchema = z.object({ diff --git a/internal/server/file_upload_routes.go b/internal/server/file_upload_routes.go index 982949b..9ea523b 100644 --- a/internal/server/file_upload_routes.go +++ b/internal/server/file_upload_routes.go @@ -34,6 +34,7 @@ const ( fileUploadCleanupInterval = 15 * time.Minute fileUploadDefaultRegistryDirName = ".revisions" fileUploadRegistryFileName = "file-upload-temp-registry.json" + fileUploadMaxChunkSize = 8 * 1024 * 1024 ) type fileUploadManager struct { @@ -46,6 +47,7 @@ type fileUploadManager struct { sessions map[string]*fileUploadSession reservations map[string]string stopCh chan struct{} + stopDoneCh chan struct{} started bool } @@ -88,7 +90,6 @@ func newFileUploadManager(registryDir string, fileEditor services.FileEditorServ registryPath: filepath.Join(registryDir, fileUploadRegistryFileName), sessions: make(map[string]*fileUploadSession), reservations: make(map[string]string), - stopCh: make(chan struct{}), } } @@ -99,14 +100,29 @@ func (m *fileUploadManager) Start() error { return nil } - m.started = true m.mu.Unlock() if err := m.cleanupRegisteredTempRoots(); err != nil { return err } + stopCh := make(chan struct{}) + stopDoneCh := make(chan struct{}) + + m.mu.Lock() + if m.started { + m.mu.Unlock() + return nil + } + + m.stopCh = stopCh + m.stopDoneCh = stopDoneCh + m.started = true + m.mu.Unlock() + go func() { + defer close(stopDoneCh) + ticker := time.NewTicker(fileUploadCleanupInterval) defer ticker.Stop() @@ -114,7 +130,7 @@ func (m *fileUploadManager) Start() error { select { case <-ticker.C: m.cleanupExpiredSessions(time.Now()) - case <-m.stopCh: + case <-stopCh: return } } @@ -130,10 +146,15 @@ func (m *fileUploadManager) Stop() { return } + stopCh := m.stopCh + stopDoneCh := m.stopDoneCh m.started = false - close(m.stopCh) - m.stopCh = make(chan struct{}) + m.stopCh = nil + m.stopDoneCh = nil + close(stopCh) m.mu.Unlock() + + <-stopDoneCh } func (s *Server) ensureUploadManager() *fileUploadManager { @@ -413,40 +434,59 @@ func (m *fileUploadManager) UploadChunk(uploadID string, fileID string, chunkInd } expectedSize := expectedUploadChunkSize(file.Size, file.ChunkSize, chunkIndex) + chunkSize := file.ChunkSize + tempPath := file.TempPath + tempRoot := session.TempRoot + m.mu.Unlock() + chunkData, err := io.ReadAll(io.LimitReader(body, expectedSize+1)) if err != nil { - m.mu.Unlock() return nil, classifyFileUploadError(err, "Failed to read upload chunk") } if int64(len(chunkData)) != expectedSize { - m.mu.Unlock() return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Chunk size does not match expected size") } - if err := m.fileEditor.MkdirAll(filepath.Dir(file.TempPath), 0700); err != nil { - m.mu.Unlock() + if err := m.fileEditor.MkdirAll(filepath.Dir(tempPath), 0700); err != nil { return nil, classifyFileUploadError(err, "Failed to create upload temp directory") } - tempFile, err := m.fileEditor.OpenFile(file.TempPath, os.O_CREATE|os.O_WRONLY, 0600) + tempFile, err := m.fileEditor.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { - m.mu.Unlock() return nil, classifyFileUploadError(err, "Failed to open upload temp file") } - _, writeErr := tempFile.WriteAt(chunkData, int64(chunkIndex)*file.ChunkSize) + _, writeErr := tempFile.WriteAt(chunkData, int64(chunkIndex)*chunkSize) closeErr := tempFile.Close() if writeErr != nil { - m.mu.Unlock() return nil, classifyFileUploadError(writeErr, "Failed to write upload chunk") } if closeErr != nil { - m.mu.Unlock() return nil, classifyFileUploadError(closeErr, "Failed to close upload temp file") } + m.mu.Lock() + session, file, err = m.activeFileLocked(uploadID, fileID, time.Now()) + if err != nil { + m.mu.Unlock() + m.removeTempRoot(tempRoot) + return nil, err + } + + if file.Completed { + m.mu.Unlock() + _ = m.fileEditor.Remove(tempPath) + return nil, newFileUploadHTTPError(http.StatusConflict, constants.ErrorCodeBadRequest, "File is already complete") + } + + if chunkIndex >= file.TotalChunks { + m.mu.Unlock() + _ = m.fileEditor.Remove(tempPath) + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "Chunk index is out of range") + } + file.ReceivedChunks[chunkIndex] = expectedSize session.LastSeenAt = time.Now() session.ExpiresAt = session.LastSeenAt.Add(fileUploadSessionTTL) @@ -849,6 +889,10 @@ func cleanUploadFiles(req CreateFileUploadRequest) ([]cleanUploadFile, error) { return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "chunk_size must be greater than zero") } + if req.ChunkSize > fileUploadMaxChunkSize { + return nil, newFileUploadHTTPError(http.StatusBadRequest, constants.ErrorCodeBadRequest, "chunk_size exceeds maximum allowed size") + } + cleanFiles := make([]cleanUploadFile, 0, len(req.Files)) seenRelativePaths := map[string]bool{} for _, file := range req.Files { diff --git a/internal/server/file_upload_routes_test.go b/internal/server/file_upload_routes_test.go index b0a31c7..976dd8e 100644 --- a/internal/server/file_upload_routes_test.go +++ b/internal/server/file_upload_routes_test.go @@ -15,11 +15,12 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/omnihance/omnihance-a3-agent/internal/config" "github.com/omnihance/omnihance-a3-agent/internal/constants" "github.com/omnihance/omnihance-a3-agent/internal/services" "github.com/omnihance/omnihance-a3-agent/internal/utils" - "github.com/stretchr/testify/require" ) func TestCreateFileUploadRejectsViewerAndRootDestination(t *testing.T) { @@ -48,6 +49,33 @@ func TestCreateFileUploadRejectsViewerAndRootDestination(t *testing.T) { require.Equal(t, http.StatusBadRequest, rr.Code) } +func TestCreateFileUploadRejectsOversizedChunkSize(t *testing.T) { + manager := newFileUploadTestManager(t) + destination := t.TempDir() + + _, err := manager.CreateSession(CreateFileUploadRequest{ + DestinationPath: destination, + ChunkSize: fileUploadMaxChunkSize + 1, + Files: []CreateFileUploadRequestFile{ + {ClientFileID: "file-1", RelativePath: "server.txt", Size: 1}, + }, + }) + + require.Error(t, err) + var uploadErr *fileUploadHTTPError + require.ErrorAs(t, err, &uploadErr) + require.Equal(t, http.StatusBadRequest, uploadErr.status) +} + +func TestFileUploadManagerStartStopCanRestart(t *testing.T) { + manager := newFileUploadTestManager(t) + + require.NoError(t, manager.Start()) + manager.Stop() + require.NoError(t, manager.Start()) + manager.Stop() +} + func TestFileUploadSessionReservesDuplicateNamesAcrossTabs(t *testing.T) { manager := newFileUploadTestManager(t) destination := t.TempDir() From 2a4f330c60f2e22d0bf9330cb4ee387808c6aadb Mon Sep 17 00:00:00 2001 From: cyberinferno Date: Tue, 23 Jun 2026 14:47:09 +0530 Subject: [PATCH 3/3] Document per-file upload size limit --- README.md | 16 +-- cmd/omnihance-a3-agent/docs/openapi.yml | 69 +++++++-- .../src/components/client-data-page.tsx | 9 ++ .../client-data/item-file-upload.tsx | 35 +++-- .../client-data/map-file-upload.tsx | 43 ++++-- .../client-data/monster-file-upload.tsx | 43 ++++-- .../client-data/upload-validation.ts | 13 ++ .../src/components/file-browser-uploader.tsx | 32 ++++- .../src/components/file-tree.tsx | 3 + .../omnihance-a3-agent-ui/src/lib/api.ts | 1 + .../src/lib/upload-validation.ts | 27 ++++ internal/config/config.go | 28 +++- internal/config/config_test.go | 88 ++++++++++++ internal/constants/constants.go | 1 + internal/server/file_upload_routes.go | 24 +++- internal/server/file_upload_routes_test.go | 45 +++++- internal/server/game_client_data_routes.go | 131 +++--------------- .../server/game_client_data_routes_test.go | 48 +++++++ internal/server/server.go | 2 +- internal/server/status_routes.go | 34 ++--- internal/server/status_routes_test.go | 3 +- internal/server/upload_limits.go | 64 +++++++++ 22 files changed, 574 insertions(+), 185 deletions(-) create mode 100644 cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data/upload-validation.ts create mode 100644 cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/lib/upload-validation.ts create mode 100644 internal/config/config_test.go create mode 100644 internal/server/upload_limits.go diff --git a/README.md b/README.md index a068d50..2745a1f 100644 --- a/README.md +++ b/README.md @@ -525,7 +525,7 @@ The application uses environment variables for configuration. A `.env` file is a | `REVISIONS_DIRECTORY` | `.revisions` | Directory for file revision backups | | `BACKUPS_DIRECTORY` | `.backups` | Directory for internal backup lock files | | `DIRECTORY_DOWNLOADS_DIRECTORY` | `.directory-download` | Directory for generated directory-download ZIP archives | -| `MAX_FILE_UPLOAD_SIZE_MB` | `2` | Maximum multipart upload size in MB | +| `MAX_FILE_UPLOAD_SIZE_MB` | `1024` | Maximum per-file upload size in MB for file browser and game client uploads | | `DIRECTORY_SHORTCUTS_LIMIT` | `5` | Maximum pinned directories per user (`0` disables the limit) | | `RUNNING_IN_DOCKER` | `false` | Disable host metrics collection when running in Docker | | `SESSION_TIMEOUT_SECONDS` | `2592000` | Session timeout (30 days) | @@ -584,7 +584,7 @@ Only stable GitHub releases are considered because GitHub's latest release endpo - `PUT /api/file-tree/item-combination-data` - Update A3 item combination data - `POST /api/file-tree/revert-file` - Revert file to previous revision - `POST /api/file-tree/duplicate-file` - Duplicate a file in the same directory -- `POST /api/file-tree/uploads` - Start a file-browser upload batch for a non-root destination directory and reserve conflict-safe target names +- `POST /api/file-tree/uploads` - Start a file-browser upload batch for a non-root destination directory, enforce the configured per-file upload size limit, and reserve conflict-safe target names - `PUT /api/file-tree/uploads/{upload_id}/files/{file_id}/chunks/{chunk_index}` - Upload one binary file chunk with retry support - `POST /api/file-tree/uploads/{upload_id}/files/{file_id}/complete` - Verify SHA-256 and finalize one uploaded file - `POST /api/file-tree/uploads/{upload_id}/heartbeat` - Keep an active upload session alive while the browser tab is open @@ -605,14 +605,14 @@ Only stable GitHub releases are considered because GitHub's latest release endpo - `GET /api/game-client-data/counts` - Get imported record counts for monsters, maps, and item file types - `GET /api/game-client-data/monsters` - Get monster client data (supports optional `s` query parameter for search) -- `POST /api/game-client-data/upload-mon-file` - Upload MON.ull file to populate monster database +- `POST /api/game-client-data/upload-mon-file` - Upload MON.ull file to populate monster database, enforcing the configured per-file upload size limit - `GET /api/game-client-data/maps` - Get map client data (supports optional `s` query parameter for search) -- `POST /api/game-client-data/upload-mc-file` - Upload MC.ull file to populate map database +- `POST /api/game-client-data/upload-mc-file` - Upload MC.ull file to populate map database, enforcing the configured per-file upload size limit - `GET /api/game-client-data/items` - Get item client data (supports optional `s` query parameter for search) -- `POST /api/game-client-data/upload-it0-file` - Upload IT0.ull file to populate item data -- `POST /api/game-client-data/upload-it1-file` - Upload IT1.ull file to populate item data -- `POST /api/game-client-data/upload-it2-file` - Upload IT2.ull file to populate item data -- `POST /api/game-client-data/upload-it3-file` - Upload IT3.ull file to populate item data +- `POST /api/game-client-data/upload-it0-file` - Upload IT0.ull file to populate item data, enforcing the configured per-file upload size limit +- `POST /api/game-client-data/upload-it1-file` - Upload IT1.ull file to populate item data, enforcing the configured per-file upload size limit +- `POST /api/game-client-data/upload-it2-file` - Upload IT2.ull file to populate item data, enforcing the configured per-file upload size limit +- `POST /api/game-client-data/upload-it3-file` - Upload IT3.ull file to populate item data, enforcing the configured per-file upload size limit ### Directory Shortcuts diff --git a/cmd/omnihance-a3-agent/docs/openapi.yml b/cmd/omnihance-a3-agent/docs/openapi.yml index 28f600d..b02f8d0 100644 --- a/cmd/omnihance-a3-agent/docs/openapi.yml +++ b/cmd/omnihance-a3-agent/docs/openapi.yml @@ -227,7 +227,7 @@ paths: tags: - status summary: Get server status - description: Returns the current server status including name, version, setup status, and new version availability. + description: Returns the current server status including name, version, setup status, configured upload size limit, and new version availability. responses: '200': description: Server status retrieved successfully @@ -1367,7 +1367,7 @@ paths: tags: - file-system summary: Start file-browser upload batch - description: Creates a chunked upload session for a non-root destination directory. Requires edit file permission. The server reserves final target names with the existing copy-name pattern so concurrent uploads cannot overwrite each other. + description: Creates a chunked upload session for a non-root destination directory. Requires edit file permission. Each file must not exceed the server's configured per-file upload size limit. The server reserves final target names with the existing copy-name pattern so concurrent uploads cannot overwrite each other. security: - ApiKeyAuth: [] requestBody: @@ -1407,6 +1407,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - One or more files exceed the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '507': description: Insufficient storage while creating upload temp storage content: @@ -1997,7 +2003,7 @@ paths: tags: - game-data summary: Upload monster list client data file - description: Uploads and processes a monster list client data file (MON.ull). The file is decoded using ULL decryption, parsed into structured monster data, and bulk replaces all existing monster client data in the database. The file size must not exceed the maximum upload size configured in the server. All uploaded monsters are associated with the current user as both creator and updater. + description: Uploads and processes a monster list client data file (MON.ull). The file is decoded using ULL decryption, parsed into structured monster data, and bulk replaces all existing monster client data in the database. The file size must not exceed the maximum per-file upload size configured in the server. All uploaded monsters are associated with the current user as both creator and updater. security: - ApiKeyAuth: [] requestBody: @@ -2030,7 +2036,7 @@ paths: description: The number of monster records that were uploaded example: 150 '400': - description: Bad Request - File size exceeds maximum allowed size, failed to parse multipart form, file not found in form, or failed to parse monster file + description: Bad Request - Failed to parse multipart form, file not found in form, or failed to parse monster file content: application/json: schema: @@ -2041,6 +2047,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save monster data content: @@ -2052,7 +2064,7 @@ paths: tags: - game-data summary: Upload map list client data file - description: Uploads and processes a map list client data file (MC.ull). The file is decoded using ULL decryption, parsed into structured map data, and bulk replaces all existing map client data in the database. The file size must not exceed the maximum upload size configured in the server. All uploaded maps are associated with the current user as both creator and updater. + description: Uploads and processes a map list client data file (MC.ull). The file is decoded using ULL decryption, parsed into structured map data, and bulk replaces all existing map client data in the database. The file size must not exceed the maximum per-file upload size configured in the server. All uploaded maps are associated with the current user as both creator and updater. security: - ApiKeyAuth: [] requestBody: @@ -2085,7 +2097,7 @@ paths: description: The number of map list records that were uploaded example: 50 '400': - description: Bad Request - File size exceeds maximum allowed size, failed to parse multipart form, file not found in form, or failed to parse map file + description: Bad Request - Failed to parse multipart form, file not found in form, or failed to parse map file content: application/json: schema: @@ -2096,6 +2108,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save map data content: @@ -2191,7 +2209,7 @@ paths: tags: - game-data summary: Upload IT0 item client data file - description: Uploads and processes an IT0.ull item client data file. The file is decoded using ULL decryption, parsed with agonyl-utils-go itemfile helpers, and replaces existing IT0 item client data only. + description: Uploads and processes an IT0.ull item client data file. The file must not exceed the maximum per-file upload size configured in the server. The file is decoded using ULL decryption, parsed with agonyl-utils-go itemfile helpers, and replaces existing IT0 item client data only. security: - ApiKeyAuth: [] requestBody: @@ -2226,6 +2244,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save item data content: @@ -2237,7 +2261,7 @@ paths: tags: - game-data summary: Upload IT1 item client data file - description: Uploads and processes an IT1.ull item client data file and replaces existing IT1 item client data only. + description: Uploads and processes an IT1.ull item client data file. The file must not exceed the maximum per-file upload size configured in the server and replaces existing IT1 item client data only. security: - ApiKeyAuth: [] requestBody: @@ -2272,6 +2296,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save item data content: @@ -2283,7 +2313,7 @@ paths: tags: - game-data summary: Upload IT2 item client data file - description: Uploads and processes an IT2.ull item client data file and replaces existing IT2 item client data only. + description: Uploads and processes an IT2.ull item client data file. The file must not exceed the maximum per-file upload size configured in the server and replaces existing IT2 item client data only. security: - ApiKeyAuth: [] requestBody: @@ -2318,6 +2348,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save item data content: @@ -2329,7 +2365,7 @@ paths: tags: - game-data summary: Upload IT3 item client data file - description: Uploads and processes an IT3.ull item client data file and replaces existing IT3 item client data only. + description: Uploads and processes an IT3.ull item client data file. The file must not exceed the maximum per-file upload size configured in the server and replaces existing IT3 item client data only. security: - ApiKeyAuth: [] requestBody: @@ -2364,6 +2400,12 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + '413': + description: Request Entity Too Large - File exceeds the configured per-file upload size limit + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' '500': description: Internal server error - Failed to read file or save item data content: @@ -4843,6 +4885,11 @@ components: type: boolean description: Whether metrics collection is enabled example: true + max_file_upload_size_bytes: + type: integer + format: int64 + description: Configured maximum upload size per file in bytes + example: 1073741824 ErrorResponse: type: object properties: @@ -5198,7 +5245,7 @@ components: type: integer format: int64 minimum: 0 - description: File size in bytes. + description: File size in bytes. Must not exceed the server's configured per-file upload size limit. example: 7340032 CreateFileUploadResponse: type: object diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data-page.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data-page.tsx index 9d689d3..4285884 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data-page.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data-page.tsx @@ -11,10 +11,13 @@ import { uploadIt2File, uploadIt3File, } from '@/lib/api'; +import { useStatus } from '@/hooks/use-status'; export function ClientDataPage() { const { hasPermission } = usePermissions(); + const { status } = useStatus(); const canUploadGameData = hasPermission('upload_game_data'); + const maxFileUploadSizeBytes = status?.max_file_upload_size_bytes; const { data: counts, isLoading: countsLoading, @@ -40,11 +43,13 @@ export function ClientDataPage() { existingCount={counts?.monsters} countLoading={countsLoading} countError={countsError} + maxFileUploadSizeBytes={maxFileUploadSizeBytes} />
) : ( diff --git a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data/item-file-upload.tsx b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data/item-file-upload.tsx index fd17f19..3781149 100644 --- a/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data/item-file-upload.tsx +++ b/cmd/omnihance-a3-agent/omnihance-a3-agent-ui/src/components/client-data/item-file-upload.tsx @@ -11,9 +11,10 @@ import { import { Button } from '@/components/ui/button'; import { Alert, AlertDescription } from '@/components/ui/alert'; import { APIError, type UploadFileResponse } from '@/lib/api'; -import { cn } from '@/lib/util'; +import { cn, formatBytes } from '@/lib/util'; import { queryKeys } from '@/constants'; import { ClientDataCountBadge } from '@/components/client-data/client-data-count-badge'; +import { validateGameClientUploadFile } from '@/components/client-data/upload-validation'; type ItemFileUploadProps = { fileLabel: string; @@ -21,6 +22,7 @@ type ItemFileUploadProps = { countLoading: boolean; countError: boolean; uploadFile: (file: File) => Promise; + maxFileUploadSizeBytes?: number; }; export function ItemFileUpload({ @@ -29,6 +31,7 @@ export function ItemFileUpload({ countLoading, countError, uploadFile, + maxFileUploadSizeBytes, }: ItemFileUploadProps) { const [file, setFile] = useState(null); const [isDragging, setIsDragging] = useState(false); @@ -40,6 +43,7 @@ export function ItemFileUpload({ mutationFn: uploadFile, onSuccess: () => { setFile(null); + setValidationError(null); if (fileInputRef.current) { fileInputRef.current.value = ''; } @@ -74,13 +78,7 @@ export function ItemFileUpload({ return; } - if (!droppedFile.name.toLowerCase().endsWith('.ull')) { - setValidationError(`Please select a valid ${fileLabel}.ull file.`); - return; - } - - setValidationError(null); - setFile(droppedFile); + selectFile(droppedFile); }; const handleFileSelect = (e: React.ChangeEvent) => { @@ -89,8 +87,22 @@ export function ItemFileUpload({ return; } - if (!selectedFile.name.toLowerCase().endsWith('.ull')) { - setValidationError(`Please select a valid ${fileLabel}.ull file.`); + selectFile(selectedFile); + }; + + const selectFile = (selectedFile: File) => { + uploadMutation.reset(); + const errorMessage = validateGameClientUploadFile( + selectedFile, + fileLabel, + maxFileUploadSizeBytes, + ); + if (errorMessage) { + setFile(null); + setValidationError(errorMessage); + if (fileInputRef.current) { + fileInputRef.current.value = ''; + } return; } @@ -111,6 +123,7 @@ export function ItemFileUpload({ const resetFile = () => { setFile(null); setValidationError(null); + uploadMutation.reset(); if (fileInputRef.current) { fileInputRef.current.value = ''; } @@ -197,7 +210,7 @@ export function ItemFileUpload({ {file.name} - ({(file.size / 1024).toFixed(2)} KB) + ({formatBytes(file.size)})